diff --git a/.vscode/settings.json b/.vscode/settings.json index b55aba3..f44cbc2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,7 @@ { + "rust-analyzer.cargo.target": "riscv32imac-unknown-xous-elf", "rust-analyzer.diagnostics.disabled": [ "macro-error" ] } + diff --git a/curve25519-dalek/Cargo.toml b/curve25519-dalek/Cargo.toml index 30868cf..74054d4 100644 --- a/curve25519-dalek/Cargo.toml +++ b/curve25519-dalek/Cargo.toml @@ -62,6 +62,8 @@ engine25519-as = {git = "https://github.com/betrusted-io/engine25519-as.git", re utralib = {version = "0.1.24", default-features = false} zeroize = { version = "1", default-features = false } xous = "0.9.58" +# for fallback when hardware is unavailable +fiat-crypto = { version = "0.2.1", default-features = false} [target.'cfg(target_arch = "x86_64")'.dependencies] cpufeatures = "0.2.6" @@ -69,7 +71,9 @@ cpufeatures = "0.2.6" fiat-crypto = { version = "0.2.1", default-features = false , optional = true} [features] -default = ["alloc", "precomputed-tables", "zeroize"] +auto-release = [] +warn-fallback = [] +default = ["alloc", "precomputed-tables", "zeroize", "auto-release", "warn-fallback"] alloc = ["zeroize?/alloc"] precomputed-tables = [] legacy_compatibility = [] diff --git a/curve25519-dalek/src/backend/serial/u32e/field.rs b/curve25519-dalek/src/backend/serial/u32e/field.rs index 85a560b..e7153ac 100644 --- a/curve25519-dalek/src/backend/serial/u32e/field.rs +++ b/curve25519-dalek/src/backend/serial/u32e/field.rs @@ -24,8 +24,34 @@ use core::ops::{Sub, SubAssign}; use subtle::Choice; use subtle::ConditionallySelectable; +use core::fmt::Debug; +use fiat_crypto::curve25519_32::*; use zeroize::Zeroize; +#[derive(Copy, Clone)] +pub struct FieldElement2625(pub(crate) fiat_25519_tight_field_element); + +impl Debug for FieldElement2625 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "FieldElement2625({:?})", &(self.0).0[..]) + } +} + +#[cfg(feature = "zeroize")] +impl Zeroize for FieldElement2625 { + fn zeroize(&mut self) { + (self.0).0.zeroize(); + } +} + +impl FieldElement2625 { + pub(crate) const fn from_limbs(limbs: [u32; 10]) -> FieldElement2625 { + FieldElement2625(fiat_25519_tight_field_element(limbs)) + } + + pub const ZERO: FieldElement2625 = FieldElement2625::from_limbs([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); +} + /// A `Engine25519` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// @@ -50,102 +76,180 @@ use zeroize::Zeroize; /// outside of the `curve25519_dalek::field` module. #[derive(Copy, Clone, Debug)] -pub struct Engine25519( - pub (crate) [u8; 32] -); +pub struct Engine25519(pub(crate) [u8; 32]); pub(crate) enum EngineOp { Mul, Add, Sub, } +pub fn bytes_to_fiat(data: &[u8; 32]) -> FieldElement2625 { + let mut temp = [0u8; 32]; + temp.copy_from_slice(data); + temp[31] &= 127u8; + let mut output = fiat_25519_tight_field_element([0u32; 10]); + fiat_25519_from_bytes(&mut output, &temp); + FieldElement2625(output) +} + +pub fn fiat_to_bytes(fiat_rep: &fiat_25519_tight_field_element) -> [u8; 32] { + let mut bytes = [0u8; 32]; + fiat_25519_to_bytes(&mut bytes, fiat_rep); + bytes +} + #[allow(unused_qualifications)] pub(crate) fn engine(a: &[u8; 32], b: &[u8; 32], op: EngineOp) -> Engine25519 { - use utralib::generated::*; use crate::backend::serial::u32e::*; - crate::backend::serial::u32e::ensure_engine(); - let mut engine = utralib::CSR::new(unsafe{ENGINE_BASE.unwrap()}.as_mut_ptr() as *mut u32); - let mcode: &'static mut [u32] = unsafe{ - core::slice::from_raw_parts_mut(ENGINE_MEM.unwrap().as_mut_ptr() as *mut u32, 1024) - }; - let rf: [&'static mut [u32]; 3] = [ - unsafe{core::slice::from_raw_parts_mut( - (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 0 * 32) as *mut u32, 8)}, - unsafe{core::slice::from_raw_parts_mut( - (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 1 * 32) as *mut u32, 8)}, - unsafe{core::slice::from_raw_parts_mut( - (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 2 * 32) as *mut u32, 8)}, - ]; + match crate::backend::serial::u32e::ensure_engine() { + Ok(_) => { + let mut engine = + utralib::CSR::new(unsafe { ENGINE_BASE.unwrap() }.as_mut_ptr() as *mut u32); + let mcode: &'static mut [u32] = unsafe { + core::slice::from_raw_parts_mut(ENGINE_MEM.unwrap().as_mut_ptr() as *mut u32, 1024) + }; + let rf: [&'static mut [u32]; 3] = [ + unsafe { + core::slice::from_raw_parts_mut( + (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 0 * 32) as *mut u32, + 8, + ) + }, + unsafe { + core::slice::from_raw_parts_mut( + (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 1 * 32) as *mut u32, + 8, + ) + }, + unsafe { + core::slice::from_raw_parts_mut( + (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 2 * 32) as *mut u32, + 8, + ) + }, + ]; + loop { + let prog_len = match op { + EngineOp::Mul => { + let prog = assemble_engine25519!( + start: + mul %2, %0, %1 + fin + ); + for (&src, dest) in prog.iter().zip(mcode.iter_mut()) { + *dest = src; + } + engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32); + prog.len() + } + EngineOp::Add => { + let prog = assemble_engine25519!( + start: + add %2, %0, %1 + trd %30, %2 + sub %2, %2, %30 + fin + ); + for (&src, dest) in prog.iter().zip(mcode.iter_mut()) { + *dest = src; + } + engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32); + prog.len() + } + EngineOp::Sub => { + let prog = assemble_engine25519!( + start: + sub %1, #3, %1 + add %2, %0, %1 + trd %30, %2 + sub %2, %2, %30 + fin + ); + for (&src, dest) in prog.iter().zip(mcode.iter_mut()) { + *dest = src; + } + engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32); + prog.len() + } + }; + // copy a arg + for (src, dst) in a.chunks_exact(4).zip(rf[0].iter_mut()) { + let bytes: [u8; 4] = [src[0], src[1], src[2], src[3]]; + unsafe { + (dst as *mut u32).write_volatile(u32::from_le_bytes(bytes)); + } + /* this is a bad idea: src[0..4].try_into().unwrap() + because "unwrap()" adds in a whole bunch of string formatting stuff, adds +16k or so to the binary size + */ + } - match op { - EngineOp::Mul => { - let prog = assemble_engine25519!( - start: - mul %2, %0, %1 - fin - ); - for (&src, dest) in prog.iter().zip(mcode.iter_mut()) { - *dest = src; + // copy b arg + for (src, dst) in b.chunks_exact(4).zip(rf[1].iter_mut()) { + let bytes: [u8; 4] = [src[0], src[1], src[2], src[3]]; + unsafe { + (dst as *mut u32).write_volatile(u32::from_le_bytes(bytes)); + } + } + + engine.wfo(utra::engine::CONTROL_GO, 1); + while engine.rf(utra::engine::STATUS_RUNNING) != 0 {} + if !was_engine_error(prog_len) { + break; + } } - engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32); - }, - EngineOp::Add => { - let prog = assemble_engine25519!( - start: - add %2, %0, %1 - trd %30, %2 - sub %2, %2, %30 - fin - ); - for (&src, dest) in prog.iter().zip(mcode.iter_mut()) { - *dest = src; + + // return result, always in reg 2 + let mut result: [u8; 32] = [0; 32]; + for (&src, dst) in rf[2].iter().zip(result.chunks_exact_mut(4)) { + for (&sb, db) in src.to_le_bytes().iter().zip(dst.iter_mut()) { + *db = sb; + } } - engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32); - }, - EngineOp::Sub => { - let prog = assemble_engine25519!( - start: - sub %1, #3, %1 - add %2, %0, %1 - trd %30, %2 - sub %2, %2, %30 - fin - ); - for (&src, dest) in prog.iter().zip(mcode.iter_mut()) { - *dest = src; - } - engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32); - }, - } - // copy a arg - for (src, dst) in a.chunks_exact(4).zip(rf[0].iter_mut()) { - let bytes: [u8; 4] = [src[0], src[1], src[2], src[3]]; - unsafe{ (dst as *mut u32).write_volatile(u32::from_le_bytes(bytes));} - /* this is a bad idea: src[0..4].try_into().unwrap() - because "unwrap()" adds in a whole bunch of string formatting stuff, adds +16k or so to the binary size - */ - } - // copy b arg - for (src, dst) in b.chunks_exact(4).zip(rf[1].iter_mut()) { - let bytes: [u8; 4] = [src[0], src[1], src[2], src[3]]; - unsafe{ (dst as *mut u32).write_volatile(u32::from_le_bytes(bytes));} - } + #[cfg(feature="auto-release")] + free_engine(); - engine.wfo(utra::engine::CONTROL_GO, 1); - while engine.rf(utra::engine::STATUS_RUNNING) != 0 {} - - // return result, always in reg 2 - let mut result: [u8; 32] = [0; 32]; - for (&src, dst) in rf[2].iter().zip(result.chunks_exact_mut(4)) { - for (&sb, db) in src.to_le_bytes().iter().zip(dst.iter_mut()) { - *db = sb; + Engine25519 { 0: result } + } + _ => { + // fallback to fiat crypto field arithmetic... + #[cfg(feature="warn-fallback")] + log::warn!("Hardware acceleration unavailable, falling back to software"); + let fiat_a = bytes_to_fiat(a); + let fiat_b = bytes_to_fiat(b); + match op { + EngineOp::Mul => { + let mut self_loose = fiat_25519_loose_field_element([0; 10]); + fiat_25519_relax(&mut self_loose, &fiat_a.0); + let mut rhs_loose = fiat_25519_loose_field_element([0; 10]); + fiat_25519_relax(&mut rhs_loose, &fiat_b.0); + let mut output = FieldElement2625::ZERO; + fiat_25519_carry_mul(&mut output.0, &self_loose, &rhs_loose); + Engine25519 { + 0: fiat_to_bytes(&output.0), + } + } + EngineOp::Add => { + let mut result_loose = fiat_25519_loose_field_element([0; 10]); + fiat_25519_add(&mut result_loose, &fiat_a.0, &fiat_b.0); + let mut output = FieldElement2625::ZERO; + fiat_25519_carry(&mut output.0, &result_loose); + Engine25519 { + 0: fiat_to_bytes(&output.0), + } + } + EngineOp::Sub => { + let mut result_loose = fiat_25519_loose_field_element([0; 10]); + fiat_25519_sub(&mut result_loose, &fiat_a.0, &fiat_b.0); + let mut output = FieldElement2625::ZERO; + fiat_25519_carry(&mut output.0, &result_loose); + Engine25519 { + 0: fiat_to_bytes(&output.0), + } + } + } } - } - - Engine25519 { - 0: result } } @@ -210,11 +314,7 @@ impl<'a> Neg for &'a Engine25519 { } impl ConditionallySelectable for Engine25519 { - fn conditional_select( - a: &Engine25519, - b: &Engine25519, - choice: Choice, - ) -> Engine25519 { + fn conditional_select(a: &Engine25519, b: &Engine25519, choice: Choice) -> Engine25519 { Engine25519([ u8::conditional_select(&a.0[0], &b.0[0], choice), u8::conditional_select(&a.0[1], &b.0[1], choice), @@ -330,22 +430,23 @@ impl Engine25519 { } /// Construct zero. - pub const ZERO: Engine25519 = Engine25519([ 0 ; 32 ]); + pub const ZERO: Engine25519 = Engine25519([0; 32]); /// Construct one. - pub const ONE: Engine25519 = Engine25519([ 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - ]); + pub const ONE: Engine25519 = Engine25519([ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, + ]); /// Construct -1. - pub const MINUS_ONE: Engine25519 = - Engine25519([236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127]); + pub const MINUS_ONE: Engine25519 = Engine25519([ + 236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127, + ]); /// Given `k > 0`, return `self^(2^k)`. pub fn pow2k(&self, k: u32) -> Engine25519 { - debug_assert!( k > 0 ); + debug_assert!(k > 0); let mut z = self.square(); for _ in 1..k { z = z.square(); @@ -364,12 +465,11 @@ impl Engine25519 { /// encoding of every field element should decode, re-encode to /// the canonical encoding, and check that the input was /// canonical. - pub fn from_bytes(data: &[u8; 32]) -> Engine25519 { //FeFromBytes + pub fn from_bytes(data: &[u8; 32]) -> Engine25519 { + //FeFromBytes let mut mask_data = data.clone(); mask_data[31] &= 0x7F; // mask off the high bit per comment above - Engine25519 { - 0: mask_data, - } + Engine25519 { 0: mask_data } } /// Serialize this `FieldElement51` to a 32-byte array. The diff --git a/curve25519-dalek/src/backend/serial/u32e/mod.rs b/curve25519-dalek/src/backend/serial/u32e/mod.rs index 61c6b4b..8e87b57 100644 --- a/curve25519-dalek/src/backend/serial/u32e/mod.rs +++ b/curve25519-dalek/src/backend/serial/u32e/mod.rs @@ -34,11 +34,12 @@ pub(crate) const RF_U8_BASE: usize = 0x1_0000; #[allow(dead_code)] pub(crate) const RF_U32_BASE: usize = 0x1_0000 / 4; +/// It is safe to call this multiple times. pub fn free_engine() { log::debug!("free engine"); if let Some(base) = unsafe { ENGINE_BASE.take() } { let mut engine = utralib::CSR::new(base.as_mut_ptr() as *mut u32); - engine.rmwf(utra::engine::POWER_ON, 1); + engine.rmwf(utra::engine::POWER_ON, 0); xous::unmap_memory(base).unwrap(); } if let Some(mem) = unsafe { ENGINE_MEM.take() } { @@ -46,15 +47,34 @@ pub fn free_engine() { } } -pub fn ensure_engine() { +/// Only safe to call this after ensure_engine() has been called. +pub fn was_engine_error(job_len: usize) -> bool { + let mut engine = utralib::CSR::new(unsafe { ENGINE_BASE.unwrap() }.as_mut_ptr() as *mut u32); + + let reason = engine.r(utra::engine::EV_PENDING); + if reason & engine.ms(utra::engine::EV_PENDING_ILLEGAL_OPCODE, 1) != 0 { + panic!("Illegal opcode encountered in engine25519"); + } + // if the job length isn't what we had set it to, conclude that the + // microcode engine went through a suspend/resume cycle + if engine.rf(utra::engine::MPLEN_MPLEN) != job_len as u32 { + log::warn!("Suspend during engine25519 hw acceleration"); + return true; + } + + engine.wo(utra::engine::EV_PENDING, reason); + false +} + +/// It is safe to call this multiple times. +pub fn ensure_engine() -> Result<(), xous::Error> { if unsafe { ENGINE_BASE.is_none() } { let base = xous::syscall::map_memory( xous::MemoryAddress::new(utra::engine::HW_ENGINE_BASE), None, 4096, xous::MemoryFlags::R | xous::MemoryFlags::W, - ) - .expect("couldn't map engine CSR range"); + )?; log::debug!("claiming engine csr {:x?}", base.as_ptr()); unsafe { ENGINE_BASE = Some(base); @@ -66,13 +86,14 @@ pub fn ensure_engine() { None, HW_ENGINE_MEM_LEN, xous::MemoryFlags::R | xous::MemoryFlags::W, - ) - .expect("couldn't map engine memory window range"); + )?; log::debug!("claiming engine mem {:x?}", mem.as_ptr()); unsafe { ENGINE_MEM = Some(mem) }; } let mut engine = utralib::CSR::new(unsafe { ENGINE_BASE.unwrap() }.as_mut_ptr() as *mut u32); engine.rmwf(utra::engine::POWER_ON, 1); + engine.wo(utra::engine::EV_PENDING, 0xFFFF_FFFF); // clear all pending bits + Ok(()) } /// Safety: must be called after ensure_engine() diff --git a/curve25519-dalek/src/field.rs b/curve25519-dalek/src/field.rs index 44faa8d..80f51ea 100644 --- a/curve25519-dalek/src/field.rs +++ b/curve25519-dalek/src/field.rs @@ -43,7 +43,7 @@ cfg_if! { /// /// The `FieldElement` type is an alias for one of the platform-specific /// implementations. - pub type FieldElement = backend::serial::u32e::field::Engine25519; + pub type FieldElement = Engine25519; } else if #[cfg(curve25519_dalek_backend = "fiat")] { /// A `FieldElement` represents an element of the field diff --git a/curve25519-dalek/src/montgomery.rs b/curve25519-dalek/src/montgomery.rs index c78bea3..da618fa 100644 --- a/curve25519-dalek/src/montgomery.rs +++ b/curve25519-dalek/src/montgomery.rs @@ -54,10 +54,7 @@ use core::{ ops::{Mul, MulAssign}, }; -#[cfg(not(curve25519_dalek_backend = "u32e_backend"))] use crate::constants::{APLUS2_OVER_FOUR, MONTGOMERY_A, MONTGOMERY_A_NEG}; -#[cfg(curve25519_dalek_backend = "u32e_backend")] -use crate::constants::{MONTGOMERY_A, MONTGOMERY_A_NEG}; // eliminate constants absorbed into the microcode engine use crate::edwards::{CompressedEdwardsY, EdwardsPoint}; use crate::field::FieldElement; @@ -465,15 +462,33 @@ impl ProjectivePoint { ); use crate::backend::serial::u32e::*; - ensure_engine(); - // safety: these were called after ensure_engine() - let mut ucode_hw = unsafe { get_ucode() }; - let rf_hw = unsafe { get_rf() }; + match ensure_engine() { + Ok(_) => { + // safety: these were called after ensure_engine() + let mut ucode_hw = unsafe { get_ucode() }; + let rf_hw = unsafe { get_rf() }; - copy_to_rf(self.U.as_bytes(), 29, rf_hw, 0); - copy_to_rf(self.W.as_bytes(), 30, rf_hw, 0); + let mut r; + loop { + copy_to_rf(self.U.as_bytes(), 29, rf_hw, 0); + copy_to_rf(self.W.as_bytes(), 30, rf_hw, 0); - MontgomeryPoint(run_job(&mut ucode_hw, &rf_hw, &mcode, 0)) + r = MontgomeryPoint(run_job(&mut ucode_hw, &rf_hw, &mcode, 0)); + if !was_engine_error(mcode.len()) { + break; + } + } + #[cfg(feature="auto-release")] + free_engine(); + r + } + _ => { + #[cfg(feature="warn-fallback")] + log::warn!("Hardware acceleration unavailable, falling back to software"); + let u = &self.U * &self.W.invert(); + MontgomeryPoint(u.as_bytes()) + } + } } } @@ -622,29 +637,76 @@ pub(crate) fn differential_add_and_double( fin // finish execution ); use crate::backend::serial::u32e::*; - ensure_engine(); - // safety: these were called after ensure_engine() - let mut ucode_hw = unsafe { get_ucode() }; - let rf_hw = unsafe { get_rf() }; + match ensure_engine() { + Ok(_) => { + // safety: these were called after ensure_engine() + let mut ucode_hw = unsafe { get_ucode() }; + let rf_hw = unsafe { get_rf() }; - // P.U in %20 - // P.W in %21 - // Q.U in %22 - // Q.W in %23 - // affine_PmQ in %24 - copy_to_rf(P.U.as_bytes(), 20, rf_hw, 0); - copy_to_rf(P.W.as_bytes(), 21, rf_hw, 0); - copy_to_rf(Q.U.as_bytes(), 22, rf_hw, 0); - copy_to_rf(Q.W.as_bytes(), 23, rf_hw, 0); - copy_to_rf(affine_PmQ.as_bytes(), 24, rf_hw, 0); + loop { + // P.U in %20 + // P.W in %21 + // Q.U in %22 + // Q.W in %23 + // affine_PmQ in %24 + copy_to_rf(P.U.as_bytes(), 20, rf_hw, 0); + copy_to_rf(P.W.as_bytes(), 21, rf_hw, 0); + copy_to_rf(Q.U.as_bytes(), 22, rf_hw, 0); + copy_to_rf(Q.W.as_bytes(), 23, rf_hw, 0); + copy_to_rf(affine_PmQ.as_bytes(), 24, rf_hw, 0); - // start the run - run_job(&mut ucode_hw, &rf_hw, &mcode, 0); + // start the run + run_job(&mut ucode_hw, &rf_hw, &mcode, 0); + if !was_engine_error(mcode.len()) { + break; + } + } - P.U = FieldElement::from_bytes(©_from_rf(20, &rf_hw, 0)); - P.W = FieldElement::from_bytes(©_from_rf(21, &rf_hw, 0)); - Q.U = FieldElement::from_bytes(©_from_rf(22, &rf_hw, 0)); - Q.W = FieldElement::from_bytes(©_from_rf(23, &rf_hw, 0)); + P.U = FieldElement::from_bytes(©_from_rf(20, &rf_hw, 0)); + P.W = FieldElement::from_bytes(©_from_rf(21, &rf_hw, 0)); + Q.U = FieldElement::from_bytes(©_from_rf(22, &rf_hw, 0)); + Q.W = FieldElement::from_bytes(©_from_rf(23, &rf_hw, 0)); + #[cfg(feature="auto-release")] + free_engine(); + } + _ => { + #[cfg(feature="warn-fallback")] + log::warn!("Hardware acceleration unavailable, falling back to software"); + let t0 = &P.U + &P.W; + let t1 = &P.U - &P.W; + let t2 = &Q.U + &Q.W; + let t3 = &Q.U - &Q.W; + + let t4 = t0.square(); // (U_P + W_P)^2 = U_P^2 + 2 U_P W_P + W_P^2 + let t5 = t1.square(); // (U_P - W_P)^2 = U_P^2 - 2 U_P W_P + W_P^2 + + let t6 = &t4 - &t5; // 4 U_P W_P + + let t7 = &t0 * &t3; // (U_P + W_P) (U_Q - W_Q) = U_P U_Q + W_P U_Q - U_P W_Q - W_P W_Q + let t8 = &t1 * &t2; // (U_P - W_P) (U_Q + W_Q) = U_P U_Q - W_P U_Q + U_P W_Q - W_P W_Q + + let t9 = &t7 + &t8; // 2 (U_P U_Q - W_P W_Q) + let t10 = &t7 - &t8; // 2 (W_P U_Q - U_P W_Q) + + let t11 = t9.square(); // 4 (U_P U_Q - W_P W_Q)^2 + let t12 = t10.square(); // 4 (W_P U_Q - U_P W_Q)^2 + + let t13 = &APLUS2_OVER_FOUR * &t6; // (A + 2) U_P U_Q + + let t14 = &t4 * &t5; // ((U_P + W_P)(U_P - W_P))^2 = (U_P^2 - W_P^2)^2 + let t15 = &t13 + &t5; // (U_P - W_P)^2 + (A + 2) U_P W_P + + let t16 = &t6 * &t15; // 4 (U_P W_P) ((U_P - W_P)^2 + (A + 2) U_P W_P) + + let t17 = affine_PmQ * &t12; // U_D * 4 (W_P U_Q - U_P W_Q)^2 + let t18 = t11; // W_D * 4 (U_P U_Q - W_P W_Q)^2 + + P.U = t14; // U_{P'} = (U_P + W_P)^2 (U_P - W_P)^2 + P.W = t16; // W_{P'} = (4 U_P W_P) ((U_P - W_P)^2 + ((A + 2)/4) 4 U_P W_P) + Q.U = t18; // U_{Q'} = W_D * 4 (U_P U_Q - W_P W_Q)^2 + Q.W = t17; // W_{Q'} = U_D * 4 (W_P U_Q - U_P W_Q)^2 + } + } } define_mul_assign_variants!(LHS = MontgomeryPoint, RHS = Scalar); @@ -945,35 +1007,53 @@ impl Mul<&Scalar> for &MontgomeryPoint { ); let window = 0; - ensure_engine(); - // safety: these were called after ensure_engine() - let mut ucode_hw = unsafe { get_ucode() }; - let mut rf_hw = unsafe { get_rf() }; + match ensure_engine() { + Ok(_) => { + let mut r; + loop { + // safety: these were called after ensure_engine() + let mut ucode_hw = unsafe { get_ucode() }; + let mut rf_hw = unsafe { get_rf() }; - copy_to_rf(x0.U.as_bytes(), 25, &mut rf_hw, window); - copy_to_rf(x0.W.as_bytes(), 26, &mut rf_hw, window); - copy_to_rf(x1.U.as_bytes(), 27, &mut rf_hw, window); - copy_to_rf(x1.W.as_bytes(), 28, &mut rf_hw, window); - copy_to_rf(affine_u.as_bytes(), 24, &mut rf_hw, window); - copy_to_rf(scalar.bytes, 31, &mut rf_hw, window); - copy_to_rf( - [ - 254, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - ], - 19, - &mut rf_hw, - window, - ); // 254 as loop counter + copy_to_rf(x0.U.as_bytes(), 25, &mut rf_hw, window); + copy_to_rf(x0.W.as_bytes(), 26, &mut rf_hw, window); + copy_to_rf(x1.U.as_bytes(), 27, &mut rf_hw, window); + copy_to_rf(x1.W.as_bytes(), 28, &mut rf_hw, window); + copy_to_rf(affine_u.as_bytes(), 24, &mut rf_hw, window); + copy_to_rf(scalar.bytes, 31, &mut rf_hw, window); + copy_to_rf( + [ + 254, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + 19, + &mut rf_hw, + window, + ); // 254 as loop counter - MontgomeryPoint(run_job(&mut ucode_hw, &rf_hw, &mcode, window)) + r = MontgomeryPoint(run_job(&mut ucode_hw, &rf_hw, &mcode, window)); + if !was_engine_error(mcode.len()) { + break; + } + } + #[cfg(feature="auto-release")] + free_engine(); + r + } + _ => { + #[cfg(feature="warn-fallback")] + log::warn!("Hardware acceleration unavailable, falling back to software"); + // We multiply by the integer representation of the given Scalar. By scalar invariant #1, + // the MSB is 0, so we can skip it. + self.mul_bits_be(scalar.bits_le().rev().skip(1)) + } + } } /// Given `self` \\( = u\_0(P) \\), and a `Scalar` \\(n\\), return \\( u\_0(\[n\]P) \\) #[cfg(not(curve25519_dalek_backend = "u32e_backend"))] fn mul(self, scalar: &Scalar) -> MontgomeryPoint { - // TODO: consider feature "panic_on_sw_eval" #[cfg(all(not(test), curve25519_dalek_backend = "u32e_backend"))] // due to issue https://github.com/rust-lang/rust/issues/59168, you will have to manually comment this out when running a test on the full system and not just this crate. log::warn!("sw montgomery multiply being used - check for build config errors!"); // We multiply by the integer representation of the given Scalar. By scalar invariant #1, diff --git a/curve25519-dalek/src/scalar.rs b/curve25519-dalek/src/scalar.rs index def7cec..7a76eb4 100644 --- a/curve25519-dalek/src/scalar.rs +++ b/curve25519-dalek/src/scalar.rs @@ -844,7 +844,6 @@ impl Scalar { } /// Get the bits of the scalar, in little-endian order - #[cfg(not(curve25519_dalek_backend = "u32e_backend"))] pub(crate) fn bits_le(&self) -> impl DoubleEndedIterator + '_ { (0..256).map(|i| { // As i runs from 0..256, the bottom 3 bits index the bit, while the upper bits index