From e01676d563d2cbee5210765e5e215c201d525f7e Mon Sep 17 00:00:00 2001 From: bunnie Date: Mon, 28 Jun 2021 18:41:34 +0800 Subject: [PATCH] move the montgomery multiply code into the engine server this saves a fairly substantial amount of time marshalling re-used bits of microcode around on the benchmarks. --- Cargo.lock | 3 +- services/engine-25519/Cargo.toml | 6 + services/engine-25519/src/api.rs | 14 + services/engine-25519/src/lib.rs | 33 +++ services/engine-25519/src/main.rs | 369 +++++++++++++++++++++++++- services/shellchat/Cargo.toml | 1 + services/shellchat/src/cmds/engine.rs | 2 + 7 files changed, 426 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 630618a46..8af2b666e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,7 +505,7 @@ dependencies = [ [[package]] name = "curve25519-dalek" version = "3.1.0" -source = "git+https://github.com/betrusted-io/curve25519-dalek.git?branch=main#41cc7c4ab3ccc9e42f8894d96924585bafc23074" +source = "git+https://github.com/betrusted-io/curve25519-dalek.git?branch=main#1a79b76e90d49366b47eb98398a6104678686c15" dependencies = [ "byteorder", "digest", @@ -577,6 +577,7 @@ dependencies = [ name = "engine-25519" version = "0.1.0" dependencies = [ + "engine25519-as", "llio", "log", "log-server", diff --git a/services/engine-25519/Cargo.toml b/services/engine-25519/Cargo.toml index 119b4655a..33e9ea5b5 100644 --- a/services/engine-25519/Cargo.toml +++ b/services/engine-25519/Cargo.toml @@ -19,6 +19,12 @@ num-derive = {version = "0.3", default-features = false} num-traits = {version = "0.2", default-features = false} rkyv = {version = "0.4.3", default-features = false, features = ["const_generics"]} +[dependencies.engine25519-as] +git="https://github.com/betrusted-io/engine25519-as.git" +rev="6681e73c1fdc4a460b5ef9f9c7c91aef546d00f3" +default-features = false +features = [] + [target.'cfg(not(any(windows,unix)))'.dependencies] utralib = { path = "../../utralib"} diff --git a/services/engine-25519/src/api.rs b/services/engine-25519/src/api.rs index f6bd22e4b..1ca97eb72 100644 --- a/services/engine-25519/src/api.rs +++ b/services/engine-25519/src/api.rs @@ -43,11 +43,24 @@ pub struct Job { pub window: Option, } +#[derive(Debug, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Clone, Copy)] +pub struct MontgomeryJob { + pub x0_u: [u8; 32], + pub x0_w: [u8; 32], + pub x1_u: [u8; 32], + pub x1_w: [u8; 32], + pub affine_u: [u8; 32], + pub scalar: [u8; 32], +} + #[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)] pub(crate) enum Opcode { /// Runs a job, if the server is not already occupied RunJob, + /// MontgomeryJob + MontgomeryJob, + /// a function that can be polled to determine if the block has been currently acquired IsFree, @@ -75,6 +88,7 @@ pub(crate) enum Return { pub enum JobResult { /// returns a copy of the entire register file as a result Result([u32; RF_SIZE_IN_U32]), + SingleResult([u8; 32]), Started, EngineUnavailable, NotAsyncObject, // attempt to run an async job on an object that was setup for sync jobs diff --git a/services/engine-25519/src/lib.rs b/services/engine-25519/src/lib.rs index 169121435..046f57ed8 100644 --- a/services/engine-25519/src/lib.rs +++ b/services/engine-25519/src/lib.rs @@ -67,6 +67,39 @@ impl Engine25519 { } } + pub fn montgomery_job(&mut self, job: MontgomeryJob) -> Result<[u8; 32], xous::Error> { + let mut buf = Buffer::into_buf(job).or(Err(xous::Error::OutOfMemory))?; + match buf.lend_mut(self.conn, Opcode::MontgomeryJob.to_u32().unwrap()) { + Ok(_) => (), + Err(e) => { + if e == xous::Error::ServerNotFound { + log::error!("Looks like another thread called disconnect() on us while we weren't looking: {:?}", e); + } else { + log::error!("couldn't lend buffer: {:?}", e); + } + return Err(e); + } + } + + match buf.to_original().unwrap() { + JobResult::SingleResult(r) => { + Ok(r) + }, + JobResult::EngineUnavailable => { + log::debug!("spawn job: engine unavailable"); + Err(xous::Error::ServerQueueFull) + }, + JobResult::IllegalOpcodeException => { + log::error!("spawn job: illegal opcode"); + Err(xous::Error::InvalidString) + }, + _ => { + log::error!("spawn job: other error"); + Err(xous::Error::UnknownError) + } + } + } + /// this is a blocking version of spawn_async_job. /// if the engine is free, it will block until a result is returned /// if the engine is busy, it will return an EngineUnavailable result. diff --git a/services/engine-25519/src/main.rs b/services/engine-25519/src/main.rs index 4adae829b..9b14082c1 100644 --- a/services/engine-25519/src/main.rs +++ b/services/engine-25519/src/main.rs @@ -1,6 +1,8 @@ #![cfg_attr(target_os = "none", no_std)] #![cfg_attr(target_os = "none", no_main)] +#![recursion_limit="512"] + mod api; use api::*; @@ -9,6 +11,9 @@ use core::sync::atomic::{AtomicBool, Ordering}; use xous::msg_blocking_scalar_unpack; use xous_ipc::Buffer; +#[macro_use] +extern crate engine25519_as; + static RUN_IN_PROGRESS: AtomicBool = AtomicBool::new(false); static DISALLOW_SUSPEND: AtomicBool = AtomicBool::new(false); static SUSPEND_IN_PROGRESS: AtomicBool = AtomicBool::new(false); @@ -19,10 +24,10 @@ mod implementation { use crate::api::*; use susres::{RegManager, RegOrField, SuspendResume}; use num_traits::*; - //use core::num::NonZeroUsize; use core::sync::atomic::Ordering; use crate::RUN_IN_PROGRESS; use crate::DISALLOW_SUSPEND; + use core::convert::TryInto; pub struct Engine25519Hw { csr: utralib::CSR, @@ -38,6 +43,7 @@ mod implementation { clean_resume: Option, do_notify: bool, illegal_opcode: bool, + montgomery_len: Option, } fn handle_engine_irq(_irq_no: usize, arg: *mut usize) { let engine = unsafe { &mut *(arg as *mut Engine25519Hw) }; @@ -116,6 +122,7 @@ mod implementation { clean_resume: None, do_notify: false, illegal_opcode: false, + montgomery_len: None, }; log::trace!("claiming interrupt"); @@ -259,6 +266,7 @@ mod implementation { } } pub fn run(&mut self, job: Job) { + self.montgomery_len = None; log::trace!("entering run"); // block any suspends from happening while we set up the engine DISALLOW_SUSPEND.store(true, Ordering::Relaxed); @@ -317,6 +325,322 @@ mod implementation { // we are now in a stable config, suspends are allowed DISALLOW_SUSPEND.store(false, Ordering::Relaxed); } + fn copy_reg(&mut self, r: [u8; 32], ra: usize, window: usize) { + for (src, dst) in r.chunks_exact(4).zip(self.rf_hw[window * RF_SIZE_IN_U32 + ra * 8..window * RF_SIZE_IN_U32 + (ra+1) * 8].iter_mut()) { + unsafe{ (dst as *mut u32).write_volatile(u32::from_le_bytes(src[0..4].try_into().unwrap()));} + } + } + fn load_montgomery(&mut self, mpstart: u32) -> u32 { + let mcode = assemble_engine25519!( + start: + // P.U in %20 + // P.W in %21 + // Q.U in %22 + // Q.W in %23 + // affine_PmQ in %24 + // %30 is the TRD scratch register and cswap dummy + // %29 is the subtraction temporary value register and k_t + // x0.U in %25 + // x0.W in %26 + // x1.U in %27 + // x1.W in %28 + // %19 is the loop counter, starts with 254 (if 0, loop runs exactly once) + // %31 is the scalar + // %18 is the swap variable + psa %18, #0 + + // for i in (0..255).rev() + mainloop: + // let choice: u8 = (bits[i + 1] ^ bits[i]) as u8; + // ProjectivePoint::conditional_swap(&mut x0, &mut x1, choice.into()); + xbt %29, %31 // orignally[k_t = (k>>t) & 1] now[k_t = k[254]] + shl %31, %31 // k = k<<1 + xor %18, %18, %29 // swap ^= k_t + + // cswap x0.U (%25), x1.U (%27) + xor %30, %25, %27 + msk %30, %18, %30 + xor %25, %30, %25 + xor %27, %30, %27 + // cswap x0.W (%26), x1.W (%28) + xor %30, %26, %28 + msk %30, %18, %30 + xor %26, %30, %26 + xor %28, %30, %28 + + psa %18, %29 // swap = k_t + + // differential_add_and_double(&mut x0, &mut x1, &affine_u); + psa %20, %25 + psa %21, %26 + psa %22, %27 + psa %23, %28 + // affine_u is already in %24 + + // let t0 = &P.U + &P.W; + add %0, %20, %21 + trd %30, %0 + sub %0, %0, %30 + // let t1 = &P.U - &P.W; + sub %21, #3, %21 // negate &P.W using #FIELDPRIME (#3) + add %1, %20, %21 + trd %30, %1 + sub %1, %1, %30 + // let t2 = &Q.U + &Q.W; + add %2, %22, %23 + trd %30, %2 + sub %2, %2, %30 + // let t3 = &Q.U - &Q.W; + sub %23, #3, %23 + add %3, %22, %23 + trd %30, %3 + sub %3, %3, %30 + // let t4 = t0.square(); // (U_P + W_P)^2 = U_P^2 + 2 U_P W_P + W_P^2 + mul %4, %0, %0 + // let t5 = t1.square(); // (U_P - W_P)^2 = U_P^2 - 2 U_P W_P + W_P^2 + mul %5, %1, %1 + // let t6 = &t4 - &t5; // 4 U_P W_P + sub %29, #3, %5 + add %6, %4, %29 + trd %30, %6 + sub %6, %6, %30 + // let t7 = &t0 * &t3; // (U_P + W_P) (U_Q - W_Q) = U_P U_Q + W_P U_Q - U_P W_Q - W_P W_Q + mul %7, %0, %3 + // let t8 = &t1 * &t2; // (U_P - W_P) (U_Q + W_Q) = U_P U_Q - W_P U_Q + U_P W_Q - W_P W_Q + mul %8, %1, %2 + // let t9 = &t7 + &t8; // 2 (U_P U_Q - W_P W_Q) + add %9, %7, %8 + trd %30, %9 + sub %9, %9, %30 + // let t10 = &t7 - &t8; // 2 (W_P U_Q - U_P W_Q) + sub %29, #3, %8 + add %10, %7, %29 + trd %30, %10 + sub %10, %10, %30 + // let t11 = t9.square(); // 4 (U_P U_Q - W_P W_Q)^2 + mul %11, %9, %9 + // let t12 = t10.square(); // 4 (W_P U_Q - U_P W_Q)^2 + mul %12, %10, %10 + // let t13 = &APLUS2_OVER_FOUR * &t6; // (A + 2) U_P U_Q + mul %13, #4, %6 // #4 is A+2/4 + // let t14 = &t4 * &t5; // ((U_P + W_P)(U_P - W_P))^2 = (U_P^2 - W_P^2)^2 + mul %14, %4, %5 + // let t15 = &t13 + &t5; // (U_P - W_P)^2 + (A + 2) U_P W_P + add %15, %13, %5 + trd %30, %15 + sub %15, %15, %30 + // let t16 = &t6 * &t15; // 4 (U_P W_P) ((U_P - W_P)^2 + (A + 2) U_P W_P) + mul %16, %6, %15 + // let t17 = affine_PmQ * &t12; // U_D * 4 (W_P U_Q - U_P W_Q)^2 + mul %17, %24, %12 // affine_PmQ loaded into %24 + + ///// these can be eliminated down the road, but included for 1:1 algorithm correspodence to reference in early testing + // P.U = t14; // U_{P'} = (U_P + W_P)^2 (U_P - W_P)^2 + psa %20, %14 + // P.W = t16; // W_{P'} = (4 U_P W_P) ((U_P - W_P)^2 + ((A + 2)/4) 4 U_P W_P) + psa %21, %16 + // let t18 = t11; // W_D * 4 (U_P U_Q - W_P W_Q)^2 + // Q.U = t18; // U_{Q'} = W_D * 4 (U_P U_Q - W_P W_Q)^2 + psa %22, %11 // collapsed two to save a register + // Q.W = t17; // W_{Q'} = U_D * 4 (W_P U_Q - U_P W_Q)^2 + psa %23, %17 + + ///// 'return' arguments for next iteration, can be optimized out later + psa %25, %20 + psa %26, %21 + psa %27, %22 + psa %28, %23 + + brz end, %19 // if loop counter is 0, quit + sub %19, %19, #1 // subtract one from the loop counter and run again + brz mainloop, #0 // go back to the top + end: + // ProjectivePoint::conditional_swap(&mut x0, &mut x1, Choice::from(bits[0] as u8)); + // cswap x0.U (%25), x1.U (%27) + xor %30, %25, %27 + msk %30, %18, %30 + xor %25, %30, %25 + xor %27, %30, %27 + // cswap x0.W (%26), x1.W (%28) + xor %30, %26, %28 + msk %30, %18, %30 + xor %26, %30, %26 + xor %28, %30, %28 + + // AFFINE SPLICE -- pass arguments to the affine block + psa %29, %25 + psa %30, %26 + // W.invert() in %21 + // U in %29 + // W in %30 + // result in %31 + // loop counter in %28 + + // from FieldElement.invert() + // let (t19, t3) = self.pow22501(); // t19: 249..0 ; t3: 3,1,0 + // let t0 = self.square(); // 1 e_0 = 2^1 + mul %0, %30, %30 // self is W, e.g. %30 + // let t1 = t0.square().square(); // 3 e_1 = 2^3 + mul %1, %0, %0 + mul %1, %1, %1 + // let t2 = self * &t1; // 3,0 e_2 = 2^3 + 2^0 + mul %2, %30, %1 + // let t3 = &t0 * &t2; // 3,1,0 + mul %3, %0, %2 + // let t4 = t3.square(); // 4,2,1 + mul %4, %3, %3 + // let t5 = &t2 * &t4; // 4,3,2,1,0 + mul %5, %2, %4 + + // let t6 = t5.pow2k(5); // 9,8,7,6,5 + psa %28, #5 // coincidentally, constant #5 is the number 5 + mul %6, %5, %5 + pow2k_5: + sub %28, %28, #1 // %28 = %28 - 1 + brz pow2k_5_exit, %28 + mul %6, %6, %6 + brz pow2k_5, #0 + pow2k_5_exit: + // let t7 = &t6 * &t5; // 9,8,7,6,5,4,3,2,1,0 + mul %7, %6, %5 + + // let t8 = t7.pow2k(10); // 19..10 + psa %28, #6 // constant #6 is the number 10 + mul %8, %7, %7 + pow2k_10: + sub %28, %28, #1 + brz pow2k_10_exit, %28 + mul %8, %8, %8 + brz pow2k_10, #0 + pow2k_10_exit: + // let t9 = &t8 * &t7; // 19..0 + mul %9, %8, %7 + + // let t10 = t9.pow2k(20); // 39..20 + psa %28, #7 // constant #7 is the number 20 + mul %10, %9, %9 + pow2k_20: + sub %28, %28, #1 + brz pow2k_20_exit, %28 + mul %10, %10, %10 + brz pow2k_20, #0 + pow2k_20_exit: + // let t11 = &t10 * &t9; // 39..0 + mul %11, %10, %9 + + // let t12 = t11.pow2k(10); // 49..10 + psa %28, #6 // constant #6 is the number 10 + mul %12, %11, %11 + pow2k_10b: + sub %28, %28, #1 + brz pow2k_10b_exit, %28 + mul %12, %12, %12 + brz pow2k_10b, #0 + pow2k_10b_exit: + // let t13 = &t12 * &t7; // 49..0 + mul %13, %12, %7 + + // let t14 = t13.pow2k(50); // 99..50 + psa %28, #8 // constant #8 is the number 50 + mul %14, %13, %13 + pow2k_50a: + sub %28, %28, #1 + brz pow2k_50a_exit, %28 + mul %14, %14, %14 + brz pow2k_50a, #0 + pow2k_50a_exit: + // let t15 = &t14 * &t13; // 99..0 + mul %15, %14, %13 + + // let t16 = t15.pow2k(100); // 199..100 + psa %28, #9 // constant #9 is the number 100 + mul %16, %15, %15 + pow2k_100: + sub %28, %28, #1 + brz pow2k_100_exit, %28 + mul %16, %16, %16 + brz pow2k_100, #0 + pow2k_100_exit: + // let t17 = &t16 * &t15; // 199..0 + mul %17, %16, %15 + + // let t18 = t17.pow2k(50); // 249..50 + psa %28, #8 // constant #8 is the number 50 + mul %18, %17, %17 + pow2k_50b: + sub %28, %28, #1 + brz pow2k_50b_exit, %28 + mul %18, %18, %18 + brz pow2k_50b, #0 + pow2k_50b_exit: + // let t19 = &t18 * &t13; // 249..0 + mul %19, %18, %13 + //(t19, t3) // just a return value, values are already there, do nothing + + //let t20 = t19.pow2k(5); // 254..5 + psa %28, #5 + mul %20, %19, %19 + pow2k_5_last: + sub %28, %28, #1 + brz pow2k_5_last_exit, %28 + mul %20, %20, %20 + brz pow2k_5_last, #0 + pow2k_5_last_exit: + + //let t21 = &t20 * &t3; // 254..5,3,1,0 + mul %21, %20, %3 + + // u = &self.U * &self.W.invert() + mul %31, %29, %21 + fin // finish execution + ); + for (&src, dst) in mcode.iter().zip(self.ucode_hw[mpstart as usize..].iter_mut()) { + unsafe { (dst as *mut u32).write_volatile(src as u32) }; + } + mcode.len() as u32 + } + pub fn montgomery(&mut self, job: MontgomeryJob) { + log::trace!("entering run"); + // block any suspends from happening while we set up the engine + DISALLOW_SUSPEND.store(true, Ordering::Relaxed); + + let window: usize = 0; + + self.copy_reg(job.x0_u, 25, window); + self.copy_reg(job.x0_w, 26, window); + self.copy_reg(job.x1_u, 27, window); + self.copy_reg(job.x1_w, 28, window); + self.copy_reg(job.affine_u, 24, window); + self.copy_reg(job.scalar, 31, window); + self.copy_reg([ + 254, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ], 19, window); // 254 as loop counter + + let mpstart = 0; + // this optimization shaves off about 0.13ms per iteration + if self.montgomery_len.is_none() { + self.montgomery_len = Some(self.load_montgomery(mpstart) as usize); + } + self.csr.wfo(utra::engine::WINDOW_WINDOW, window as u32); // this value should now be validated because an invalid window would cause a panic on slice copy + self.csr.wfo(utra::engine::MPSTART_MPSTART, mpstart); + self.csr.wfo(utra::engine::MPLEN_MPLEN, self.montgomery_len.unwrap() as u32); + + log::trace!("sanity check uc{:08x}, rf{:08x}", self.ucode_hw[0], self.rf_hw[0]); + + // sync calls poll a state variable, and thus no message is sent + self.do_notify = false; + + // setup the sync polling variable + RUN_IN_PROGRESS.store(true, Ordering::Relaxed); + // this will start the run. interrupts should *already* be enabled for the completion notification... + self.csr.wfo(utra::engine::CONTROL_GO, 1); + + // we are now in a stable config, suspends are allowed + DISALLOW_SUSPEND.store(false, Ordering::Relaxed); + } pub fn get_result(&mut self) -> JobResult { if let Some(clean_resume) = self.clean_resume { @@ -344,6 +668,26 @@ mod implementation { JobResult::Result(ret_rf) } + pub fn get_single_result(&mut self, r: usize) -> JobResult { + if let Some(clean_resume) = self.clean_resume { + if !clean_resume { + return JobResult::SuspendError; + } + } + if self.illegal_opcode { + return JobResult::IllegalOpcodeException; + } + + let mut ret_r: [u8; 32] = [0; 32]; + let window = self.csr.rf(utra::engine::WINDOW_WINDOW) as usize; + for (&src, dst) in self.rf_hw[window * RF_SIZE_IN_U32 + r * 8..window * RF_SIZE_IN_U32 + (r+1) * 8].iter().zip(ret_r.chunks_exact_mut(4)) { + for (&src_byte, dst_byte) in src.to_le_bytes().iter().zip(dst.iter_mut()) { + *dst_byte = src_byte; + } + } + + JobResult::SingleResult(ret_r) + } } } @@ -448,10 +792,33 @@ fn xmain() -> ! { let mut client_cid: Option = None; let mut job_count = 0; + let mut mont_count = 0; loop { let mut msg = xous::receive_message(engine25519_sid).unwrap(); log::trace!("Message: {:?}", msg); match FromPrimitive::from_usize(msg.body.id()) { + Some(Opcode::MontgomeryJob) => { + if mont_count % 100 == 0 { + log::info!("montgomery job {}", mont_count); // leave this here for now so we can confirm that HW acceleration is being selected when we think it is! + } + mont_count += 1; + // don't start a new job if a suspend is in progress + while SUSPEND_IN_PROGRESS.load(Ordering::Relaxed) { + log::trace!("waiting for suspend to finish"); + xous::yield_slice(); + } + engine25519.power_on(true); + let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) }; + let montgomery_job = buffer.to_original::().unwrap(); + engine25519.montgomery(montgomery_job); + while RUN_IN_PROGRESS.load(Ordering::Relaxed) { + // block until the job is done + xous::yield_slice(); + } + let result = engine25519.get_single_result(31); // return the result + engine25519.power_on(false); + buffer.replace(result).unwrap(); + } Some(Opcode::RunJob) => { if job_count % 100 == 0 { log::info!("engine job {}", job_count); // leave this here for now so we can confirm that HW acceleration is being selected when we think it is! diff --git a/services/shellchat/Cargo.toml b/services/shellchat/Cargo.toml index 7b442c5e0..1ba297851 100644 --- a/services/shellchat/Cargo.toml +++ b/services/shellchat/Cargo.toml @@ -41,6 +41,7 @@ rkyv = {version = "0.4.3", default-features = false, features = ["const_generics rand_core = "0.5" hex = {version = "0.4.3", default-features = false, features = []} [dependencies.curve25519-dalek] +# don't forget the Cargo.toml patch.crates.io file to patch over third-party dependencies as well! git="https://github.com/betrusted-io/curve25519-dalek.git" branch="main" #path = "../../../curve25519-dalek" diff --git a/services/shellchat/src/cmds/engine.rs b/services/shellchat/src/cmds/engine.rs index ccf21e2e6..bae8449c5 100644 --- a/services/shellchat/src/cmds/engine.rs +++ b/services/shellchat/src/cmds/engine.rs @@ -157,6 +157,8 @@ pub fn benchmark_thread(sid0: usize, sid1: usize, sid2: usize, sid3: usize) { 33ms/2xop (200 iters - hw) - affine done in software 190ms/2xop (200 iters - sw) 26ms/2xop (200 iters -hw) -- affine done in hardware + + 13.54ms/2xop (200 iters -hw) -- microcode pre-loaded, minimal registers transferred using MontgomeryJob call */ Some(BenchOp::StartDh) => { let mut passes = 0;