move the montgomery multiply code into the engine server

this saves a fairly substantial amount of time marshalling
re-used bits of microcode around on the benchmarks.
This commit is contained in:
bunnie 2021-06-28 18:41:34 +08:00
parent ee87fd35a8
commit e01676d563
7 changed files with 426 additions and 2 deletions

3
Cargo.lock generated
View file

@ -505,7 +505,7 @@ dependencies = [
[[package]]
name = "curve25519-dalek"
version = "3.1.0"
source = "git+https://github.com/betrusted-io/curve25519-dalek.git?branch=main#41cc7c4ab3ccc9e42f8894d96924585bafc23074"
source = "git+https://github.com/betrusted-io/curve25519-dalek.git?branch=main#1a79b76e90d49366b47eb98398a6104678686c15"
dependencies = [
"byteorder",
"digest",
@ -577,6 +577,7 @@ dependencies = [
name = "engine-25519"
version = "0.1.0"
dependencies = [
"engine25519-as",
"llio",
"log",
"log-server",

View file

@ -19,6 +19,12 @@ num-derive = {version = "0.3", default-features = false}
num-traits = {version = "0.2", default-features = false}
rkyv = {version = "0.4.3", default-features = false, features = ["const_generics"]}
[dependencies.engine25519-as]
git="https://github.com/betrusted-io/engine25519-as.git"
rev="6681e73c1fdc4a460b5ef9f9c7c91aef546d00f3"
default-features = false
features = []
[target.'cfg(not(any(windows,unix)))'.dependencies]
utralib = { path = "../../utralib"}

View file

@ -43,11 +43,24 @@ pub struct Job {
pub window: Option<u8>,
}
#[derive(Debug, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Clone, Copy)]
pub struct MontgomeryJob {
pub x0_u: [u8; 32],
pub x0_w: [u8; 32],
pub x1_u: [u8; 32],
pub x1_w: [u8; 32],
pub affine_u: [u8; 32],
pub scalar: [u8; 32],
}
#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)]
pub(crate) enum Opcode {
/// Runs a job, if the server is not already occupied
RunJob,
/// MontgomeryJob
MontgomeryJob,
/// a function that can be polled to determine if the block has been currently acquired
IsFree,
@ -75,6 +88,7 @@ pub(crate) enum Return {
pub enum JobResult {
/// returns a copy of the entire register file as a result
Result([u32; RF_SIZE_IN_U32]),
SingleResult([u8; 32]),
Started,
EngineUnavailable,
NotAsyncObject, // attempt to run an async job on an object that was setup for sync jobs

View file

@ -67,6 +67,39 @@ impl Engine25519 {
}
}
pub fn montgomery_job(&mut self, job: MontgomeryJob) -> Result<[u8; 32], xous::Error> {
let mut buf = Buffer::into_buf(job).or(Err(xous::Error::OutOfMemory))?;
match buf.lend_mut(self.conn, Opcode::MontgomeryJob.to_u32().unwrap()) {
Ok(_) => (),
Err(e) => {
if e == xous::Error::ServerNotFound {
log::error!("Looks like another thread called disconnect() on us while we weren't looking: {:?}", e);
} else {
log::error!("couldn't lend buffer: {:?}", e);
}
return Err(e);
}
}
match buf.to_original().unwrap() {
JobResult::SingleResult(r) => {
Ok(r)
},
JobResult::EngineUnavailable => {
log::debug!("spawn job: engine unavailable");
Err(xous::Error::ServerQueueFull)
},
JobResult::IllegalOpcodeException => {
log::error!("spawn job: illegal opcode");
Err(xous::Error::InvalidString)
},
_ => {
log::error!("spawn job: other error");
Err(xous::Error::UnknownError)
}
}
}
/// this is a blocking version of spawn_async_job.
/// if the engine is free, it will block until a result is returned
/// if the engine is busy, it will return an EngineUnavailable result.

View file

@ -1,6 +1,8 @@
#![cfg_attr(target_os = "none", no_std)]
#![cfg_attr(target_os = "none", no_main)]
#![recursion_limit="512"]
mod api;
use api::*;
@ -9,6 +11,9 @@ use core::sync::atomic::{AtomicBool, Ordering};
use xous::msg_blocking_scalar_unpack;
use xous_ipc::Buffer;
#[macro_use]
extern crate engine25519_as;
static RUN_IN_PROGRESS: AtomicBool = AtomicBool::new(false);
static DISALLOW_SUSPEND: AtomicBool = AtomicBool::new(false);
static SUSPEND_IN_PROGRESS: AtomicBool = AtomicBool::new(false);
@ -19,10 +24,10 @@ mod implementation {
use crate::api::*;
use susres::{RegManager, RegOrField, SuspendResume};
use num_traits::*;
//use core::num::NonZeroUsize;
use core::sync::atomic::Ordering;
use crate::RUN_IN_PROGRESS;
use crate::DISALLOW_SUSPEND;
use core::convert::TryInto;
pub struct Engine25519Hw {
csr: utralib::CSR<u32>,
@ -38,6 +43,7 @@ mod implementation {
clean_resume: Option<bool>,
do_notify: bool,
illegal_opcode: bool,
montgomery_len: Option<usize>,
}
fn handle_engine_irq(_irq_no: usize, arg: *mut usize) {
let engine = unsafe { &mut *(arg as *mut Engine25519Hw) };
@ -116,6 +122,7 @@ mod implementation {
clean_resume: None,
do_notify: false,
illegal_opcode: false,
montgomery_len: None,
};
log::trace!("claiming interrupt");
@ -259,6 +266,7 @@ mod implementation {
}
}
pub fn run(&mut self, job: Job) {
self.montgomery_len = None;
log::trace!("entering run");
// block any suspends from happening while we set up the engine
DISALLOW_SUSPEND.store(true, Ordering::Relaxed);
@ -317,6 +325,322 @@ mod implementation {
// we are now in a stable config, suspends are allowed
DISALLOW_SUSPEND.store(false, Ordering::Relaxed);
}
fn copy_reg(&mut self, r: [u8; 32], ra: usize, window: usize) {
for (src, dst) in r.chunks_exact(4).zip(self.rf_hw[window * RF_SIZE_IN_U32 + ra * 8..window * RF_SIZE_IN_U32 + (ra+1) * 8].iter_mut()) {
unsafe{ (dst as *mut u32).write_volatile(u32::from_le_bytes(src[0..4].try_into().unwrap()));}
}
}
fn load_montgomery(&mut self, mpstart: u32) -> u32 {
let mcode = assemble_engine25519!(
start:
// P.U in %20
// P.W in %21
// Q.U in %22
// Q.W in %23
// affine_PmQ in %24
// %30 is the TRD scratch register and cswap dummy
// %29 is the subtraction temporary value register and k_t
// x0.U in %25
// x0.W in %26
// x1.U in %27
// x1.W in %28
// %19 is the loop counter, starts with 254 (if 0, loop runs exactly once)
// %31 is the scalar
// %18 is the swap variable
psa %18, #0
// for i in (0..255).rev()
mainloop:
// let choice: u8 = (bits[i + 1] ^ bits[i]) as u8;
// ProjectivePoint::conditional_swap(&mut x0, &mut x1, choice.into());
xbt %29, %31 // orignally[k_t = (k>>t) & 1] now[k_t = k[254]]
shl %31, %31 // k = k<<1
xor %18, %18, %29 // swap ^= k_t
// cswap x0.U (%25), x1.U (%27)
xor %30, %25, %27
msk %30, %18, %30
xor %25, %30, %25
xor %27, %30, %27
// cswap x0.W (%26), x1.W (%28)
xor %30, %26, %28
msk %30, %18, %30
xor %26, %30, %26
xor %28, %30, %28
psa %18, %29 // swap = k_t
// differential_add_and_double(&mut x0, &mut x1, &affine_u);
psa %20, %25
psa %21, %26
psa %22, %27
psa %23, %28
// affine_u is already in %24
// let t0 = &P.U + &P.W;
add %0, %20, %21
trd %30, %0
sub %0, %0, %30
// let t1 = &P.U - &P.W;
sub %21, #3, %21 // negate &P.W using #FIELDPRIME (#3)
add %1, %20, %21
trd %30, %1
sub %1, %1, %30
// let t2 = &Q.U + &Q.W;
add %2, %22, %23
trd %30, %2
sub %2, %2, %30
// let t3 = &Q.U - &Q.W;
sub %23, #3, %23
add %3, %22, %23
trd %30, %3
sub %3, %3, %30
// let t4 = t0.square(); // (U_P + W_P)^2 = U_P^2 + 2 U_P W_P + W_P^2
mul %4, %0, %0
// let t5 = t1.square(); // (U_P - W_P)^2 = U_P^2 - 2 U_P W_P + W_P^2
mul %5, %1, %1
// let t6 = &t4 - &t5; // 4 U_P W_P
sub %29, #3, %5
add %6, %4, %29
trd %30, %6
sub %6, %6, %30
// let t7 = &t0 * &t3; // (U_P + W_P) (U_Q - W_Q) = U_P U_Q + W_P U_Q - U_P W_Q - W_P W_Q
mul %7, %0, %3
// let t8 = &t1 * &t2; // (U_P - W_P) (U_Q + W_Q) = U_P U_Q - W_P U_Q + U_P W_Q - W_P W_Q
mul %8, %1, %2
// let t9 = &t7 + &t8; // 2 (U_P U_Q - W_P W_Q)
add %9, %7, %8
trd %30, %9
sub %9, %9, %30
// let t10 = &t7 - &t8; // 2 (W_P U_Q - U_P W_Q)
sub %29, #3, %8
add %10, %7, %29
trd %30, %10
sub %10, %10, %30
// let t11 = t9.square(); // 4 (U_P U_Q - W_P W_Q)^2
mul %11, %9, %9
// let t12 = t10.square(); // 4 (W_P U_Q - U_P W_Q)^2
mul %12, %10, %10
// let t13 = &APLUS2_OVER_FOUR * &t6; // (A + 2) U_P U_Q
mul %13, #4, %6 // #4 is A+2/4
// let t14 = &t4 * &t5; // ((U_P + W_P)(U_P - W_P))^2 = (U_P^2 - W_P^2)^2
mul %14, %4, %5
// let t15 = &t13 + &t5; // (U_P - W_P)^2 + (A + 2) U_P W_P
add %15, %13, %5
trd %30, %15
sub %15, %15, %30
// let t16 = &t6 * &t15; // 4 (U_P W_P) ((U_P - W_P)^2 + (A + 2) U_P W_P)
mul %16, %6, %15
// let t17 = affine_PmQ * &t12; // U_D * 4 (W_P U_Q - U_P W_Q)^2
mul %17, %24, %12 // affine_PmQ loaded into %24
///// these can be eliminated down the road, but included for 1:1 algorithm correspodence to reference in early testing
// P.U = t14; // U_{P'} = (U_P + W_P)^2 (U_P - W_P)^2
psa %20, %14
// P.W = t16; // W_{P'} = (4 U_P W_P) ((U_P - W_P)^2 + ((A + 2)/4) 4 U_P W_P)
psa %21, %16
// let t18 = t11; // W_D * 4 (U_P U_Q - W_P W_Q)^2
// Q.U = t18; // U_{Q'} = W_D * 4 (U_P U_Q - W_P W_Q)^2
psa %22, %11 // collapsed two to save a register
// Q.W = t17; // W_{Q'} = U_D * 4 (W_P U_Q - U_P W_Q)^2
psa %23, %17
///// 'return' arguments for next iteration, can be optimized out later
psa %25, %20
psa %26, %21
psa %27, %22
psa %28, %23
brz end, %19 // if loop counter is 0, quit
sub %19, %19, #1 // subtract one from the loop counter and run again
brz mainloop, #0 // go back to the top
end:
// ProjectivePoint::conditional_swap(&mut x0, &mut x1, Choice::from(bits[0] as u8));
// cswap x0.U (%25), x1.U (%27)
xor %30, %25, %27
msk %30, %18, %30
xor %25, %30, %25
xor %27, %30, %27
// cswap x0.W (%26), x1.W (%28)
xor %30, %26, %28
msk %30, %18, %30
xor %26, %30, %26
xor %28, %30, %28
// AFFINE SPLICE -- pass arguments to the affine block
psa %29, %25
psa %30, %26
// W.invert() in %21
// U in %29
// W in %30
// result in %31
// loop counter in %28
// from FieldElement.invert()
// let (t19, t3) = self.pow22501(); // t19: 249..0 ; t3: 3,1,0
// let t0 = self.square(); // 1 e_0 = 2^1
mul %0, %30, %30 // self is W, e.g. %30
// let t1 = t0.square().square(); // 3 e_1 = 2^3
mul %1, %0, %0
mul %1, %1, %1
// let t2 = self * &t1; // 3,0 e_2 = 2^3 + 2^0
mul %2, %30, %1
// let t3 = &t0 * &t2; // 3,1,0
mul %3, %0, %2
// let t4 = t3.square(); // 4,2,1
mul %4, %3, %3
// let t5 = &t2 * &t4; // 4,3,2,1,0
mul %5, %2, %4
// let t6 = t5.pow2k(5); // 9,8,7,6,5
psa %28, #5 // coincidentally, constant #5 is the number 5
mul %6, %5, %5
pow2k_5:
sub %28, %28, #1 // %28 = %28 - 1
brz pow2k_5_exit, %28
mul %6, %6, %6
brz pow2k_5, #0
pow2k_5_exit:
// let t7 = &t6 * &t5; // 9,8,7,6,5,4,3,2,1,0
mul %7, %6, %5
// let t8 = t7.pow2k(10); // 19..10
psa %28, #6 // constant #6 is the number 10
mul %8, %7, %7
pow2k_10:
sub %28, %28, #1
brz pow2k_10_exit, %28
mul %8, %8, %8
brz pow2k_10, #0
pow2k_10_exit:
// let t9 = &t8 * &t7; // 19..0
mul %9, %8, %7
// let t10 = t9.pow2k(20); // 39..20
psa %28, #7 // constant #7 is the number 20
mul %10, %9, %9
pow2k_20:
sub %28, %28, #1
brz pow2k_20_exit, %28
mul %10, %10, %10
brz pow2k_20, #0
pow2k_20_exit:
// let t11 = &t10 * &t9; // 39..0
mul %11, %10, %9
// let t12 = t11.pow2k(10); // 49..10
psa %28, #6 // constant #6 is the number 10
mul %12, %11, %11
pow2k_10b:
sub %28, %28, #1
brz pow2k_10b_exit, %28
mul %12, %12, %12
brz pow2k_10b, #0
pow2k_10b_exit:
// let t13 = &t12 * &t7; // 49..0
mul %13, %12, %7
// let t14 = t13.pow2k(50); // 99..50
psa %28, #8 // constant #8 is the number 50
mul %14, %13, %13
pow2k_50a:
sub %28, %28, #1
brz pow2k_50a_exit, %28
mul %14, %14, %14
brz pow2k_50a, #0
pow2k_50a_exit:
// let t15 = &t14 * &t13; // 99..0
mul %15, %14, %13
// let t16 = t15.pow2k(100); // 199..100
psa %28, #9 // constant #9 is the number 100
mul %16, %15, %15
pow2k_100:
sub %28, %28, #1
brz pow2k_100_exit, %28
mul %16, %16, %16
brz pow2k_100, #0
pow2k_100_exit:
// let t17 = &t16 * &t15; // 199..0
mul %17, %16, %15
// let t18 = t17.pow2k(50); // 249..50
psa %28, #8 // constant #8 is the number 50
mul %18, %17, %17
pow2k_50b:
sub %28, %28, #1
brz pow2k_50b_exit, %28
mul %18, %18, %18
brz pow2k_50b, #0
pow2k_50b_exit:
// let t19 = &t18 * &t13; // 249..0
mul %19, %18, %13
//(t19, t3) // just a return value, values are already there, do nothing
//let t20 = t19.pow2k(5); // 254..5
psa %28, #5
mul %20, %19, %19
pow2k_5_last:
sub %28, %28, #1
brz pow2k_5_last_exit, %28
mul %20, %20, %20
brz pow2k_5_last, #0
pow2k_5_last_exit:
//let t21 = &t20 * &t3; // 254..5,3,1,0
mul %21, %20, %3
// u = &self.U * &self.W.invert()
mul %31, %29, %21
fin // finish execution
);
for (&src, dst) in mcode.iter().zip(self.ucode_hw[mpstart as usize..].iter_mut()) {
unsafe { (dst as *mut u32).write_volatile(src as u32) };
}
mcode.len() as u32
}
pub fn montgomery(&mut self, job: MontgomeryJob) {
log::trace!("entering run");
// block any suspends from happening while we set up the engine
DISALLOW_SUSPEND.store(true, Ordering::Relaxed);
let window: usize = 0;
self.copy_reg(job.x0_u, 25, window);
self.copy_reg(job.x0_w, 26, window);
self.copy_reg(job.x1_u, 27, window);
self.copy_reg(job.x1_w, 28, window);
self.copy_reg(job.affine_u, 24, window);
self.copy_reg(job.scalar, 31, window);
self.copy_reg([
254, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
], 19, window); // 254 as loop counter
let mpstart = 0;
// this optimization shaves off about 0.13ms per iteration
if self.montgomery_len.is_none() {
self.montgomery_len = Some(self.load_montgomery(mpstart) as usize);
}
self.csr.wfo(utra::engine::WINDOW_WINDOW, window as u32); // this value should now be validated because an invalid window would cause a panic on slice copy
self.csr.wfo(utra::engine::MPSTART_MPSTART, mpstart);
self.csr.wfo(utra::engine::MPLEN_MPLEN, self.montgomery_len.unwrap() as u32);
log::trace!("sanity check uc{:08x}, rf{:08x}", self.ucode_hw[0], self.rf_hw[0]);
// sync calls poll a state variable, and thus no message is sent
self.do_notify = false;
// setup the sync polling variable
RUN_IN_PROGRESS.store(true, Ordering::Relaxed);
// this will start the run. interrupts should *already* be enabled for the completion notification...
self.csr.wfo(utra::engine::CONTROL_GO, 1);
// we are now in a stable config, suspends are allowed
DISALLOW_SUSPEND.store(false, Ordering::Relaxed);
}
pub fn get_result(&mut self) -> JobResult {
if let Some(clean_resume) = self.clean_resume {
@ -344,6 +668,26 @@ mod implementation {
JobResult::Result(ret_rf)
}
pub fn get_single_result(&mut self, r: usize) -> JobResult {
if let Some(clean_resume) = self.clean_resume {
if !clean_resume {
return JobResult::SuspendError;
}
}
if self.illegal_opcode {
return JobResult::IllegalOpcodeException;
}
let mut ret_r: [u8; 32] = [0; 32];
let window = self.csr.rf(utra::engine::WINDOW_WINDOW) as usize;
for (&src, dst) in self.rf_hw[window * RF_SIZE_IN_U32 + r * 8..window * RF_SIZE_IN_U32 + (r+1) * 8].iter().zip(ret_r.chunks_exact_mut(4)) {
for (&src_byte, dst_byte) in src.to_le_bytes().iter().zip(dst.iter_mut()) {
*dst_byte = src_byte;
}
}
JobResult::SingleResult(ret_r)
}
}
}
@ -448,10 +792,33 @@ fn xmain() -> ! {
let mut client_cid: Option<xous::CID> = None;
let mut job_count = 0;
let mut mont_count = 0;
loop {
let mut msg = xous::receive_message(engine25519_sid).unwrap();
log::trace!("Message: {:?}", msg);
match FromPrimitive::from_usize(msg.body.id()) {
Some(Opcode::MontgomeryJob) => {
if mont_count % 100 == 0 {
log::info!("montgomery job {}", mont_count); // leave this here for now so we can confirm that HW acceleration is being selected when we think it is!
}
mont_count += 1;
// don't start a new job if a suspend is in progress
while SUSPEND_IN_PROGRESS.load(Ordering::Relaxed) {
log::trace!("waiting for suspend to finish");
xous::yield_slice();
}
engine25519.power_on(true);
let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
let montgomery_job = buffer.to_original::<MontgomeryJob, _>().unwrap();
engine25519.montgomery(montgomery_job);
while RUN_IN_PROGRESS.load(Ordering::Relaxed) {
// block until the job is done
xous::yield_slice();
}
let result = engine25519.get_single_result(31); // return the result
engine25519.power_on(false);
buffer.replace(result).unwrap();
}
Some(Opcode::RunJob) => {
if job_count % 100 == 0 {
log::info!("engine job {}", job_count); // leave this here for now so we can confirm that HW acceleration is being selected when we think it is!

View file

@ -41,6 +41,7 @@ rkyv = {version = "0.4.3", default-features = false, features = ["const_generics
rand_core = "0.5"
hex = {version = "0.4.3", default-features = false, features = []}
[dependencies.curve25519-dalek]
# don't forget the Cargo.toml patch.crates.io file to patch over third-party dependencies as well!
git="https://github.com/betrusted-io/curve25519-dalek.git"
branch="main"
#path = "../../../curve25519-dalek"

View file

@ -157,6 +157,8 @@ pub fn benchmark_thread(sid0: usize, sid1: usize, sid2: usize, sid3: usize) {
33ms/2xop (200 iters - hw) - affine done in software
190ms/2xop (200 iters - sw)
26ms/2xop (200 iters -hw) -- affine done in hardware
13.54ms/2xop (200 iters -hw) -- microcode pre-loaded, minimal registers transferred using MontgomeryJob call
*/
Some(BenchOp::StartDh) => {
let mut passes = 0;