diff --git a/Cargo.lock b/Cargo.lock index fa52ee70e..5df89ca82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1018,6 +1018,7 @@ dependencies = [ name = "cram-mbox1" version = "0.1.0" dependencies = [ + "aes 0.8.3", "log", "num-derive 0.4.2", "num-traits", diff --git a/kernel/src/arch/riscv/irq.rs b/kernel/src/arch/riscv/irq.rs index a3800a747..b7b2cbe4d 100644 --- a/kernel/src/arch/riscv/irq.rs +++ b/kernel/src/arch/riscv/irq.rs @@ -50,22 +50,35 @@ fn sip_read() -> usize { existing } +// using verilator-only as a proxy for the cramium-soc config; +// when the flag is off, assume precursor config +#[cfg(all(feature = "vexii-test", feature = "verilator-only"))] +use crate::platform::cramium::{ + LEGACY_INT_VMEM, + legacy_int::{SUPER_MASK, SUPER_PENDING}, +}; +#[cfg(all(feature = "vexii-test", not(feature = "verilator-only")))] +use crate::platform::precursor::{ + LEGACY_INT_VMEM, + legacy_int::{SUPER_MASK, SUPER_PENDING}, +}; + #[cfg(feature = "vexii-test")] fn sim_read() -> usize { - let legacy_int = utralib::CSR::new(crate::platform::precursor::LEGACY_INT_VMEM as *mut u32); - legacy_int.r(crate::platform::precursor::legacy_int::SUPER_MASK) as usize + let legacy_int = utralib::CSR::new(LEGACY_INT_VMEM as *mut u32); + legacy_int.r(SUPER_MASK) as usize } #[cfg(feature = "vexii-test")] fn sim_write(new: usize) { - let mut legacy_int = utralib::CSR::new(crate::platform::precursor::LEGACY_INT_VMEM as *mut u32); - legacy_int.wo(crate::platform::precursor::legacy_int::SUPER_MASK, new as u32); + let mut legacy_int = utralib::CSR::new(LEGACY_INT_VMEM as *mut u32); + legacy_int.wo(SUPER_MASK, new as u32); } #[cfg(feature = "vexii-test")] fn sip_read() -> usize { - let legacy_int = utralib::CSR::new(crate::platform::precursor::LEGACY_INT_VMEM as *mut u32); - legacy_int.r(crate::platform::precursor::legacy_int::SUPER_PENDING) as usize + let legacy_int = utralib::CSR::new(LEGACY_INT_VMEM as *mut u32); + legacy_int.r(SUPER_PENDING) as usize } /// Disable external interrupts diff --git a/kernel/src/arch/riscv/mem.rs b/kernel/src/arch/riscv/mem.rs index 9aba47244..1cff33f0c 100644 --- a/kernel/src/arch/riscv/mem.rs +++ b/kernel/src/arch/riscv/mem.rs @@ -258,7 +258,7 @@ impl MemoryMapping { /// As such, this will only have an observable effect once code returns /// to userspace. pub fn activate(self) -> Result<(), xous_kernel::Error> { - unsafe { flush_mmu() }; + // unsafe { flush_mmu() }; // redundant - adds 9% time to context switch benchmark when present! satp::write(self.satp); unsafe { flush_mmu() }; Ok(()) diff --git a/kernel/src/platform/cramium/mod.rs b/kernel/src/platform/cramium/mod.rs index f38f9e7e1..c2bc5f633 100644 --- a/kernel/src/platform/cramium/mod.rs +++ b/kernel/src/platform/cramium/mod.rs @@ -12,8 +12,56 @@ pub use rand_fake as rand; #[cfg(any(feature = "debug-print", feature = "print-panics"))] pub mod uart; +#[cfg(feature = "vexii-test")] +use crate::{MemoryFlags, MemoryType, PID, mem::MemoryManager}; +#[cfg(feature = "vexii-test")] +pub const LEGACY_INT_VMEM: usize = 0xffcf_1000; // see https://github.com/betrusted-io/xous-core/blob/master/docs/memory.md + +#[allow(dead_code)] +#[cfg(feature = "vexii-test")] +pub mod legacy_int { + // this enum is vestigal, and currently not used by anything + #[derive(Debug, Copy, Clone)] + pub enum LegacyIntOffset { + MachMask = 0, + MachPending = 1, + SuperMask = 2, + SuperPending = 3, + } + pub const LEGACY_INT_NUMREGS: usize = 4; + + pub const MACH_MASK: utralib::Register = utralib::Register::new(0, 0xffffffff); + pub const MACH_MASK_MACH_MASK: utralib::Field = utralib::Field::new(32, 0, MACH_MASK); + + pub const MACH_PENDING: utralib::Register = utralib::Register::new(1, 0xffffffff); + pub const MACH_PENDING_MACH_PENDING: utralib::Field = utralib::Field::new(32, 0, MACH_PENDING); + + pub const SUPER_MASK: utralib::Register = utralib::Register::new(2, 0xffffffff); + pub const SUPER_MASK_SUPER_MASK: utralib::Field = utralib::Field::new(32, 0, SUPER_MASK); + + pub const SUPER_PENDING: utralib::Register = utralib::Register::new(3, 0xffffffff); + pub const SUPER_PENDING_SUPER_PENDING: utralib::Field = utralib::Field::new(32, 0, SUPER_PENDING); + + pub const HW_LEGACY_INT_BASE: usize = 0xe001d000; +} + /// Precursor specific initialization. pub fn init() { + #[cfg(feature = "vexii-test")] + // Map the interrupt manager shim + MemoryManager::with_mut(|memory_manager| { + memory_manager + .map_range( + legacy_int::HW_LEGACY_INT_BASE as *mut u8, + (LEGACY_INT_VMEM & !4095) as *mut u8, + 4096, + PID::new(1).unwrap(), + MemoryFlags::R | MemoryFlags::W, + MemoryType::Default, + ) + .expect("unable to map legacy interrupt shim for vexii CPU") + }); + self::rand::init(); #[cfg(any(feature = "debug-print", feature = "print-panics"))] self::uart::init(); diff --git a/kernel/src/services.rs b/kernel/src/services.rs index 27d6f5456..8165d99f0 100644 --- a/kernel/src/services.rs +++ b/kernel/src/services.rs @@ -66,6 +66,20 @@ pub struct SystemServices { pub servers: [Option; MAX_SERVER_COUNT], } +pub enum PostActivateOp<'a> { + None, + SetThreadResult { + result: xous_kernel::Result, + }, + RememberServerMessage { + sidx: usize, + current_pid: PID, + current_thread: TID, + message: &'a Message, + client_address: Option, + }, +} + #[derive(Copy, Clone, PartialEq)] pub enum ProcessState { /// This is an unallocated, free process @@ -1042,13 +1056,21 @@ impl SystemServices { /// Resume the given process, picking up exactly where it left off. If the /// process is in the Setup state, set it up and then resume. + /// + /// `lazy_arg` is an argument that should be placed in the activated target process, + /// after the activation is complete. The purpose of this is to reduce the incidence + /// of the idiom where we swap into the a target process' memory state just to emplace + /// a small argument or piece of data, only to swap back to then return to it on activation. + /// This optimization improves message latency by around 15% in the fast path. pub fn activate_process_thread( &mut self, previous_tid: TID, new_pid: PID, mut new_tid: TID, can_resume: bool, + lazy_arg: PostActivateOp, ) -> Result { + let mut sender_idx: Option = None; let previous_pid = self.current_pid(); #[cfg(feature = "debug-print")] @@ -1060,59 +1082,63 @@ impl SystemServices { // Save state if the PID has changed. This will activate the new memory // space. - let new = self.get_process_mut(new_pid)?; if new_pid != previous_pid { - klog!("New process original state: {:?}", new.state); + { + let new = self.get_process_mut(new_pid)?; + klog!("New process original state: {:?}", new.state); - // Ensure the new process can be run. - match new.state { - ProcessState::Free => { - klog!("PID {} was free", new_pid); - return Err(xous_kernel::Error::ProcessNotFound); - } - ProcessState::Setup(_) | ProcessState::Allocated => new_tid = INITIAL_TID, - ProcessState::Exception(_) => { - new_tid = crate::arch::process::EXCEPTION_TID; - // new.current_thread = new_tid; - } - ProcessState::Ready(x) => { - // If no new context is specified, take the previous - // context. If that is not runnable, do a round-robin - // search for the next available context. - assert!(x != 0, "process was {:?} but had no runnable threads", new.state); - if new_tid == 0 { - new_tid = Self::find_next_thread(x, new.current_thread); - } - if x & (1 << new_tid) == 0 { - println!( - "process state is {:?}, but new thread {} is not runnable", - new.state, new_tid - ); + // Ensure the new process can be run. + match new.state { + ProcessState::Free => { + klog!("PID {} was free", new_pid); + return Err(xous_kernel::Error::ProcessNotFound); + } + ProcessState::Setup(_) | ProcessState::Allocated => new_tid = INITIAL_TID, + ProcessState::Exception(_) => { + new_tid = crate::arch::process::EXCEPTION_TID; + // new.current_thread = new_tid; + } + ProcessState::Ready(x) => { + // If no new context is specified, take the previous + // context. If that is not runnable, do a round-robin + // search for the next available context. + assert!(x != 0, "process was {:?} but had no runnable threads", new.state); + if new_tid == 0 { + new_tid = Self::find_next_thread(x, new.current_thread); + } + if x & (1 << new_tid) == 0 { + println!( + "process state is {:?}, but new thread {} is not runnable", + new.state, new_tid + ); + return Err(xous_kernel::Error::ProcessNotFound); + } + new.current_thread = new_tid as _; + } + ProcessState::Running(_) => { + panic!("process was running even though the pid was different") + } + #[cfg(feature = "gdb-stub")] + ProcessState::Debug(_) | ProcessState::DebugIrq(_) => { + return Err(xous_kernel::Error::ProcessNotFound); + } + ProcessState::Sleeping | ProcessState::BlockedException(_) => { + // println!("PID {} was sleeping or being debugged", new_pid); return Err(xous_kernel::Error::ProcessNotFound); } - new.current_thread = new_tid as _; - } - ProcessState::Running(_) => { - panic!("process was running even though the pid was different") - } - #[cfg(feature = "gdb-stub")] - ProcessState::Debug(_) | ProcessState::DebugIrq(_) => { - return Err(xous_kernel::Error::ProcessNotFound); - } - ProcessState::Sleeping | ProcessState::BlockedException(_) => { - // println!("PID {} was sleeping or being debugged", new_pid); - return Err(xous_kernel::Error::ProcessNotFound); } } // Perform the actual switch to the new memory space. From this // point onward, we will need to activate the previous memory space // if we encounter an error. + let new = self.get_process(new_pid)?; new.mapping.activate()?; // Set up the new process, if necessary. Remove the new thread from // the list of ready threads. // let old_state = new.state; + let new = self.get_process_mut(new_pid)?; new.state = match new.state { ProcessState::Setup(thread_init) => { // klog!("Setting up new process..."); @@ -1141,6 +1167,27 @@ impl SystemServices { }; // log_process_update(file!(), line!(), new, old_state); new.activate()?; + match lazy_arg { + PostActivateOp::RememberServerMessage { + sidx, + current_pid, + current_thread, + message, + client_address, + } => { + let si = { + let server = + self.server_from_sidx_mut(sidx).expect("couldn't re-discover server index"); + server.queue_response(current_pid, current_thread, message, client_address)? + }; + sender_idx = Some(si); + } + PostActivateOp::SetThreadResult { result } => { + let mut arch_process = ArchProcess::current(); + arch_process.set_thread_result(new_tid, result); + } + PostActivateOp::None => (), + } // Mark the previous process as ready to run, since we just switched // away @@ -1196,6 +1243,26 @@ impl SystemServices { // can_resume // ); } else { + match lazy_arg { + PostActivateOp::RememberServerMessage { + sidx, + current_pid, + current_thread, + message, + client_address, + } => { + let si = { + let server = + self.server_from_sidx_mut(sidx).expect("couldn't re-discover server index"); + server.queue_response(current_pid, current_thread, message, client_address)? + }; + sender_idx = Some(si); + } + PostActivateOp::SetThreadResult { result } => { + ArchProcess::current().set_thread_result(new_tid, result); + } + PostActivateOp::None => (), + } let new = self.get_process_mut(new_pid)?; // If we wanted to switch to a "new" thread, and it's the same @@ -1253,7 +1320,9 @@ impl SystemServices { self.get_process_mut(new_pid)?.state ); - Ok(new_tid) + // return argument based upon the lazy_arg type + // we get "lucky" in that sender_idx *and* new_tid are both `usize` types underneath + Ok(sender_idx.unwrap_or(new_tid)) } /// Move memory from one process to another. @@ -1725,7 +1794,7 @@ impl SystemServices { if arch_process.thread_exists(join_tid) { // The target thread exists -- put this thread to sleep let ppid = self.get_process(pid).unwrap().ppid; - self.activate_process_thread(tid, ppid, 0, false) + self.activate_process_thread(tid, ppid, 0, false, PostActivateOp::None) .map(|_| Ok(xous_kernel::Result::ResumeProcess)) .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)) } else { diff --git a/kernel/src/swap.rs b/kernel/src/swap.rs index 3239af1e8..0544d3c09 100644 --- a/kernel/src/swap.rs +++ b/kernel/src/swap.rs @@ -431,7 +431,7 @@ impl Swap { true, ) .ok(); - unsafe { crate::arch::mem::flush_mmu() }; + // unsafe { crate::arch::mem::flush_mmu() }; // redundant? } else { already_mapped = true; } diff --git a/kernel/src/syscall.rs b/kernel/src/syscall.rs index 0e1097cc8..d46097cea 100644 --- a/kernel/src/syscall.rs +++ b/kernel/src/syscall.rs @@ -11,7 +11,7 @@ use crate::arch::process::Process as ArchProcess; use crate::irq::{interrupt_claim, interrupt_free}; use crate::mem::MemoryManager; use crate::server::{SenderID, WaitingMessage}; -use crate::services::SystemServices; +use crate::services::{PostActivateOp, SystemServices}; #[cfg(feature = "swap")] use crate::swap::{Swap, SwapAbi}; @@ -70,7 +70,7 @@ fn do_yield(_pid: PID, tid: TID) -> SysCallResult { SystemServices::with_mut(|ss| { // TODO: Advance thread let result = ss - .activate_process_thread(tid, parent_pid, parent_ctx, true) + .activate_process_thread(tid, parent_pid, parent_ctx, true, PostActivateOp::None) .map(|_| Ok(xous_kernel::Result::ResumeProcess)) .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)); @@ -162,17 +162,7 @@ fn send_message(pid: PID, tid: TID, cid: CID, message: Message) -> SysCallResult // "there are threads available in PID {} to handle this message -- marking as Ready", // server_pid // ); - let sender_idx = if message.is_blocking() { - ss.remember_server_message(sidx, pid, tid, &message, client_address).map_err(|e| { - klog!("error remembering server message: {:?}", e); - ss.server_from_sidx_mut(sidx) - .expect("server couldn't be located") - .return_available_thread(server_tid); - e - })? - } else { - 0 - }; + let sender_idx = 0; // lazy-evaluate to a real number for blocking messages; eagerly set to 0 for non-blocking let sender = SenderID::new(sidx, sender_idx, Some(pid)); klog!("server connection data: sidx: {}, idx: {}, server pid: {}", sidx, sender_idx, server_pid); let envelope = MessageEnvelope { sender: sender.into(), body: message }; @@ -191,19 +181,35 @@ fn send_message(pid: PID, tid: TID, cid: CID, message: Message) -> SysCallResult // --- NOTE: Returning this value // return if blocking && cfg!(baremetal) { if !runnable { + // re-bind the envelope because the sender_idx binding is lazily evaluated + let message = envelope.take_message(); + let sender_idx = ss + .remember_server_message(sidx, pid, tid, &message, client_address) + .map_err(|e| { + klog!("error remembering server message: {:?}", e); + ss.server_from_sidx_mut(sidx) + .expect("server couldn't be located") + .return_available_thread(server_tid); + e + })?; + let sender = SenderID::new(sidx, sender_idx, Some(pid)); + let envelope = MessageEnvelope { sender: sender.into(), body: message }; + // If it's not runnable (e.g. it's being debugged), switch to the parent. let (ppid, ptid) = unsafe { (&mut *(&raw mut SWITCHTO_CALLER)).take().unwrap() }; klog!( "Activating Server parent process (server is blocked) and switching away from Client" ); - ss.set_thread_result( - server_pid, - server_tid, - xous_kernel::Result::MessageEnvelope(envelope), - ) - .expect("couldn't set result for server thread"); let result = ss - .activate_process_thread(tid, ppid, ptid, false) + .activate_process_thread( + tid, + ppid, + ptid, + false, + PostActivateOp::SetThreadResult { + result: xous_kernel::Result::MessageEnvelope(envelope), + }, + ) .map(|_| Ok(xous_kernel::Result::ResumeProcess)) .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)); @@ -219,11 +225,42 @@ fn send_message(pid: PID, tid: TID, cid: CID, message: Message) -> SysCallResult } else { // Switch to the server, since it's in a state to be run. klog!("Activating Server context and switching away from Client"); - ss.activate_process_thread(tid, server_pid, server_tid, false) - .map(|_| Ok(xous_kernel::Result::MessageEnvelope(envelope))) - .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)) + let message = envelope.take_message(); + match ss.activate_process_thread( + tid, + server_pid, + server_tid, + false, + PostActivateOp::RememberServerMessage { + sidx, + current_pid: pid, + current_thread: tid, + message: &message, + client_address, + }, + ) { + Ok(sender_idx) => { + let sender = SenderID::new(sidx, sender_idx, Some(pid)); + let envelope = MessageEnvelope { sender: sender.into(), body: message }; + Ok(xous_kernel::Result::MessageEnvelope(envelope)) + } + _ => Err(xous_kernel::Error::ProcessNotFound), + } } } else if blocking && !cfg!(baremetal) { + // re-bind the envelope: lazy eval has to happen now. + let message = envelope.take_message(); + let sender_idx = + ss.remember_server_message(sidx, pid, tid, &message, client_address).map_err(|e| { + klog!("error remembering server message: {:?}", e); + ss.server_from_sidx_mut(sidx) + .expect("server couldn't be located") + .return_available_thread(server_tid); + e + })?; + let sender = SenderID::new(sidx, sender_idx, Some(pid)); + let envelope = MessageEnvelope { sender: sender.into(), body: message }; + klog!("Blocking client, since it sent a blocking message"); ss.unschedule_thread(pid, tid)?; ss.switch_to_thread(server_pid, Some(server_tid))?; @@ -262,7 +299,7 @@ fn send_message(pid: PID, tid: TID, cid: CID, message: Message) -> SysCallResult let ppid = process.ppid; unsafe { SWITCHTO_CALLER = None }; let result = ss - .activate_process_thread(tid, ppid, 0, false) + .activate_process_thread(tid, ppid, 0, false, PostActivateOp::None) .map(|_| Ok(xous_kernel::Result::ResumeProcess)) .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)); @@ -590,13 +627,16 @@ fn reply_and_receive_next( } else { // For baremetal targets, switch away from this process. if cfg!(baremetal) { - // Set the thread result for the client - ss.set_thread_result(response.pid, response.tid, response.result)?; - - // Activate the client thread and switch to it - ss.activate_process_thread(server_tid, response.pid, response.tid, false) - .map(|_| Ok(xous_kernel::Result::ResumeProcess)) - .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)) + // Set the thread result for the client and activate the client thread and switch to it + ss.activate_process_thread( + server_tid, + response.pid, + response.tid, + false, + PostActivateOp::SetThreadResult { result: response.result }, + ) + .map(|_| Ok(xous_kernel::Result::ResumeProcess)) + .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)) } // For hosted targets, simply return `BlockedProcess` indicating we'll make // a callback to their socket at a later time. @@ -650,7 +690,7 @@ fn receive_message(pid: PID, tid: TID, sid: SID, blocking: ExecutionType) -> Sys let ppid = ss.get_process(pid).expect("Can't get current process").ppid; // TODO: Advance thread let result = ss - .activate_process_thread(tid, ppid, 0, false) + .activate_process_thread(tid, ppid, 0, false, PostActivateOp::None) .map(|_| Ok(xous_kernel::Result::ResumeProcess)) .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)); ss.set_last_thread(PID::new(ORIGINAL_PID.load(Relaxed)).unwrap(), ORIGINAL_TID.load(Relaxed)) @@ -841,7 +881,7 @@ pub fn handle_inner(pid: PID, tid: TID, in_irq: bool, call: SysCall) -> SysCallR // "Activating process thread {} in pid {} coming from pid {} thread {}", // new_context, new_pid, pid, tid // ); - let new_tid = ss.activate_process_thread(tid, new_pid, new_tid, true)?; + let new_tid = ss.activate_process_thread(tid, new_pid, new_tid, true, PostActivateOp::None)?; ORIGINAL_PID.store(new_pid.get(), Relaxed); ORIGINAL_TID.store(new_tid, Relaxed); Ok(xous_kernel::Result::ResumeProcess) @@ -870,7 +910,7 @@ pub fn handle_inner(pid: PID, tid: TID, in_irq: bool, call: SysCall) -> SysCallR // TODO: Advance thread if cfg!(baremetal) { let result = ss - .activate_process_thread(tid, ppid, 0, false) + .activate_process_thread(tid, ppid, 0, false, PostActivateOp::None) .map(|_| Ok(xous_kernel::Result::ResumeProcess)) .unwrap_or(Err(xous_kernel::Error::ProcessNotFound)); ss.set_last_thread(PID::new(ORIGINAL_PID.load(Relaxed)).unwrap(), ORIGINAL_TID.load(Relaxed)) diff --git a/loader/Cargo.toml b/loader/Cargo.toml index ece31a285..ab23348d7 100644 --- a/loader/Cargo.toml +++ b/loader/Cargo.toml @@ -101,7 +101,7 @@ board-baosec = [ ] board-baosor = ["cramium-hal/loader-baosor"] board-dabao = ["cramium-hal/loader-dabao"] -cramium-fpga = ["utralib/cramium-fpga", "debug-print"] +cramium-fpga = ["utralib/cramium-fpga", "debug-print", "linked_list_allocator"] atsama5d27 = ["utralib/atsama5d27", "armv7", "dep:atsama5d27"] # precursor flags diff --git a/loader/build.rs b/loader/build.rs index 19d75180d..f41eab36d 100644 --- a/loader/build.rs +++ b/loader/build.rs @@ -13,7 +13,7 @@ fn main() { #[cfg(feature = "cramium-soc")] let p = PathBuf::from("src/platform/cramium/link-soc.x"); #[cfg(feature = "cramium-fpga")] - let p = PathBuf::from("src/platform/cramium/link-soc.x"); // note deviation -- this is because we are still simulating the FPGA model and offsets are not yet settled for that + let p = PathBuf::from("src/platform/cramium/link-fpga.x"); #[cfg(not(any(feature = "cramium-soc", feature = "cramium-fpga")))] let p = PathBuf::from("src/platform/precursor/link.x"); diff --git a/loader/src/main.rs b/loader/src/main.rs index 0a4b897d5..976041287 100644 --- a/loader/src/main.rs +++ b/loader/src/main.rs @@ -167,7 +167,7 @@ pub unsafe extern "C" fn rust_entry(signed_buffer: *const usize, signature: u32) loop {} }; - #[cfg(feature = "vexii-test")] + #[cfg(all(feature = "vexii-test", not(feature = "verilator-only")))] { // fake the prehash use ed25519_dalek_loader::Digest; diff --git a/loader/src/phase2.rs b/loader/src/phase2.rs index eb2e67607..16a6c1c2e 100644 --- a/loader/src/phase2.rs +++ b/loader/src/phase2.rs @@ -339,7 +339,7 @@ pub fn phase_2(cfg: &mut BootConfig, fs_prehash: &[u8; 64]) { 1, // Kernel is PID 1 ); } - #[cfg(feature = "debug-print")] + #[cfg(all(feature = "debug-print", not(feature = "verilator-only")))] // print the kernel's page table mappings as a sanity check on the loader debug::print_pagetable(cfg.processes[0].satp); diff --git a/loader/src/platform/cramium/link-fpga.x b/loader/src/platform/cramium/link-fpga.x index 8b0e921ca..12b341504 100644 --- a/loader/src/platform/cramium/link-fpga.x +++ b/loader/src/platform/cramium/link-fpga.x @@ -1,12 +1,11 @@ MEMORY { /* we are using an unsigned loader, so the offset is not 1k offset */ - FLASH : ORIGIN = 0x60500000, LENGTH = 32k - RAM : ORIGIN = 0x61000000, LENGTH = 16M + /* if the font location is adjusted, this needs to be updated in libs/blitstr2 codegen templates.go FONT_BASE value */ + FONTS : ORIGIN = 0x60040000, LENGTH = 128k + FLASH : ORIGIN = 0x60001000, LENGTH = 252k + RAM : ORIGIN = 0x61000000, LENGTH = 2M } -/* -Fonts go from 0x2053_0000 to 0x2097_0000 -*/ REGION_ALIAS("REGION_TEXT", FLASH); REGION_ALIAS("REGION_RODATA", FLASH); @@ -88,12 +87,12 @@ SECTIONS . = ALIGN(4); _edata = .; } > REGION_DATA AT > REGION_RODATA - - /* .fonts : ALIGN(65536) +/* + .fonts : ALIGN(65536) { KEEP(*(.fontdata)); - } > FONTS */ - + } > FONTS +*/ .bss (NOLOAD) : { _sbss = .; diff --git a/services/aes/src/lib.rs b/services/aes/src/lib.rs index 044d2293a..588eff84b 100644 --- a/services/aes/src/lib.rs +++ b/services/aes/src/lib.rs @@ -44,7 +44,8 @@ pub use vex::{Aes128, Aes256}; #[cfg(feature = "vexii-test")] mod zkn; #[cfg(feature = "vexii-test")] -pub use soft::Aes128Soft as Aes128; +// pub use soft::Aes128Soft as Aes128; +pub use zkn::Aes128; #[cfg(feature = "vexii-test")] pub use zkn::Aes256; diff --git a/services/aes/src/zkn.rs b/services/aes/src/zkn.rs index 760c2a5f6..65d56fc98 100644 --- a/services/aes/src/zkn.rs +++ b/services/aes/src/zkn.rs @@ -1,3 +1,8 @@ +// See https://github.com/riscv/riscv-crypto/tree/main/benchmarks/aes/zscrypto_rv32 +// for reference assembly code from which the routines in here are derived. +// Note that the order of the AES ops is shuffled across blocks versus ref code +// to break data dependencies and remove pipeline bubbles. + use cipher::{ AlgorithmName, BlockBackend, BlockCipher, BlockClosure, BlockDecrypt, BlockEncrypt, BlockSizeUser, Key, KeyInit, KeySizeUser, ParBlocksSizeUser, @@ -13,7 +18,7 @@ pub type Block = GenericArray; pub(crate) type BatchBlocks = GenericArray; -use core::{convert::TryInto, fmt}; +use core::fmt; /* The ratified AES instruction format is as follows: @@ -34,40 +39,147 @@ use core::{convert::TryInto, fmt}; dec-last 0010_1010 => 2A, 6A, AA, EA */ -core::arch::global_asm!( - ".global vex_aes_enc_id_0", - "vex_aes_enc_id_0:", - " .word 0x26b50533", // vex_aes_enc_id a0, a1, a0, #0 - " ret", - ".global vex_aes_enc_id_1", - "vex_aes_enc_id_1:", - " .word 0x66b50533", // vex_aes_enc_id a0, a1, a0, #1 - " ret", - ".global vex_aes_enc_id_2", - "vex_aes_enc_id_2:", - " .word 0xa6b50533", // vex_aes_enc_id a0, a1, a0, #2 - " ret", - ".global vex_aes_enc_id_3", - "vex_aes_enc_id_3:", - " .word 0xe6b50533", // vex_aes_enc_id a0, a1, a0, #3 - " ret", - ".global vex_aes_enc_id_last_0", - "vex_aes_enc_id_last_0:", - " .word 0x22b50533", // vex_aes_enc_id_last a0, a1, a0, #0 - " ret", - ".global vex_aes_enc_id_last_1", - "vex_aes_enc_id_last_1:", - " .word 0x62b50533", // vex_aes_enc_id_last a0, a1, a0, #1 - " ret", - ".global vex_aes_enc_id_last_2", - "vex_aes_enc_id_last_2:", - " .word 0xa2b50533", // vex_aes_enc_id_last a0, a1, a0, #2 - " ret", - ".global vex_aes_enc_id_last_3", - "vex_aes_enc_id_last_3:", - " .word 0xe2b50533", // vex_aes_enc_id_last a0, a1, a0, #3 - " ret", -); + +#[repr(align(32))] +#[derive(Default)] +struct AlignedCk128 { + d: [u8; 16], +} + +/// AES-128 round keys +pub(crate) type VexKeys128 = [u32; 60]; + +fn aes_key_schedule_128_wrapper(ck: &[u8]) -> VexKeys128 { + let mut ck_a = AlignedCk128::default(); + ck_a.d.copy_from_slice(&ck); + let mut rk: VexKeys128 = [0; 60]; + // Safety: safe because our target has the "zkn" RV32 extensions. + unsafe { aes_key_schedule_128(&mut rk, &ck_a.d) }; + rk +} + +#[target_feature(enable = "zkn")] +unsafe fn aes_key_schedule_128(rk: &mut VexKeys128, ck: &[u8]) { + #[rustfmt::skip] + unsafe { + // a0 - uint32_t rk [AES_256_RK_WORDS] + // a1 - uint8_t ck [AES_256_CK_BYTE ] + core::arch::asm!( + "lw a2, 0(a1)", // C0 + "lw a3, 4(a1)", // C1 + "lw a4, 8(a1)", // C2 + "lw a5, 12(a1)", // C3 + // RK a0 + // RKP a6 + // CK a1 + // RKE t0 + // RCP t1 + // RCT t2 + // T1 t3 + // T2 t4 + + "mv a6, a0", + "addi t0, a0, 160", + "la t1, 50f",// t1 = round constant pointer + + "30:", // Loop start + + "sw a2, 0(a6)", // rkp[0] = a2 + "sw a3, 4(a6)", // rkp[1] = a3 + "sw a4, 8(a6)", // rkp[2] = a4 + "sw a5, 12(a6)", // rkp[3] = a5 + + // if rke==rkp, return - loop break + "beq t0, a6, 40f", + + "addi a6, a6, 16", // increment rkp + + "lbu t2, 0(t1)", // Load round constant byte + "addi t1, t1, 1", // Increment round constant byte + "xor a2, a2, t2", // c0 ^= rcp + + // "ror32i t3, t4, a5, 8", // tr = ROR32(c3, 8) + "srli t4, a5, 8", + "slli t3, a5, 32-8", + "or t3, t3, t4", + + "aes32esi a2, a2, t3, 0", // tr = sbox(tr) + "aes32esi a2, a2, t3, 1", // + "aes32esi a2, a2, t3, 2", // + "aes32esi a2, a2, t3, 3", // + + "xor a3, a3, a2", // C1 ^= C0 + "xor a4, a4, a3", // C1 ^= C0 + "xor a5, a5, a4", // C1 ^= C0 + + "j 30b", // Loop continue + + "50:", + ".byte 0x01, 0x02, 0x04, 0x08, 0x10", + ".byte 0x20, 0x40, 0x80, 0x1b, 0x36", + + "40:", + "nop", // was ret + + in("a0") rk.as_mut_ptr(), + in("a1") ck.as_ptr(), + ); + }; +} + +fn aes128_dec_key_schedule_asm_wrapper(user_key: &[u8]) -> VexKeys128 { + let mut rk: VexKeys128 = [0; 60]; + let mut uk_a = AlignedCk128::default(); + uk_a.d.copy_from_slice(&user_key); + + unsafe { + aes_key_schedule_128(&mut rk, &uk_a.d); + } + unsafe { aes128_dec_key_schedule_asm(&mut rk, &uk_a.d) }; + rk +} + +#[target_feature(enable = "zkn")] +unsafe fn aes128_dec_key_schedule_asm(rk: &mut VexKeys128, user_key: &[u8]) { + #[rustfmt::skip] + unsafe { + core::arch::asm!( + // a0 - uint32_t RK [AES_256_RK_WORDS] + // a1 - uint8_t CK [AES_256_CK_BYTE ] + // a2 - RKP + // a3 - RKE + // T0 - t0 + // T1 - t1 + + "addi a2, a0, 16", // a0 = &rk[ 4] + "addi a3, a0, 160", // a1 = &rk[40] + + "20:", + + "lw t0, 0(a2)", // Load key word + + "li t1, 0", + "aes32esi t1, t1, t0, 0", // Sub Word Forward + "aes32esi t1, t1, t0, 1 ", + "aes32esi t1, t1, t0, 2", + "aes32esi t1, t1, t0, 3", + + "li t0, 0", + "aes32dsmi t0, t0, t1, 0", // Sub Word Inverse & Inverse MixColumns + "aes32dsmi t0, t0, t1, 1", + "aes32dsmi t0, t0, t1, 2", + "aes32dsmi t0, t0, t1, 3", + + "sw t0, 0(a2)", // Store key word. + + "addi a2, a2, 4", // Increment round key pointer + "bne a2, a3, 20b", // Finished yet? + + in("a0") rk.as_mut_ptr(), + in("a1") user_key.as_ptr(), + ); + }; +} #[repr(align(32))] #[derive(Default)] @@ -222,23 +334,23 @@ pub unsafe fn aes256_vexriscv_decrypt_asm( "lw t3, 28(a3)", "aes32dsmi t0, t0, a4, 0", // Even Round - "aes32dsmi t0, t0, a7, 1", - "aes32dsmi t0, t0, a6, 2", - "aes32dsmi t0, t0, a5, 3", - "aes32dsmi t1, t1, a5, 0", - "aes32dsmi t1, t1, a4, 1", - "aes32dsmi t1, t1, a7, 2", - "aes32dsmi t1, t1, a6, 3", - "aes32dsmi t2, t2, a6, 0", - "aes32dsmi t2, t2, a5, 1", - "aes32dsmi t2, t2, a4, 2", - "aes32dsmi t2, t2, a7, 3", - "aes32dsmi t3, t3, a7, 0", + + "aes32dsmi t0, t0, a7, 1", + "aes32dsmi t1, t1, a4, 1", + "aes32dsmi t2, t2, a5, 1", "aes32dsmi t3, t3, a6, 1", + + "aes32dsmi t0, t0, a6, 2", + "aes32dsmi t1, t1, a7, 2", + "aes32dsmi t2, t2, a4, 2", "aes32dsmi t3, t3, a5, 2", + + "aes32dsmi t0, t0, a5, 3", + "aes32dsmi t1, t1, a6, 3", + "aes32dsmi t2, t2, a7, 3", "aes32dsmi t3, t3, a4, 3", // U* contains new state "lw a4, 0(a3)", // Load Round Key @@ -250,23 +362,23 @@ pub unsafe fn aes256_vexriscv_decrypt_asm( "addi a3, a3, -32", // Step Key pointer "aes32dsmi a4, a4, t0, 0", // Odd Round - "aes32dsmi a4, a4, t3, 1", - "aes32dsmi a4, a4, t2, 2", - "aes32dsmi a4, a4, t1, 3", - "aes32dsmi a5, a5, t1, 0", - "aes32dsmi a5, a5, t0, 1", - "aes32dsmi a5, a5, t3, 2", - "aes32dsmi a5, a5, t2, 3", - "aes32dsmi a6, a6, t2, 0", - "aes32dsmi a6, a6, t1, 1", - "aes32dsmi a6, a6, t0, 2", - "aes32dsmi a6, a6, t3, 3", - "aes32dsmi a7, a7, t3, 0", + + "aes32dsmi a4, a4, t3, 1", + "aes32dsmi a5, a5, t0, 1", + "aes32dsmi a6, a6, t1, 1", "aes32dsmi a7, a7, t2, 1", + + "aes32dsmi a4, a4, t2, 2", + "aes32dsmi a5, a5, t3, 2", + "aes32dsmi a6, a6, t0, 2", "aes32dsmi a7, a7, t1, 2", + + "aes32dsmi a4, a4, t1, 3", + "aes32dsmi a5, a5, t2, 3", + "aes32dsmi a6, a6, t3, 3", "aes32dsmi a7, a7, t0, 3", // T* contains new state "j 20b", // .aes_dec_block_l0 // repeat loop @@ -274,23 +386,23 @@ pub unsafe fn aes256_vexriscv_decrypt_asm( "30:", //.aes_dec_block_l_finish: "aes32dsi a4, a4, t0, 0", // Final round, no MixColumns - "aes32dsi a4, a4, t3, 1", - "aes32dsi a4, a4, t2, 2", - "aes32dsi a4, a4, t1, 3", - "aes32dsi a5, a5, t1, 0", - "aes32dsi a5, a5, t0, 1", - "aes32dsi a5, a5, t3, 2", - "aes32dsi a5, a5, t2, 3", - "aes32dsi a6, a6, t2, 0", - "aes32dsi a6, a6, t1, 1", - "aes32dsi a6, a6, t0, 2", - "aes32dsi a6, a6, t3, 3", - "aes32dsi a7, a7, t3, 0", + + "aes32dsi a4, a4, t3, 1", + "aes32dsi a5, a5, t0, 1", + "aes32dsi a6, a6, t1, 1", "aes32dsi a7, a7, t2, 1", + + "aes32dsi a4, a4, t2, 2", + "aes32dsi a5, a5, t3, 2", + "aes32dsi a6, a6, t0, 2", "aes32dsi a7, a7, t1, 2", + + "aes32dsi a4, a4, t1, 3", + "aes32dsi a5, a5, t2, 3", + "aes32dsi a6, a6, t3, 3", "aes32dsi a7, a7, t0, 3", // T* contains new state "sw a4, 0(a0)", @@ -359,54 +471,136 @@ unsafe fn aes256_dec_key_schedule_asm(rk: &mut VexKeys256, user_key: &[u8]) { }; } -#[derive(Clone, Copy, Debug)] -pub(crate) enum AesByte { - Byte0 = 0, - Byte1 = 1, - Byte2 = 2, - Byte3 = 3, +pub fn aes_vexriscv_encrypt_asm_wrapper(key: &VexKeys256, block: &[u8], rounds: u32) -> [u8; 16] { + let mut pt = AlignedBlock { data: [0u8; 16] }; + pt.data.copy_from_slice(block); + let mut ct = AlignedBlock { data: [0u8; 16] }; + + // safe because our target architecture supports "zkn" + unsafe { aes256_vexriscv_encrypt_asm(key, &pt, &mut ct, rounds * 16) }; + ct.data } -pub(crate) fn aes_enc_round(arg1: u32, arg2: u32, id: AesByte) -> u32 { - extern "C" { - fn vex_aes_enc_id_0(arg1: u32, arg2: u32) -> u32; - fn vex_aes_enc_id_1(arg1: u32, arg2: u32) -> u32; - fn vex_aes_enc_id_2(arg1: u32, arg2: u32) -> u32; - fn vex_aes_enc_id_3(arg1: u32, arg2: u32) -> u32; - } - match id { - AesByte::Byte0 => unsafe { vex_aes_enc_id_0(arg1, arg2) }, - AesByte::Byte1 => unsafe { vex_aes_enc_id_1(arg1, arg2) }, - AesByte::Byte2 => unsafe { vex_aes_enc_id_2(arg1, arg2) }, - AesByte::Byte3 => unsafe { vex_aes_enc_id_3(arg1, arg2) }, - } -} +#[target_feature(enable = "zkn")] +pub unsafe fn aes256_vexriscv_encrypt_asm( + key: &VexKeys256, + pt: &AlignedBlock, + ct: &mut AlignedBlock, + rounds: u32, +) { + #[rustfmt::skip] + unsafe { + // a0 - uint8_t ct [16], + // a1 - uint8_t pt [16], + // a2 - uint32_t * rk, + core::arch::asm!( + "add a3, a2, a3", // kp = rk + 4*nr (rk = a2, kp = a3) -pub(crate) fn aes_enc_round_last(arg1: u32, arg2: u32, id: AesByte) -> u32 { - extern "C" { - fn vex_aes_enc_id_last_0(arg1: u32, arg2: u32) -> u32; - fn vex_aes_enc_id_last_1(arg1: u32, arg2: u32) -> u32; - fn vex_aes_enc_id_last_2(arg1: u32, arg2: u32) -> u32; - fn vex_aes_enc_id_last_3(arg1: u32, arg2: u32) -> u32; - } - match id { - AesByte::Byte0 => unsafe { vex_aes_enc_id_last_0(arg1, arg2) }, - AesByte::Byte1 => unsafe { vex_aes_enc_id_last_1(arg1, arg2) }, - AesByte::Byte2 => unsafe { vex_aes_enc_id_last_2(arg1, arg2) }, - AesByte::Byte3 => unsafe { vex_aes_enc_id_last_3(arg1, arg2) }, - } -} + "lw a4, 0(a1)", + "lw a5, 4(a1)", + "lw a6, 8(a1)", + "lw a7, 12(a1)", -pub(crate) fn get_u32_le(input: &[u8], offset: usize) -> u32 { - u32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()) -} + "lw t0, 0(a2)", // Load Round Key + "lw t1, 4(a2)", + "lw t2, 8(a2)", + "lw t3, 12(a2)", -pub(crate) fn set_u32(output: &mut [u8], offset: usize, value: u32) { - let tmp = value.to_le_bytes(); - output[offset + 0] = tmp[0]; - output[offset + 1] = tmp[1]; - output[offset + 2] = tmp[2]; - output[offset + 3] = tmp[3]; + "xor a4, a4, t0", // Add Round Key + "xor a5, a5, t1", + "xor a6, a6, t2", + "xor a7, a7, t3", + + "20:", // .aes_enc_block_l0: + + "lw t0, 16(a2)", // Load Round Key + "lw t1, 20(a2)", + "lw t2, 24(a2)", + "lw t3, 28(a2)", + + "aes32esmi t0, t0, a4, 0", // s0 // Even Round + "aes32esmi t1, t1, a5, 0", // s1 + "aes32esmi t2, t2, a6, 0", // s2 + "aes32esmi t3, t3, a7, 0", // s3 + + "aes32esmi t0, t0, a5, 1", + "aes32esmi t1, t1, a6, 1", + "aes32esmi t2, t2, a7, 1", + "aes32esmi t3, t3, a4, 1", + + "aes32esmi t0, t0, a6, 2", + "aes32esmi t1, t1, a7, 2", + "aes32esmi t2, t2, a4, 2", + "aes32esmi t3, t3, a5, 2", + + "aes32esmi t0, t0, a7, 3", + "aes32esmi t1, t1, a4, 3", + "aes32esmi t2, t2, a5, 3", + "aes32esmi t3, t3, a6, 3", // U* contains new state + + "lw a4, 32(a2)", // Load Round Key + "lw a5, 36(a2)", + "lw a6, 40(a2)", + "lw a7, 44(a2)", + + "addi a2, a2, 32", // Step Key pointer + "beq a2, a3, 30f", // aes_enc_block_l_finish Break from loop + + "aes32esmi a4, a4, t0, 0", // Odd Round + "aes32esmi a5, a5, t1, 0", + "aes32esmi a6, a6, t2, 0", + "aes32esmi a7, a7, t3, 0", + + "aes32esmi a4, a4, t1, 1", + "aes32esmi a5, a5, t2, 1", + "aes32esmi a6, a6, t3, 1", + "aes32esmi a7, a7, t0, 1", + + "aes32esmi a4, a4, t2, 2", + "aes32esmi a5, a5, t3, 2", + "aes32esmi a6, a6, t0, 2", + "aes32esmi a7, a7, t1, 2", + + "aes32esmi a4, a4, t3, 3", + "aes32esmi a5, a5, t0, 3", + "aes32esmi a6, a6, t1, 3", + "aes32esmi a7, a7, t2, 3", // T* contains new state + + "j 20b", // .aes_enc_block_l0 // repeat loop + + "30:", //.aes_enc_block_l_finish: + + "aes32esi a4, a4, t0, 0", // Final round, no MixColumns + "aes32esi a5, a5, t1, 0", + "aes32esi a6, a6, t2, 0", + "aes32esi a7, a7, t3, 0", + + "aes32esi a4, a4, t1, 1", + "aes32esi a5, a5, t2, 1", + "aes32esi a6, a6, t3, 1", + "aes32esi a7, a7, t0, 1", + + "aes32esi a4, a4, t2, 2", + "aes32esi a5, a5, t3, 2", + "aes32esi a6, a6, t0, 2", + "aes32esi a7, a7, t1, 2", + + "aes32esi a4, a4, t3, 3", + "aes32esi a5, a5, t0, 3", + "aes32esi a6, a6, t1, 3", + "aes32esi a7, a7, t2, 3", // T* contains new state + + "sw a4, 0(a0)", + "sw a5, 4(a0)", + "sw a6, 8(a0)", + "sw a7, 12(a0)", + + in("a0") ct.data.as_mut_ptr(), + in("a1") pt.data.as_ptr(), + in("a2") key.as_ptr(), + in("a3") rounds, + ); + }; } macro_rules! define_aes_impl { @@ -666,123 +860,22 @@ macro_rules! define_aes_impl { }; } -// We leave this one encoded in legacy style to prove interop between styles -pub fn aes_vexriscv_encrypt(key: &VexKeys256, block: &[u8], rounds: u32) -> [u8; 16] { - let rk = key; - let mut input: [u8; 16] = [0; 16]; - for (&src, dst) in block.iter().zip(input.iter_mut()) { - *dst = src; - } - - // We do two rounds per loop - let mut round_count = rounds / 2; - - let mut s0 = get_u32_le(&input, 0); - let mut s1 = get_u32_le(&input, 4); - let mut s2 = get_u32_le(&input, 8); - let mut s3 = get_u32_le(&input, 12); - - let mut t0 = rk[0]; - let mut t1 = rk[1]; - let mut t2 = rk[2]; - let mut t3 = rk[3]; - - s0 ^= t0; - s1 ^= t1; - s2 ^= t2; - s3 ^= t3; - - let mut rk_offset = 0; - loop { - t0 = rk[rk_offset + 4]; - t1 = rk[rk_offset + 5]; - t2 = rk[rk_offset + 6]; - t3 = rk[rk_offset + 7]; - - t0 = aes_enc_round(t0, s0, AesByte::Byte0); - t1 = aes_enc_round(t1, s1, AesByte::Byte0); - t2 = aes_enc_round(t2, s2, AesByte::Byte0); - t3 = aes_enc_round(t3, s3, AesByte::Byte0); - - t0 = aes_enc_round(t0, s1, AesByte::Byte1); - t1 = aes_enc_round(t1, s2, AesByte::Byte1); - t2 = aes_enc_round(t2, s3, AesByte::Byte1); - t3 = aes_enc_round(t3, s0, AesByte::Byte1); - - t0 = aes_enc_round(t0, s2, AesByte::Byte2); - t1 = aes_enc_round(t1, s3, AesByte::Byte2); - t2 = aes_enc_round(t2, s0, AesByte::Byte2); - t3 = aes_enc_round(t3, s1, AesByte::Byte2); - - t0 = aes_enc_round(t0, s3, AesByte::Byte3); - t1 = aes_enc_round(t1, s0, AesByte::Byte3); - t2 = aes_enc_round(t2, s1, AesByte::Byte3); - t3 = aes_enc_round(t3, s2, AesByte::Byte3); - - rk_offset += 8; - round_count -= 1; - if round_count == 0 { - break; - } - - s0 = rk[rk_offset + 0]; - s1 = rk[rk_offset + 1]; - s2 = rk[rk_offset + 2]; - s3 = rk[rk_offset + 3]; - - s0 = aes_enc_round(s0, t0, AesByte::Byte0); - s1 = aes_enc_round(s1, t1, AesByte::Byte0); - s2 = aes_enc_round(s2, t2, AesByte::Byte0); - s3 = aes_enc_round(s3, t3, AesByte::Byte0); - - s0 = aes_enc_round(s0, t1, AesByte::Byte1); - s1 = aes_enc_round(s1, t2, AesByte::Byte1); - s2 = aes_enc_round(s2, t3, AesByte::Byte1); - s3 = aes_enc_round(s3, t0, AesByte::Byte1); - - s0 = aes_enc_round(s0, t2, AesByte::Byte2); - s1 = aes_enc_round(s1, t3, AesByte::Byte2); - s2 = aes_enc_round(s2, t0, AesByte::Byte2); - s3 = aes_enc_round(s3, t1, AesByte::Byte2); - - s0 = aes_enc_round(s0, t3, AesByte::Byte3); - s1 = aes_enc_round(s1, t0, AesByte::Byte3); - s2 = aes_enc_round(s2, t1, AesByte::Byte3); - s3 = aes_enc_round(s3, t2, AesByte::Byte3); - } - - s0 = rk[rk_offset + 0]; - s1 = rk[rk_offset + 1]; - s2 = rk[rk_offset + 2]; - s3 = rk[rk_offset + 3]; - - s0 = aes_enc_round_last(s0, t0, AesByte::Byte0); - s1 = aes_enc_round_last(s1, t1, AesByte::Byte0); - s2 = aes_enc_round_last(s2, t2, AesByte::Byte0); - s3 = aes_enc_round_last(s3, t3, AesByte::Byte0); - - s0 = aes_enc_round_last(s0, t1, AesByte::Byte1); - s1 = aes_enc_round_last(s1, t2, AesByte::Byte1); - s2 = aes_enc_round_last(s2, t3, AesByte::Byte1); - s3 = aes_enc_round_last(s3, t0, AesByte::Byte1); - - s0 = aes_enc_round_last(s0, t2, AesByte::Byte2); - s1 = aes_enc_round_last(s1, t3, AesByte::Byte2); - s2 = aes_enc_round_last(s2, t0, AesByte::Byte2); - s3 = aes_enc_round_last(s3, t1, AesByte::Byte2); - - s0 = aes_enc_round_last(s0, t3, AesByte::Byte3); - s1 = aes_enc_round_last(s1, t0, AesByte::Byte3); - s2 = aes_enc_round_last(s2, t1, AesByte::Byte3); - s3 = aes_enc_round_last(s3, t2, AesByte::Byte3); - - let mut output: [u8; 16] = [0; 16]; - set_u32(&mut output, 0, s0); - set_u32(&mut output, 4, s1); - set_u32(&mut output, 8, s2); - set_u32(&mut output, 12, s3); - output -} +define_aes_impl!( + Aes128, + Aes128Enc, + Aes128Dec, + Aes128BackEnc, + Aes128BackDec, + U16, + 128, + VexKeys128, + 10, + aes128_dec_key_schedule_asm_wrapper, + aes_key_schedule_128_wrapper, + aes_vexriscv_decrypt_asm_wrapper, + aes_vexriscv_encrypt_asm_wrapper, + "AES-128 block cipher instance" +); define_aes_impl!( Aes256, @@ -797,6 +890,6 @@ define_aes_impl!( aes256_dec_key_schedule_asm_wrapper, aes_key_schedule_256_wrapper, aes_vexriscv_decrypt_asm_wrapper, - aes_vexriscv_encrypt, + aes_vexriscv_encrypt_asm_wrapper, "AES-256 block cipher instance" ); diff --git a/services/cram-mbox1/Cargo.toml b/services/cram-mbox1/Cargo.toml index 2e5b8506a..2b11503ff 100644 --- a/services/cram-mbox1/Cargo.toml +++ b/services/cram-mbox1/Cargo.toml @@ -14,10 +14,14 @@ log-server = { package = "xous-api-log", version = "0.1.65" } log = "0.4.14" num-derive = { version = "0.4.2", default-features = false } num-traits = { version = "0.2.14", default-features = false } +aes = { path = "../aes" } [features] "cramium" = ["utralib/cramium-fpga"] +"vexii-test" = ["aes/vexii-test"] +"aestests" = [] "hwsim" = [] +"message-test" = [] # ext specifies to use the APB client, instead of the loopback local "ext" = [] # must match setting in cram-mbox2 default = ["cramium"] diff --git a/services/cram-mbox1/src/main.rs b/services/cram-mbox1/src/main.rs index da9d4f748..c9c7eb170 100644 --- a/services/cram-mbox1/src/main.rs +++ b/services/cram-mbox1/src/main.rs @@ -168,6 +168,104 @@ fn main() { // enable the interrupt mailbox.csr.wo(utra::mailbox::EV_ENABLE, !0); // enable everything + #[cfg(feature = "message-test")] + { + #[cfg(feature = "hwsim")] + let c_csr = xous::syscall::map_memory( + xous::MemoryAddress::new(utra::main::HW_MAIN_BASE), + None, + 4096, + xous::MemoryFlags::R | xous::MemoryFlags::W, + ) + .expect("couldn't map Core Control CSR range"); + #[cfg(feature = "hwsim")] + let mut core_csr = CSR::new(c_csr.as_mut_ptr() as *mut u32); + + #[cfg(feature = "aestests")] + { + use aes::cipher::generic_array::GenericArray; + use aes::cipher::{BlockDecrypt, BlockEncrypt, KeyInit}; + log::info!("AES"); + const TEST_MAX_LEN: usize = 4096; + let mut dataset_op = [0u8; TEST_MAX_LEN]; + for (i, d) in dataset_op.iter_mut().enumerate() { + *d = i as u8; + } + log::info!("key"); + let key_array: [u8; 32] = [0; 32]; + let key = GenericArray::from_slice(&key_array); + let cipher_hw = aes::Aes256::new(&key); + log::info!("round"); + for iter in 0..2 { + core_csr.wfo(utra::main::REPORT_REPORT, 0xae50_0000 + iter); + for (i, mut chunk) in dataset_op.chunks_exact_mut(aes::BLOCK_SIZE).enumerate() { + let mut block = GenericArray::clone_from_slice(&mut chunk); + core_csr.wfo(utra::main::REPORT_REPORT, 0xae5e_0000 + iter + ((i as u32) << 16)); + cipher_hw.encrypt_block(&mut block); + core_csr.wfo(utra::main::REPORT_REPORT, 0xae5e_1000 + iter + ((i as u32) << 16)); + for (&src, dst) in block.iter().zip(chunk.iter_mut()) { + *dst = src; + } + } + for (i, mut chunk) in dataset_op.chunks_exact_mut(aes::BLOCK_SIZE).enumerate() { + let mut block = GenericArray::clone_from_slice(&mut chunk); + core_csr.wfo(utra::main::REPORT_REPORT, 0xae5d_0000 + iter + ((i as u32) << 16)); + cipher_hw.decrypt_block(&mut block); + core_csr.wfo(utra::main::REPORT_REPORT, 0xae5d_1000 + iter + ((i as u32) << 16)); + for (&src, dst) in block.iter().zip(chunk.iter_mut()) { + *dst = src; + } + } + if iter == 0 { + log::info!("0 done"); + } + } + log::info!("done"); + } + + let tt = xous_api_ticktimer::Ticktimer::new().unwrap(); + let mut total = 0; + let mut iter = 0; + log::info!("running message passing test"); + loop { + // this conjures a scalar message + #[cfg(feature = "hwsim")] + core_csr.wfo(utra::main::REPORT_REPORT, 0x1111_0000 + iter); + let now = tt.elapsed_ms(); + #[cfg(feature = "hwsim")] + core_csr.wfo(utra::main::REPORT_REPORT, 0x2222_0000 + iter); + total += now; + + if iter >= 8 && iter < 12 { + #[cfg(feature = "hwsim")] + core_csr.wfo(utra::main::REPORT_REPORT, 0x5133_D001); + tt.sleep_ms(1).ok(); + } else if iter >= 12 && iter < 13 { + #[cfg(feature = "hwsim")] + core_csr.wfo(utra::main::REPORT_REPORT, 0x5133_D002); + tt.sleep_ms(2).ok(); + } else if iter >= 13 && iter < 14 { + #[cfg(feature = "hwsim")] + core_csr.wfo(utra::main::REPORT_REPORT, 0x5133_D002); + tt.sleep_ms(3).ok(); + } else if iter >= 14 { + break; + } + + // something lame to just conjure a memory message + #[cfg(feature = "hwsim")] + core_csr.wfo(utra::main::REPORT_REPORT, 0x3333_0000 + iter); + let version = tt.get_version(); + #[cfg(feature = "hwsim")] + core_csr.wfo(utra::main::REPORT_REPORT, 0x4444_0000 + iter); + total += version.len() as u64; + iter += 1; + #[cfg(feature = "hwsim")] + core_csr.wfo(utra::main::REPORT_REPORT, now as u32); + log::info!("message passing test progress: {}ms", tt.elapsed_ms()); + } + log::info!("total: {}", total); // consume the total value so it's not optimized out + } // tests to run: // all tests take the form of loopback, data transmit -> data receive ^ 0xaaaa_0000 // diff --git a/services/shellchat/src/cmds/aes_cmd.rs b/services/shellchat/src/cmds/aes_cmd.rs index 58ad6160b..f5ea0b5d7 100644 --- a/services/shellchat/src/cmds/aes_cmd.rs +++ b/services/shellchat/src/cmds/aes_cmd.rs @@ -161,7 +161,8 @@ software 140.84us/block enc+dec aes256 (500 iters, 8192 len) with Vex-ii CPU (50MHz) (note that Vex-i runs at 100MHz) on vexii-testing branch: software 202us/block enc+dec aes256 (500 iters, 8192 len) -hardware 60us/block enc+dec aes256 (500 iters, 8192 len) +hardware 60us/block enc+dec aes256 (500 iters, 8192 len) [initial] +hardware 25.9us/block enc+dec aes256 (500 iters, 8192 len) [optimized] */ pub fn benchmark_thread(sid0: usize, sid1: usize, sid2: usize, sid3: usize) { let sid = xous::SID::from_u32(sid0 as u32, sid1 as u32, sid2 as u32, sid3 as u32); diff --git a/utralib/cramium/soc.svd b/utralib/cramium/soc.svd index 3612be918..d2a669edd 100644 --- a/utralib/cramium/soc.svd +++ b/utralib/cramium/soc.svd @@ -3,7 +3,7 @@ litex SOC - + 8 32 @@ -75,10 +75,25 @@ + + REPORT + 0x0008 + 0x00 + 32 + + + report + 31 + [31:0] + 0 + + + + 0 - 0x8 + 0xc registers diff --git a/utralib/src/generated/cramium_fpga.rs b/utralib/src/generated/cramium_fpga.rs index 2ab8da9cc..54f5118ca 100644 --- a/utralib/src/generated/cramium_fpga.rs +++ b/utralib/src/generated/cramium_fpga.rs @@ -458,7 +458,7 @@ pub mod utra { } pub mod main { - pub const MAIN_NUMREGS: usize = 2; + pub const MAIN_NUMREGS: usize = 3; pub const DONE: crate::Register = crate::Register::new(0, 0x1); pub const DONE_DONE: crate::Field = crate::Field::new(1, 0, DONE); @@ -466,6 +466,9 @@ pub mod utra { pub const IRQTEST0: crate::Register = crate::Register::new(1, 0xffff); pub const IRQTEST0_TRIGGER: crate::Field = crate::Field::new(16, 0, IRQTEST0); + pub const REPORT: crate::Register = crate::Register::new(2, 0xffffffff); + pub const REPORT_REPORT: crate::Field = crate::Field::new(32, 0, REPORT); + pub const HW_MAIN_BASE: usize = 0x40081000; } @@ -5676,6 +5679,14 @@ mod tests { let mut baz = main_csr.zf(utra::main::IRQTEST0_TRIGGER, bar); baz |= main_csr.ms(utra::main::IRQTEST0_TRIGGER, 1); main_csr.wfo(utra::main::IRQTEST0_TRIGGER, baz); + + let foo = main_csr.r(utra::main::REPORT); + main_csr.wo(utra::main::REPORT, foo); + let bar = main_csr.rf(utra::main::REPORT_REPORT); + main_csr.rmwf(utra::main::REPORT_REPORT, bar); + let mut baz = main_csr.zf(utra::main::REPORT_REPORT, bar); + baz |= main_csr.ms(utra::main::REPORT_REPORT, 1); + main_csr.wfo(utra::main::REPORT_REPORT, baz); } #[test] diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 8ef94547a..ad6af81e7 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -560,17 +560,25 @@ fn main() -> Result<(), Box> { // placement in flash is a tension between dev convenience and RAM usage. Things in flash // are resident, non-swapable, but end up making the slow kernel burn process take longer. - let cramium_flash_pkgs = - ["xous-log", "xous-names", "cram-mbox1", "cram-mbox2" /* "cram-hal-service" */].to_vec(); + let cramium_flash_pkgs = [ + "xous-log", + "xous-names", + "xous-ticktimer", + "cram-mbox1", + "cram-mbox2", /* "cram-hal-service" */ + ] + .to_vec(); let cramium_swap_pkgs = [].to_vec(); builder.add_loader_feature("debug-print"); + builder.add_loader_feature("verilator-only"); // builder.add_kernel_feature("debug-print"); // builder.add_kernel_feature("debug-swap-verbose"); // builder.add_feature("quantum-timer"); // builder.add_feature("auto-trng"); // automatically initialize TRNG tester inside USB stack builder.add_kernel_feature("v2p"); + builder.add_kernel_feature("verilator-only"); // builder.add_feature("mass-storage"); // builder.add_feature("ditherpunk");