From 317162913191eacb72db3f06ee6f79549cd62c2b Mon Sep 17 00:00:00 2001 From: bunnie Date: Sun, 23 Oct 2022 23:15:48 +0800 Subject: [PATCH] add PDDB performance profiling to shellchat --- services/shellchat/Cargo.toml | 5 +- services/shellchat/src/cmds.rs | 14 +-- services/shellchat/src/cmds/net_cmd.rs | 16 +-- services/shellchat/src/cmds/pddb_cmd.rs | 159 ++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 16 deletions(-) diff --git a/services/shellchat/Cargo.toml b/services/shellchat/Cargo.toml index a1a9861ab..eb1bb36bc 100644 --- a/services/shellchat/Cargo.toml +++ b/services/shellchat/Cargo.toml @@ -80,7 +80,10 @@ webpki-roots = {version = "0.22.4", optional = true} ring = {version = "0.16.20", optional = true} # for testing ring math functions # note requirement for patch to xous-ring in workspace Cargo.toml tungstenite = {version = "0.17.3", optional = true, features = ["rustls-tls-webpki-roots"]} url = {version = "2.2.2", optional = true} + +# for performance testing perflib = {path="../../tools/perflib", optional = true} +random-pick = {version = "1.2.15", optional = true} # for the curve25519 tests # hardware acceleration adaptations are inserted into a fork of the main branch. @@ -121,5 +124,5 @@ test-rekey = [] no-codec = [] nettest = [] # batch network tests tls = ["rustls", "webpki-roots", "ring", "tungstenite", "url"] -perfcounter = ["ring", "perflib", "xous/v2p"] +shellperf = ["ring", "perflib", "xous/v2p", "random-pick"] # this also needs the project-wide "perfcounter" feature to be selected. shellchat is the exclusive manager of the performance counter, do not use with e.g. "vaultperf" feature default = [] # "debugprint" diff --git a/services/shellchat/src/cmds.rs b/services/shellchat/src/cmds.rs index e73a8a6a6..397eeb8e9 100644 --- a/services/shellchat/src/cmds.rs +++ b/services/shellchat/src/cmds.rs @@ -1,7 +1,7 @@ use xous::{MessageEnvelope}; use xous_ipc::String; use core::fmt::Write; -#[cfg(feature="perfcounter")] +#[cfg(feature="shellperf")] use utralib::generated::*; use std::collections::HashMap; @@ -52,9 +52,9 @@ pub struct CommonEnv { xns: xous_names::XousNames, boot_instant: std::time::Instant, /// make this communal so any number of commands can trigger or reset the performance counter, and/or perform logging - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] perf_csr: AtomicCsr, - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] event_csr: AtomicCsr::, } impl CommonEnv { @@ -175,7 +175,7 @@ pub struct CmdEnv { impl CmdEnv { pub fn new(xns: &xous_names::XousNames) -> CmdEnv { let ticktimer = ticktimer_server::Ticktimer::new().expect("Couldn't connect to Ticktimer"); - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] let perf_csr = xous::syscall::map_memory( xous::MemoryAddress::new(utra::perfcounter::HW_PERFCOUNTER_BASE), None, @@ -183,7 +183,7 @@ impl CmdEnv { xous::MemoryFlags::R | xous::MemoryFlags::W, ) .expect("couldn't map perfcounter CSR range"); - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] let event1_csr = xous::syscall::map_memory( xous::MemoryAddress::new(utra::event_source1::HW_EVENT_SOURCE1_BASE), None, @@ -202,9 +202,9 @@ impl CmdEnv { xns: xous_names::XousNames::new().unwrap(), netmgr: net::NetManager::new(), boot_instant: std::time::Instant::now(), - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] perf_csr: AtomicCsr::new(perf_csr.as_mut_ptr() as *mut u32), - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] event_csr: AtomicCsr::new(event1_csr.as_mut_ptr() as *mut u32), }; //let fcc = Fcc::new(&mut common); diff --git a/services/shellchat/src/cmds/net_cmd.rs b/services/shellchat/src/cmds/net_cmd.rs index a862095ab..97f42393f 100644 --- a/services/shellchat/src/cmds/net_cmd.rs +++ b/services/shellchat/src/cmds/net_cmd.rs @@ -22,7 +22,7 @@ use gam::DecodePng; use std::convert::TryInto; #[cfg(feature="tls")] use tungstenite::{WebSocket, stream::MaybeTlsStream}; -#[cfg(feature="perfcounter")] +#[cfg(feature="shellperf")] use perflib::*; pub struct NetCmd { callback_id: Option, @@ -32,12 +32,12 @@ pub struct NetCmd { ping: Option, #[cfg(feature="tls")] ws: Option>>, - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] perfbuf: xous::MemoryRange, } impl NetCmd { pub fn new(xns: &xous_names::XousNames) -> Self { - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] let perfbuf = xous::syscall::map_memory( None, None, @@ -53,7 +53,7 @@ impl NetCmd { ping: None, #[cfg(feature="tls")] ws: None, - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] perfbuf, } } @@ -511,7 +511,7 @@ impl<'a> ShellCmdApi<'a> for NetCmd { log::info!("{}", std::str::from_utf8(&plaintext).unwrap_or("utf-error")); log::set_max_level(log::LevelFilter::Info); } - // note: to use the `perfcounter` option, you need to load a version of the SOC that has the performance counters built in. + // note: to use the `shellperf` option, you need to load a version of the SOC that has the performance counters built in. // this can be generated using the command `python3 .\betrusted_soc.py -e .\dummy.nky --perfcounter` in the betrusted-soc repo. // // to read out performance monitoring data, use the `usb_update.py` script as follows: @@ -521,7 +521,7 @@ impl<'a> ShellCmdApi<'a> for NetCmd { // the command below can generate this page table; it must be manually copy/pasted from the console // into the `v2p.txt` file. Note that it will change on every boot, but it is stable throughout each boot, // even across multiple runs of the command. - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] "v2p" => { // don't generate a new object since it clears the data, just do a raw dump log::info!("Buf vmem loc: {:x}", self.perfbuf.as_ptr() as u32); @@ -535,7 +535,7 @@ impl<'a> ShellCmdApi<'a> for NetCmd { ); } } - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] "cta1" => { log::info!("constant time RISC-V HW AES test"); @@ -583,7 +583,7 @@ impl<'a> ShellCmdApi<'a> for NetCmd { } // pm.print_page_table(); } - #[cfg(feature="perfcounter")] + #[cfg(feature="shellperf")] "cta2" => { let pm = PerfMgr::new( self.perfbuf.as_mut_ptr(), diff --git a/services/shellchat/src/cmds/pddb_cmd.rs b/services/shellchat/src/cmds/pddb_cmd.rs index b66c0c0b9..bb63bb419 100644 --- a/services/shellchat/src/cmds/pddb_cmd.rs +++ b/services/shellchat/src/cmds/pddb_cmd.rs @@ -6,15 +6,46 @@ use xous_ipc::String; use std::io::{Write, Read, Seek, SeekFrom}; use core::fmt::Write as FmtWrite; +#[cfg(feature="shellperf")] +const TEST_DICT: &'static str = "perftest"; +#[cfg(feature="shellperf")] +use perflib::*; +#[cfg(feature="shellperf")] +use sha2::Digest; +#[cfg(feature="shellperf")] +const FILE_ID_SERVICES_SHELLCHAT_SRC_CMDS_PDDB_CMD: u32 = 2; + pub struct PddbCmd { pddb: pddb::Pddb, + #[cfg(feature="shellperf")] + perfbuf: xous::MemoryRange, + #[cfg(feature="shellperf")] + my_pid: u32, } impl PddbCmd { pub fn new(_xns: &xous_names::XousNames) -> PddbCmd { + #[cfg(feature="shellperf")] + let perfbuf = xous::syscall::map_memory( + None, + None, + BUFLEN, + xous::MemoryFlags::R | xous::MemoryFlags::W | xous::MemoryFlags::RESERVE, + ).expect("couldn't map in the performance buffer"); + PddbCmd { pddb: pddb::Pddb::new(), + #[cfg(feature="shellperf")] + perfbuf, + #[cfg(feature="shellperf")] + my_pid: xous::process::id() as u32, } } + #[cfg(feature="shellperf")] + #[inline] + pub fn perfentry(&self, pm: &PerfMgr, meta: u32, index: u32, line: u32) { + let event = perf_entry!(self.my_pid, FILE_ID_SERVICES_SHELLCHAT_SRC_CMDS_PDDB_CMD, meta, index, line); + pm.log_event_unchecked(event); + } } impl<'a> ShellCmdApi<'a> for PddbCmd { @@ -896,6 +927,123 @@ impl<'a> ShellCmdApi<'a> for PddbCmd { } } } + #[cfg(feature="shellperf")] + "v2p" => { + // don't generate a new object since it clears the data, just do a raw dump + log::info!("Buf vmem loc: {:x}", self.perfbuf.as_ptr() as u32); + log::info!("Buf pmem loc: {:x}", xous::syscall::virt_to_phys(self.perfbuf.as_ptr() as usize).unwrap_or(0)); + log::info!("PerfLogEntry size: {}", core::mem::size_of::()); + log::info!("Now printing the page table mapping for the performance buffer:"); + for page in (0..BUFLEN).step_by(4096) { + log::info!("V|P {:x} {:x}", + self.perfbuf.as_ptr() as usize + page, + xous::syscall::virt_to_phys(self.perfbuf.as_ptr() as usize + page).unwrap_or(0), + ); + } + } + #[cfg(feature="shellperf")] + "perf" => { + log::info!("PDDB performance profiling"); + + // event_source0 is for the kernel + // event_source1 is for shellchat + // event_source2 is for pddb process + + let pm = PerfMgr::new( + self.perfbuf.as_mut_ptr(), + _env.perf_csr.clone(), + _env.event_csr.clone() // event_source1 + ); + log::info!("cleaning out old records (some stale records are expected)"); + pm.stop_and_reset(); + log::info!("run starting..."); + + // this starts the performance counter + pm.start(); + + for index in 0..10 { + self.perfentry(&pm, PERFMETA_STARTBLOCK, index, std::line!()); + let pws = self.pddb.list_keys(TEST_DICT, None); + self.perfentry(&pm, PERFMETA_NONE, index, std::line!()); + let entries = pws.unwrap_or(Vec::new()); + self.perfentry(&pm, PERFMETA_NONE, index, std::line!()); + log::info!("total pws found: {}", entries.len()); + self.perfentry(&pm, PERFMETA_ENDBLOCK, index, std::line!()); + pm.flush().ok(); + } + + match pm.stop_and_flush() { + Ok(entries) => { + log::info!("entries: {}", entries); + } + _ => { + log::info!("Perfcounter OOM'd during run"); + } + } + } + #[cfg(feature="shellperf")] + "perfprep" => { + log::info!("Preparing test set for performance profiling"); + let words = [ + "bunnie", "foo", "turtle.net", "Fox.ng", "Bear", "dog food", "Cat.com", "FUzzy", "1off", "www_test_site_com/long_name/stupid/foo.htm", + "._weird~yy%\":'test", "//WHYwhyWHY", "Xyz|zy", "foo:bar", "f🍕🍔🍟🌭d", "💎🙌", "some ノート", "笔录4u", "@u", "sane text", "Käsesoßenrührlöffel"]; + let weights = [1; 21]; + const TARGET_ENTRIES: usize = 300; + + let pws = self.pddb.list_keys(TEST_DICT, None).unwrap_or(Vec::new()); + if pws.len() < TARGET_ENTRIES { + let extra_count = TARGET_ENTRIES - pws.len(); + for _index in 0..extra_count { + let desc = random_pick::pick_multiple_from_slice(&words, &weights, 3); + let description = format!("{} {} {}", desc[0], desc[1], desc[2]); + let username = random_pick::pick_from_slice(&words, &weights).unwrap().to_string(); + let notes = random_pick::pick_from_slice(&words, &weights).unwrap().to_string(); + let record = format!( + "{}:{}\n{}:{}\n{}:{}\n{}:{}\n{}:{}\n{}:{}\n{}:{}\n{}:{}\n", + "version", + 2, + "description", + description, + "username", + username, + "password", + "dummy password", + "notes", + notes, + "ctime", + 0, + "atime", + 0, + "count", + 1, + ) + .into_bytes(); + + let mut h = sha2::Sha256::new(); + h.update(description.as_bytes()); + h.update(username.as_bytes()); + let hash = h.finalize().to_vec(); + + match self.pddb.get( + TEST_DICT, + &hex(hash), + None, + true, + true, + Some(128), + None::, + ) { + Ok(mut data) => match data.write(&record) { + Ok(_) => {}, + Err(e) => log::info!("error creating test keys: {:?}", e), + }, + Err(e) => log::info!("error creating test keys: {:?}", e), + } + } + log::info!("test dictionary should have {} keys", pws.len() + extra_count); + self.pddb.sync().unwrap(); + } + } _ => { write!(ret, "{}", helpstring).unwrap(); } @@ -967,3 +1115,14 @@ fn join_tokens<'a>(buf: &mut String<1024>, tokens: impl Iterator } } } + +#[cfg(feature="perflib")] +fn hex(data: Vec) -> std::string::String { + use std::fmt::Write; + let mut s = std::string::String::with_capacity(2 * data.len()); + for byte in data { + write!(s, "{:02X}", byte).unwrap(); + } + + s +} \ No newline at end of file