saymrwulf/xous-core
1
0
Fork 0
mirror of https://github.com/saymrwulf/xous-core.git synced 2026-11-08 04:56:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'pddb-bulk' into pddb-bulk-bufserializer

Browse source
This commit is contained in:
bunnie 2022-10-27 01:30:27 +08:00
commit ac01088ec7
3 changed files with 77 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
apps/vault/src/actions.rs
View file

@ -684,6 +684,50 @@ impl<'a> ActionManager<'a> {
match self.mode_cache {
VaultMode::Password => {
let start = self.tt.elapsed_ms();
let mut klen = 0;
match self.pddb.borrow().read_dict(VAULT_PASSWORD_DICT, None, Some(256 * 1024)) {
Ok(keys) => {
let mut oom_keys = 0;
for key in keys {
if let Some(data) = key.data {
if let Some(pw) = storage::PasswordRecord::try_from(data).ok() {
let extra = format!("{}; {}{}",
crate::ux::atime_to_str(pw.atime),
t!("vault.u2f.appinfo.authcount", xous::LANG),
pw.count,
);
let desc = format!("{}/{}", pw.description, pw.username);
let li = ListItem {
name: desc,
extra,
dirty: true,
guid: key.name,
};
klen += 1;
il.push(li);
}
} else {
oom_keys += 1;
}
}
if oom_keys != 0 {
log::warn!("Ran out of cache space handling password keys. {} keys are not loaded.", oom_keys);
self.report_err(&format!("Ran out of cache space handling passwords. {} passwords not loaded", oom_keys), None::<crate::storage::Error>);
}
}
Err(e) => {
match e.kind() {
ErrorKind::NotFound => {
// this is fine, it just means no passwords have been entered yet
},
_ => {
log::error!("Error opening password dictionary");
self.report_err("Error opening password dictionary", Some(e))
}
}
},
}
/*
#[cfg(feature="vaultperf")]
self.perfentry(&self.pm, PERFMETA_STARTBLOCK, 1, std::line!());
let keylist = match self.pddb.borrow().list_keys(VAULT_PASSWORD_DICT, None) {
@ -771,6 +815,7 @@ impl<'a> ActionManager<'a> {
}
#[cfg(feature="vaultperf")]
self.perfentry(&self.pm, PERFMETA_ENDBLOCK, 2, std::line!());
*/
log::info!("readout took {} ms for {} elements", self.tt.elapsed_ms() - start, klen);
}
VaultMode::Fido => {

8
services/pddb/src/lib.rs
View file

@ -767,7 +767,7 @@ impl Pddb {
/// Defaults to a size limit of up to 32k of bulk data returned, if it is not explicitly specified.
pub fn read_dict(&self, dict: &str, basis: Option<&str>, size_limit: Option<usize>) -> Result<Vec::<PddbKeyRecord>> {
// about the biggest we can move around in Precursor and not break heap.
const MAX_BUFLEN: usize = 128 * 1024;
const MAX_BUFLEN: usize = 32 * 1024;
// compromise between memory zeroing time and latency to send a message
const DEFAULT_LIMIT: usize = 32 * 1024;
/*
@ -898,7 +898,7 @@ impl Pddb {
log::error!("local key index did not match remote key index: {}, {}", check_key_index, header.starting_key_index);
}
let mut key_count = 0;
// log::info!("header: {:?}", header);
log::info!("header: {:?}", header);
while key_count < header.len {
if index + size_of::<u32>() * 2 > msg_mem.len() {
// quit if we don't have enough space to decode at least another two indices
@ -908,7 +908,7 @@ impl Pddb {
index += size_of::<u32>();
let pos = u32::from_le_bytes(msg_mem.as_slice()[index..index + size_of::<u32>()].try_into().unwrap());
index += size_of::<u32>();
// log::info!("unpacking message at {}({})", size, pos);
log::info!("unpacking message at {}({})", size, pos);
if size != 0 && pos != 0 {
let archived = unsafe {
archived_value::<PddbKeyRecord>(&msg_mem.as_slice()[index..index + (size as usize)], pos as usize)
@ -939,7 +939,7 @@ impl Pddb {
if check_total_keys != check_key_index {
log::error!("Number of keys read does not match expected value: {}, {}", check_total_keys, check_key_index);
}
xous::unmap_memory(msg_mem).unwrap();
Ok(ret)
}

41
services/pddb/src/main.rs
View file

@ -407,6 +407,12 @@ use rkyv::{
use core::mem::size_of;
use core::ops::Deref;
//--------------------------------
//--------------------------------
#[cfg(feature="perfcounter")]
const FILE_ID_SERVICES_PDDB_SRC_MAIN: u32 = 0;
#[cfg(feature="perfcounter")]
@ -1405,13 +1411,16 @@ fn wrapped_main() -> ! {
}
// Optimized bulk data read handler. See lib.rs for documentation.
// TO INVESTIGATE: probably serialization can be made a lot more efficient by
// using a `BufferSerializer` instead of a WriteSeriaizer. I think we can also
// stack multiple records into the buffer without having to re-allocate the serializer,
// but it's not clear to me how the deserialization would work in that case. The main
// problem of using the BufferSerializer is the interior mutability problem of re-using the
// allocated slice across multiple iterations of the loop, especially with the property where
// if the record doesn't fit we want to stash it elsewhere for use on the next iteration.
// Note: it would be tempting to use a `BufferSerializer` instead of a WriteSeriaizer.
// This would prevent the extra allocation and copy into the buffer. The problem is that
// the BufferSerializer can return you the `pos` of the archive, but it won't tell you
// the length of the data that's been archived. Thus, when you call `into_inner()` you get
// back the original buffer (which is great -- we could split that and pass it on to future
// iterations) but we don't know how much data was put in there. Maybe later versions of rkyv
// have a call for that, but the current version we are using doesn't seem to be written to
// allow for multiple rkyv objects to be packed into a single packet. I think the assumption
// is rather that you're writing rkyv objects into a single file and the files are then packed
// into disk by the filesystem.
Opcode::DictBulkRead => {
const BULKREAD_TIMEOUT_MS: u64 = 5000;
let range = msg.body.memory_message_mut().unwrap();
@ -1475,6 +1484,7 @@ fn wrapped_main() -> ! {
};
bulkread_state = Some(state);
}
let mut finished = false;
// handle data packing into the structure
if let Some(state) = &mut bulkread_state {
// check that the token matches, if this isn't a first call to the function
@ -1491,8 +1501,7 @@ fn wrapped_main() -> ! {
header.token = state.token;
// now loop through and pack data into the slice
let (header_buf, buf) = buf.split_at_mut(size_of::<BulkReadHeader>());
let mut finished = false;
let (header_buf, mut buf) = buf.split_at_mut(size_of::<BulkReadHeader>());
enum SerializeResult<'a> {
Success(usize, &'a mut [u8], String, &'a mut [u8]),
Failure(String)
@ -1549,7 +1558,7 @@ fn wrapped_main() -> ! {
}
}
} else {
// log::info!("hit size limit: limit {} total {}", state.read_limit, state.read_total);
log::info!("hit size limit: limit {} total {}", state.read_limit, state.read_total);
// report the key, but with no data
let rec = PddbKeyRecord {
name: key_name.to_string(),
@ -1578,20 +1587,23 @@ fn wrapped_main() -> ! {
// pointer to, to stack the next archived result. The other option would be to use the .write()
// method but then we'd need to...make a serializer for the structure. which is agains the whole point of it...
match ser_result {
SerializeResult::Success(pos, buf, key_name, pre_buf) => {
log::info!("packing message of {}({})", buf.len(), pos);
SerializeResult::Success(pos, sbuf, _key_name, pre_buf) => {
log::info!("packing message of {}({})", sbuf.len(), pos);
// data can fit, copy it into the buffer.
state.buf_starting_key_index += 1;
header.len += 1;
// read length increment was handled when the data was copied into the serialization buffer.
pre_buf[..4].copy_from_slice(
&(buf.len() as u32).to_le_bytes()
//&(sbuf.len() as u32).to_le_bytes()
&(pos as u32).to_le_bytes()
);
pre_buf[4..8].copy_from_slice(
&(pos as u32).to_le_bytes()
);
(_, buf) = sbuf.split_at_mut(pos);
}
SerializeResult::Failure(key_name) => {
log::info!("ran out of space filling buffer, pushing {} back into the queue", key_name);
// data didn't fit, quit with finished = false;
state.key_list.push(key_name);
break;
@ -1611,6 +1623,9 @@ fn wrapped_main() -> ! {
} else {
log::warn!("This should be unreachable, the state should always be initialized by this point");
}
if finished {
bulkread_state = None;
}
}
Opcode::SeekKeyStd => {