mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-18 08:52:45 +00:00
Merge branch 'pddb-bulk' into pddb-bulk-bufserializer
This commit is contained in:
commit
ac01088ec7
3 changed files with 77 additions and 17 deletions
|
|
@ -684,6 +684,50 @@ impl<'a> ActionManager<'a> {
|
||||||
match self.mode_cache {
|
match self.mode_cache {
|
||||||
VaultMode::Password => {
|
VaultMode::Password => {
|
||||||
let start = self.tt.elapsed_ms();
|
let start = self.tt.elapsed_ms();
|
||||||
|
let mut klen = 0;
|
||||||
|
match self.pddb.borrow().read_dict(VAULT_PASSWORD_DICT, None, Some(256 * 1024)) {
|
||||||
|
Ok(keys) => {
|
||||||
|
let mut oom_keys = 0;
|
||||||
|
for key in keys {
|
||||||
|
if let Some(data) = key.data {
|
||||||
|
if let Some(pw) = storage::PasswordRecord::try_from(data).ok() {
|
||||||
|
let extra = format!("{}; {}{}",
|
||||||
|
crate::ux::atime_to_str(pw.atime),
|
||||||
|
t!("vault.u2f.appinfo.authcount", xous::LANG),
|
||||||
|
pw.count,
|
||||||
|
);
|
||||||
|
let desc = format!("{}/{}", pw.description, pw.username);
|
||||||
|
let li = ListItem {
|
||||||
|
name: desc,
|
||||||
|
extra,
|
||||||
|
dirty: true,
|
||||||
|
guid: key.name,
|
||||||
|
};
|
||||||
|
klen += 1;
|
||||||
|
il.push(li);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
oom_keys += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if oom_keys != 0 {
|
||||||
|
log::warn!("Ran out of cache space handling password keys. {} keys are not loaded.", oom_keys);
|
||||||
|
self.report_err(&format!("Ran out of cache space handling passwords. {} passwords not loaded", oom_keys), None::<crate::storage::Error>);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
match e.kind() {
|
||||||
|
ErrorKind::NotFound => {
|
||||||
|
// this is fine, it just means no passwords have been entered yet
|
||||||
|
},
|
||||||
|
_ => {
|
||||||
|
log::error!("Error opening password dictionary");
|
||||||
|
self.report_err("Error opening password dictionary", Some(e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
/*
|
||||||
#[cfg(feature="vaultperf")]
|
#[cfg(feature="vaultperf")]
|
||||||
self.perfentry(&self.pm, PERFMETA_STARTBLOCK, 1, std::line!());
|
self.perfentry(&self.pm, PERFMETA_STARTBLOCK, 1, std::line!());
|
||||||
let keylist = match self.pddb.borrow().list_keys(VAULT_PASSWORD_DICT, None) {
|
let keylist = match self.pddb.borrow().list_keys(VAULT_PASSWORD_DICT, None) {
|
||||||
|
|
@ -771,6 +815,7 @@ impl<'a> ActionManager<'a> {
|
||||||
}
|
}
|
||||||
#[cfg(feature="vaultperf")]
|
#[cfg(feature="vaultperf")]
|
||||||
self.perfentry(&self.pm, PERFMETA_ENDBLOCK, 2, std::line!());
|
self.perfentry(&self.pm, PERFMETA_ENDBLOCK, 2, std::line!());
|
||||||
|
*/
|
||||||
log::info!("readout took {} ms for {} elements", self.tt.elapsed_ms() - start, klen);
|
log::info!("readout took {} ms for {} elements", self.tt.elapsed_ms() - start, klen);
|
||||||
}
|
}
|
||||||
VaultMode::Fido => {
|
VaultMode::Fido => {
|
||||||
|
|
|
||||||
|
|
@ -767,7 +767,7 @@ impl Pddb {
|
||||||
/// Defaults to a size limit of up to 32k of bulk data returned, if it is not explicitly specified.
|
/// Defaults to a size limit of up to 32k of bulk data returned, if it is not explicitly specified.
|
||||||
pub fn read_dict(&self, dict: &str, basis: Option<&str>, size_limit: Option<usize>) -> Result<Vec::<PddbKeyRecord>> {
|
pub fn read_dict(&self, dict: &str, basis: Option<&str>, size_limit: Option<usize>) -> Result<Vec::<PddbKeyRecord>> {
|
||||||
// about the biggest we can move around in Precursor and not break heap.
|
// about the biggest we can move around in Precursor and not break heap.
|
||||||
const MAX_BUFLEN: usize = 128 * 1024;
|
const MAX_BUFLEN: usize = 32 * 1024;
|
||||||
// compromise between memory zeroing time and latency to send a message
|
// compromise between memory zeroing time and latency to send a message
|
||||||
const DEFAULT_LIMIT: usize = 32 * 1024;
|
const DEFAULT_LIMIT: usize = 32 * 1024;
|
||||||
/*
|
/*
|
||||||
|
|
@ -898,7 +898,7 @@ impl Pddb {
|
||||||
log::error!("local key index did not match remote key index: {}, {}", check_key_index, header.starting_key_index);
|
log::error!("local key index did not match remote key index: {}, {}", check_key_index, header.starting_key_index);
|
||||||
}
|
}
|
||||||
let mut key_count = 0;
|
let mut key_count = 0;
|
||||||
// log::info!("header: {:?}", header);
|
log::info!("header: {:?}", header);
|
||||||
while key_count < header.len {
|
while key_count < header.len {
|
||||||
if index + size_of::<u32>() * 2 > msg_mem.len() {
|
if index + size_of::<u32>() * 2 > msg_mem.len() {
|
||||||
// quit if we don't have enough space to decode at least another two indices
|
// quit if we don't have enough space to decode at least another two indices
|
||||||
|
|
@ -908,7 +908,7 @@ impl Pddb {
|
||||||
index += size_of::<u32>();
|
index += size_of::<u32>();
|
||||||
let pos = u32::from_le_bytes(msg_mem.as_slice()[index..index + size_of::<u32>()].try_into().unwrap());
|
let pos = u32::from_le_bytes(msg_mem.as_slice()[index..index + size_of::<u32>()].try_into().unwrap());
|
||||||
index += size_of::<u32>();
|
index += size_of::<u32>();
|
||||||
// log::info!("unpacking message at {}({})", size, pos);
|
log::info!("unpacking message at {}({})", size, pos);
|
||||||
if size != 0 && pos != 0 {
|
if size != 0 && pos != 0 {
|
||||||
let archived = unsafe {
|
let archived = unsafe {
|
||||||
archived_value::<PddbKeyRecord>(&msg_mem.as_slice()[index..index + (size as usize)], pos as usize)
|
archived_value::<PddbKeyRecord>(&msg_mem.as_slice()[index..index + (size as usize)], pos as usize)
|
||||||
|
|
@ -939,7 +939,7 @@ impl Pddb {
|
||||||
if check_total_keys != check_key_index {
|
if check_total_keys != check_key_index {
|
||||||
log::error!("Number of keys read does not match expected value: {}, {}", check_total_keys, check_key_index);
|
log::error!("Number of keys read does not match expected value: {}, {}", check_total_keys, check_key_index);
|
||||||
}
|
}
|
||||||
|
xous::unmap_memory(msg_mem).unwrap();
|
||||||
Ok(ret)
|
Ok(ret)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -407,6 +407,12 @@ use rkyv::{
|
||||||
use core::mem::size_of;
|
use core::mem::size_of;
|
||||||
use core::ops::Deref;
|
use core::ops::Deref;
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#[cfg(feature="perfcounter")]
|
#[cfg(feature="perfcounter")]
|
||||||
const FILE_ID_SERVICES_PDDB_SRC_MAIN: u32 = 0;
|
const FILE_ID_SERVICES_PDDB_SRC_MAIN: u32 = 0;
|
||||||
#[cfg(feature="perfcounter")]
|
#[cfg(feature="perfcounter")]
|
||||||
|
|
@ -1405,13 +1411,16 @@ fn wrapped_main() -> ! {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Optimized bulk data read handler. See lib.rs for documentation.
|
// Optimized bulk data read handler. See lib.rs for documentation.
|
||||||
// TO INVESTIGATE: probably serialization can be made a lot more efficient by
|
// Note: it would be tempting to use a `BufferSerializer` instead of a WriteSeriaizer.
|
||||||
// using a `BufferSerializer` instead of a WriteSeriaizer. I think we can also
|
// This would prevent the extra allocation and copy into the buffer. The problem is that
|
||||||
// stack multiple records into the buffer without having to re-allocate the serializer,
|
// the BufferSerializer can return you the `pos` of the archive, but it won't tell you
|
||||||
// but it's not clear to me how the deserialization would work in that case. The main
|
// the length of the data that's been archived. Thus, when you call `into_inner()` you get
|
||||||
// problem of using the BufferSerializer is the interior mutability problem of re-using the
|
// back the original buffer (which is great -- we could split that and pass it on to future
|
||||||
// allocated slice across multiple iterations of the loop, especially with the property where
|
// iterations) but we don't know how much data was put in there. Maybe later versions of rkyv
|
||||||
// if the record doesn't fit we want to stash it elsewhere for use on the next iteration.
|
// have a call for that, but the current version we are using doesn't seem to be written to
|
||||||
|
// allow for multiple rkyv objects to be packed into a single packet. I think the assumption
|
||||||
|
// is rather that you're writing rkyv objects into a single file and the files are then packed
|
||||||
|
// into disk by the filesystem.
|
||||||
Opcode::DictBulkRead => {
|
Opcode::DictBulkRead => {
|
||||||
const BULKREAD_TIMEOUT_MS: u64 = 5000;
|
const BULKREAD_TIMEOUT_MS: u64 = 5000;
|
||||||
let range = msg.body.memory_message_mut().unwrap();
|
let range = msg.body.memory_message_mut().unwrap();
|
||||||
|
|
@ -1475,6 +1484,7 @@ fn wrapped_main() -> ! {
|
||||||
};
|
};
|
||||||
bulkread_state = Some(state);
|
bulkread_state = Some(state);
|
||||||
}
|
}
|
||||||
|
let mut finished = false;
|
||||||
// handle data packing into the structure
|
// handle data packing into the structure
|
||||||
if let Some(state) = &mut bulkread_state {
|
if let Some(state) = &mut bulkread_state {
|
||||||
// check that the token matches, if this isn't a first call to the function
|
// check that the token matches, if this isn't a first call to the function
|
||||||
|
|
@ -1491,8 +1501,7 @@ fn wrapped_main() -> ! {
|
||||||
header.token = state.token;
|
header.token = state.token;
|
||||||
|
|
||||||
// now loop through and pack data into the slice
|
// now loop through and pack data into the slice
|
||||||
let (header_buf, buf) = buf.split_at_mut(size_of::<BulkReadHeader>());
|
let (header_buf, mut buf) = buf.split_at_mut(size_of::<BulkReadHeader>());
|
||||||
let mut finished = false;
|
|
||||||
enum SerializeResult<'a> {
|
enum SerializeResult<'a> {
|
||||||
Success(usize, &'a mut [u8], String, &'a mut [u8]),
|
Success(usize, &'a mut [u8], String, &'a mut [u8]),
|
||||||
Failure(String)
|
Failure(String)
|
||||||
|
|
@ -1549,7 +1558,7 @@ fn wrapped_main() -> ! {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// log::info!("hit size limit: limit {} total {}", state.read_limit, state.read_total);
|
log::info!("hit size limit: limit {} total {}", state.read_limit, state.read_total);
|
||||||
// report the key, but with no data
|
// report the key, but with no data
|
||||||
let rec = PddbKeyRecord {
|
let rec = PddbKeyRecord {
|
||||||
name: key_name.to_string(),
|
name: key_name.to_string(),
|
||||||
|
|
@ -1578,20 +1587,23 @@ fn wrapped_main() -> ! {
|
||||||
// pointer to, to stack the next archived result. The other option would be to use the .write()
|
// pointer to, to stack the next archived result. The other option would be to use the .write()
|
||||||
// method but then we'd need to...make a serializer for the structure. which is agains the whole point of it...
|
// method but then we'd need to...make a serializer for the structure. which is agains the whole point of it...
|
||||||
match ser_result {
|
match ser_result {
|
||||||
SerializeResult::Success(pos, buf, key_name, pre_buf) => {
|
SerializeResult::Success(pos, sbuf, _key_name, pre_buf) => {
|
||||||
log::info!("packing message of {}({})", buf.len(), pos);
|
log::info!("packing message of {}({})", sbuf.len(), pos);
|
||||||
// data can fit, copy it into the buffer.
|
// data can fit, copy it into the buffer.
|
||||||
state.buf_starting_key_index += 1;
|
state.buf_starting_key_index += 1;
|
||||||
header.len += 1;
|
header.len += 1;
|
||||||
// read length increment was handled when the data was copied into the serialization buffer.
|
// read length increment was handled when the data was copied into the serialization buffer.
|
||||||
pre_buf[..4].copy_from_slice(
|
pre_buf[..4].copy_from_slice(
|
||||||
&(buf.len() as u32).to_le_bytes()
|
//&(sbuf.len() as u32).to_le_bytes()
|
||||||
|
&(pos as u32).to_le_bytes()
|
||||||
);
|
);
|
||||||
pre_buf[4..8].copy_from_slice(
|
pre_buf[4..8].copy_from_slice(
|
||||||
&(pos as u32).to_le_bytes()
|
&(pos as u32).to_le_bytes()
|
||||||
);
|
);
|
||||||
|
(_, buf) = sbuf.split_at_mut(pos);
|
||||||
}
|
}
|
||||||
SerializeResult::Failure(key_name) => {
|
SerializeResult::Failure(key_name) => {
|
||||||
|
log::info!("ran out of space filling buffer, pushing {} back into the queue", key_name);
|
||||||
// data didn't fit, quit with finished = false;
|
// data didn't fit, quit with finished = false;
|
||||||
state.key_list.push(key_name);
|
state.key_list.push(key_name);
|
||||||
break;
|
break;
|
||||||
|
|
@ -1611,6 +1623,9 @@ fn wrapped_main() -> ! {
|
||||||
} else {
|
} else {
|
||||||
log::warn!("This should be unreachable, the state should always be initialized by this point");
|
log::warn!("This should be unreachable, the state should always be initialized by this point");
|
||||||
}
|
}
|
||||||
|
if finished {
|
||||||
|
bulkread_state = None;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Opcode::SeekKeyStd => {
|
Opcode::SeekKeyStd => {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue