clean up/optimize the key listing routine

provides some marginal improvement on the vault UX
This commit is contained in:
bunnie 2022-08-01 23:12:26 +08:00
parent 513fe519a7
commit 13b366345c
5 changed files with 99 additions and 59 deletions

View file

@ -125,7 +125,7 @@ pub(crate) enum Opcode {
// routines to list available resources
KeyCountInDict = 11,
GetKeyNameAtIndex = 12,
// GetKeyNameAtIndex = 12, // superceded by ListKeyV2
DictCountInBasis = 13,
GetDictNameAtIndex = 14,
@ -207,6 +207,9 @@ pub(crate) enum Opcode {
/// Flush the SpaceUpdate log, restoring deniability
FlushSpaceUpdate = 44,
/// Optimized key listing
ListKeyV2 = 45,
/// This key type could not be decoded
InvalidOpcode = u32::MAX as _,
}
@ -289,9 +292,19 @@ pub struct PddbKeyRequest {
pub result: PddbRequestCode,
}
pub(crate) const MAX_PDDBKLISTLEN: usize = 4064;
/// A structure for requesting a token to access a particular key/value pair
#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
pub (crate) struct PddbKeyList {
pub token: [u32; 4],
pub data: [u8; MAX_PDDBKLISTLEN],
pub retcode: PddbRetcode,
pub end: bool,
}
/// Return codes for Read/Write API calls to the main server
#[repr(u8)]
#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)]
#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
pub(crate) enum PddbRetcode {
Uninit = 0,
Ok = 1,

View file

@ -143,7 +143,7 @@ use aes_gcm_siv::{
use aes::Aes256;
use aes::cipher::generic_array::GenericArray;
use std::iter::IntoIterator;
use std::collections::{BinaryHeap, HashMap, HashSet};
use std::collections::{BinaryHeap, HashMap, HashSet, BTreeSet};
use std::io::{Result, Error, ErrorKind};
use std::cmp::Reverse;
use std::cmp::Ordering;
@ -381,8 +381,8 @@ impl BasisCache {
}
dict_set
}
pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result<HashSet::<String>> {
let mut merge_list = HashSet::<String>::new();
pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result<BTreeSet::<String>> {
let mut merge_list = BTreeSet::<String>::new();
let mut found_dict = false;
if basis_name.is_some() {
if let Some(basis_index) = self.select_basis(basis_name) {

View file

@ -5,7 +5,7 @@ use std::num::NonZeroU32;
use core::ops::{Deref, DerefMut};
use core::mem::size_of;
use aes_gcm_siv::Aes256GcmSiv;
use std::collections::{HashMap, BinaryHeap, HashSet};
use std::collections::{HashMap, BinaryHeap, BTreeSet};
use std::io::{Result, Error, ErrorKind};
use bitfield::bitfield;
use std::cmp::{Ordering, Reverse};
@ -196,7 +196,7 @@ impl DictCacheEntry {
}
/// merges the list of keys in this dict cache entry into a merge_list.
/// The `merge_list` is used because keys are presented as a union across all open basis.
pub(crate) fn key_list(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv, merge_list: &mut HashSet<String>) {
pub(crate) fn key_list(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv, merge_list: &mut BTreeSet<String>) {
// ensure that the key cache is filled
if self.keys.len() < self.key_count as usize {
self.fill(hw, v2p_map, cipher);

View file

@ -524,34 +524,56 @@ impl Pddb {
.or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
let response = buf.to_original::<PddbDictRequest, _>().unwrap();
let count = match response.code {
PddbRequestCode::NoErr => response.index,
match response.code {
PddbRequestCode::NoErr => (),
PddbRequestCode::NotFound => return Err(Error::new(ErrorKind::NotFound, "dictionary not found")),
_ => return Err(Error::new(ErrorKind::Other, "Internal error")),
};
// very non-optimal, slow way of doing this, but let's just get it working first and optimize later.
// it's absolutely important that you access every entry, and the highest index last, because
// that is how the server knows you've finished with the list-out.
// v2 key listing packs the key list into a larger [u8] field that should cut down on the number of messages
// required to list a large dictionary by about a factor of 50.
let mut key_list = Vec::<String>::new();
for index in 0..count {
let request = PddbDictRequest {
basis_specified: basis_name.is_some(),
basis: xous_ipc::String::<BASIS_NAME_LEN>::from_str(&bname),
dict: xous_ipc::String::<DICT_NAME_LEN>::from_str(dict_name),
key: xous_ipc::String::<KEY_NAME_LEN>::new(),
index,
code: PddbRequestCode::Uninit,
// just make sure we didn't screw up the sizing of this record
assert!(core::mem::size_of::<PddbKeyList>() < 4096);
loop {
let request = PddbKeyList {
token,
end: false,
retcode: PddbRetcode::Uninit,
data: [0u8; MAX_PDDBKLISTLEN]
};
let mut buf = Buffer::into_buf(request)
.or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
buf.lend_mut(self.conn, Opcode::GetKeyNameAtIndex.to_u32().unwrap())
buf.lend_mut(self.conn, Opcode::ListKeyV2.to_u32().unwrap())
.or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
let response = buf.to_original::<PddbDictRequest, _>().unwrap();
match response.code {
PddbRequestCode::NoErr => key_list.push(String::from(response.key.as_str().expect("utf-8 parse error in key name"))),
_ => return Err(Error::new(ErrorKind::Other, "Internal error")),
let response = buf.as_flat::<PddbKeyList, _>().unwrap();
match response.retcode {
ArchivedPddbRetcode::Ok => (),
ArchivedPddbRetcode::AccessDenied => return Err(Error::new(ErrorKind::PermissionDenied, "KeyList facility locked by another process, try again later")),
_ => return Err(Error::new(ErrorKind::Other, "Internal Error")),
}
// the [u8] data is structured as a packed list of u8-len + u8 data slice. The max length of
// a PDDB key name is guaranteed to be shorter than a u8. If the length field is 0, then this
// particular response has no more data in it to read.
let mut index = 0;
while response.data[index] != 0 && index < MAX_PDDBKLISTLEN {
let strlen = response.data[index] as usize;
index += 1;
if strlen + index >= MAX_PDDBKLISTLEN {
log::error!("Logic error in key list, index would be out of bounds. Aborting");
break;
}
if strlen == 0 { // case of an empty dictionary
break;
}
let key = String::from(std::str::from_utf8(&response.data[index..index+strlen]).unwrap_or("UTF8 error"));
log::trace!("Returned {}", key);
key_list.push(key);
index += strlen;
}
if response.end {
break;
}
}
Ok(key_list)
}

View file

@ -390,7 +390,7 @@ use xous_ipc::Buffer;
use core::cell::RefCell;
use std::rc::Rc;
use std::thread;
use std::collections::HashMap;
use std::collections::{HashMap, BTreeSet};
use std::io::ErrorKind;
use core::fmt::Write;
use std::sync::atomic::{AtomicBool, Ordering};
@ -559,7 +559,7 @@ fn wrapped_main() -> ! {
}
});
// main server loop
let mut key_list = Vec::<String>::new(); // storage for key lists
let mut key_list: Option::<BTreeSet::<String>> = None; // storage for key lists
let mut key_token: Option<[u32; 4]> = None;
let mut dict_list = Vec::<String>::new(); // storage for dict lists
let mut dict_token: Option<[u32; 4]> = None;
@ -1158,7 +1158,6 @@ fn wrapped_main() -> ! {
continue;
}
key_token = Some(req.token);
key_list.clear();
let bname = if req.basis_specified {
Some(req.basis.as_str().unwrap())
} else {
@ -1166,58 +1165,64 @@ fn wrapped_main() -> ! {
};
let dict = req.dict.as_str().expect("dict utf-8 decode error");
log::debug!("counting keys in dict {} basis {:?}", dict, bname);
match basis_cache.key_list(&mut pddb_os, dict, bname) {
key_list = match basis_cache.key_list(&mut pddb_os, dict, bname) {
Ok(list) => {
log::debug!("count: {}", list.len());
if list.len() > 0 {
req.index = list.len() as u32;
for key in list {
log::debug!("key list: {}", key);
key_list.push(key);
}
} else {
log::debug!("count is 0, resetting state");
// no keys to list, so reset the state
key_token = None;
key_list.clear();
}
req.code = PddbRequestCode::NoErr;
Some(list)
}
Err(e) => {
key_token = None;
key_list.clear();
match e.kind() {
std::io::ErrorKind::NotFound => req.code = PddbRequestCode::NotFound,
_ => req.code = PddbRequestCode::InternalError,
}
None
}
}
};
buffer.replace(req).unwrap();
}
Opcode::GetKeyNameAtIndex => {
Opcode::ListKeyV2 => {
let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
let mut req = buffer.to_original::<PddbDictRequest, _>().unwrap();
let mut req = buffer.to_original::<PddbKeyList, _>().unwrap();
if let Some(token) = key_token {
if req.token != token {
req.code = PddbRequestCode::AccessDenied;
req.retcode = PddbRetcode::AccessDenied;
} else {
if req.index >= key_list.len() as u32 {
req.code = PddbRequestCode::InternalError;
} else {
req.key = xous_ipc::String::<KEY_NAME_LEN>::from_str(&key_list[req.index as usize]);
req.code = PddbRequestCode::NoErr;
log::debug!("fetching key at index {}: {}", req.index, req.key);
// the last index requested must be the highest one!
if req.index == key_list.len() as u32 - 1 {
log::debug!("last key, resetting state");
key_token = None;
key_list.clear();
if let Some(klist) = &mut key_list {
let mut index = 0;
loop {
let keyname =
if let Some(keyname) = klist.iter().next() {
keyname.to_string()
} else {
key_token = None;
req.end = true;
req.retcode = PddbRetcode::Ok;
break;
};
if keyname.len() + 1 + index < MAX_PDDBKLISTLEN {
assert!(keyname.len() < u8::MAX as usize); // this should always be true due to other limits in the PDDB
req.data[index] = keyname.len() as u8;
index += 1;
req.data[index..index + keyname.len()].copy_from_slice(keyname.as_bytes());
index += keyname.len();
klist.remove(&keyname);
} else {
// don't remove the item, and indicate there is more to come
req.end = false;
req.retcode = PddbRetcode::Ok;
break;
}
}
} else {
key_token = None;
req.end = true;
req.retcode = PddbRetcode::Ok;
}
}
} else {
log::debug!("multiple concurrent requests detected, returning error");
req.code = PddbRequestCode::AccessDenied;
req.retcode = PddbRetcode::AccessDenied;
}
buffer.replace(req).unwrap();
}