clean up/optimize the key listing routine

provides some marginal improvement on the vault UX
This commit is contained in:
bunnie 2022-08-01 23:12:26 +08:00
parent 513fe519a7
commit 13b366345c
5 changed files with 99 additions and 59 deletions

View file

@ -125,7 +125,7 @@ pub(crate) enum Opcode {
// routines to list available resources // routines to list available resources
KeyCountInDict = 11, KeyCountInDict = 11,
GetKeyNameAtIndex = 12, // GetKeyNameAtIndex = 12, // superceded by ListKeyV2
DictCountInBasis = 13, DictCountInBasis = 13,
GetDictNameAtIndex = 14, GetDictNameAtIndex = 14,
@ -207,6 +207,9 @@ pub(crate) enum Opcode {
/// Flush the SpaceUpdate log, restoring deniability /// Flush the SpaceUpdate log, restoring deniability
FlushSpaceUpdate = 44, FlushSpaceUpdate = 44,
/// Optimized key listing
ListKeyV2 = 45,
/// This key type could not be decoded /// This key type could not be decoded
InvalidOpcode = u32::MAX as _, InvalidOpcode = u32::MAX as _,
} }
@ -289,9 +292,19 @@ pub struct PddbKeyRequest {
pub result: PddbRequestCode, pub result: PddbRequestCode,
} }
pub(crate) const MAX_PDDBKLISTLEN: usize = 4064;
/// A structure for requesting a token to access a particular key/value pair
#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
pub (crate) struct PddbKeyList {
pub token: [u32; 4],
pub data: [u8; MAX_PDDBKLISTLEN],
pub retcode: PddbRetcode,
pub end: bool,
}
/// Return codes for Read/Write API calls to the main server /// Return codes for Read/Write API calls to the main server
#[repr(u8)] #[repr(u8)]
#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)] #[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
pub(crate) enum PddbRetcode { pub(crate) enum PddbRetcode {
Uninit = 0, Uninit = 0,
Ok = 1, Ok = 1,

View file

@ -143,7 +143,7 @@ use aes_gcm_siv::{
use aes::Aes256; use aes::Aes256;
use aes::cipher::generic_array::GenericArray; use aes::cipher::generic_array::GenericArray;
use std::iter::IntoIterator; use std::iter::IntoIterator;
use std::collections::{BinaryHeap, HashMap, HashSet}; use std::collections::{BinaryHeap, HashMap, HashSet, BTreeSet};
use std::io::{Result, Error, ErrorKind}; use std::io::{Result, Error, ErrorKind};
use std::cmp::Reverse; use std::cmp::Reverse;
use std::cmp::Ordering; use std::cmp::Ordering;
@ -381,8 +381,8 @@ impl BasisCache {
} }
dict_set dict_set
} }
pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result<HashSet::<String>> { pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result<BTreeSet::<String>> {
let mut merge_list = HashSet::<String>::new(); let mut merge_list = BTreeSet::<String>::new();
let mut found_dict = false; let mut found_dict = false;
if basis_name.is_some() { if basis_name.is_some() {
if let Some(basis_index) = self.select_basis(basis_name) { if let Some(basis_index) = self.select_basis(basis_name) {

View file

@ -5,7 +5,7 @@ use std::num::NonZeroU32;
use core::ops::{Deref, DerefMut}; use core::ops::{Deref, DerefMut};
use core::mem::size_of; use core::mem::size_of;
use aes_gcm_siv::Aes256GcmSiv; use aes_gcm_siv::Aes256GcmSiv;
use std::collections::{HashMap, BinaryHeap, HashSet}; use std::collections::{HashMap, BinaryHeap, BTreeSet};
use std::io::{Result, Error, ErrorKind}; use std::io::{Result, Error, ErrorKind};
use bitfield::bitfield; use bitfield::bitfield;
use std::cmp::{Ordering, Reverse}; use std::cmp::{Ordering, Reverse};
@ -196,7 +196,7 @@ impl DictCacheEntry {
} }
/// merges the list of keys in this dict cache entry into a merge_list. /// merges the list of keys in this dict cache entry into a merge_list.
/// The `merge_list` is used because keys are presented as a union across all open basis. /// The `merge_list` is used because keys are presented as a union across all open basis.
pub(crate) fn key_list(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv, merge_list: &mut HashSet<String>) { pub(crate) fn key_list(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv, merge_list: &mut BTreeSet<String>) {
// ensure that the key cache is filled // ensure that the key cache is filled
if self.keys.len() < self.key_count as usize { if self.keys.len() < self.key_count as usize {
self.fill(hw, v2p_map, cipher); self.fill(hw, v2p_map, cipher);

View file

@ -524,34 +524,56 @@ impl Pddb {
.or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
let response = buf.to_original::<PddbDictRequest, _>().unwrap(); let response = buf.to_original::<PddbDictRequest, _>().unwrap();
let count = match response.code { match response.code {
PddbRequestCode::NoErr => response.index, PddbRequestCode::NoErr => (),
PddbRequestCode::NotFound => return Err(Error::new(ErrorKind::NotFound, "dictionary not found")), PddbRequestCode::NotFound => return Err(Error::new(ErrorKind::NotFound, "dictionary not found")),
_ => return Err(Error::new(ErrorKind::Other, "Internal error")), _ => return Err(Error::new(ErrorKind::Other, "Internal error")),
}; };
// very non-optimal, slow way of doing this, but let's just get it working first and optimize later. // v2 key listing packs the key list into a larger [u8] field that should cut down on the number of messages
// it's absolutely important that you access every entry, and the highest index last, because // required to list a large dictionary by about a factor of 50.
// that is how the server knows you've finished with the list-out.
let mut key_list = Vec::<String>::new(); let mut key_list = Vec::<String>::new();
for index in 0..count { // just make sure we didn't screw up the sizing of this record
let request = PddbDictRequest { assert!(core::mem::size_of::<PddbKeyList>() < 4096);
basis_specified: basis_name.is_some(), loop {
basis: xous_ipc::String::<BASIS_NAME_LEN>::from_str(&bname), let request = PddbKeyList {
dict: xous_ipc::String::<DICT_NAME_LEN>::from_str(dict_name),
key: xous_ipc::String::<KEY_NAME_LEN>::new(),
index,
code: PddbRequestCode::Uninit,
token, token,
end: false,
retcode: PddbRetcode::Uninit,
data: [0u8; MAX_PDDBKLISTLEN]
}; };
let mut buf = Buffer::into_buf(request) let mut buf = Buffer::into_buf(request)
.or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
buf.lend_mut(self.conn, Opcode::GetKeyNameAtIndex.to_u32().unwrap()) buf.lend_mut(self.conn, Opcode::ListKeyV2.to_u32().unwrap())
.or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
let response = buf.to_original::<PddbDictRequest, _>().unwrap(); let response = buf.as_flat::<PddbKeyList, _>().unwrap();
match response.code { match response.retcode {
PddbRequestCode::NoErr => key_list.push(String::from(response.key.as_str().expect("utf-8 parse error in key name"))), ArchivedPddbRetcode::Ok => (),
_ => return Err(Error::new(ErrorKind::Other, "Internal error")), ArchivedPddbRetcode::AccessDenied => return Err(Error::new(ErrorKind::PermissionDenied, "KeyList facility locked by another process, try again later")),
_ => return Err(Error::new(ErrorKind::Other, "Internal Error")),
} }
// the [u8] data is structured as a packed list of u8-len + u8 data slice. The max length of
// a PDDB key name is guaranteed to be shorter than a u8. If the length field is 0, then this
// particular response has no more data in it to read.
let mut index = 0;
while response.data[index] != 0 && index < MAX_PDDBKLISTLEN {
let strlen = response.data[index] as usize;
index += 1;
if strlen + index >= MAX_PDDBKLISTLEN {
log::error!("Logic error in key list, index would be out of bounds. Aborting");
break;
}
if strlen == 0 { // case of an empty dictionary
break;
}
let key = String::from(std::str::from_utf8(&response.data[index..index+strlen]).unwrap_or("UTF8 error"));
log::trace!("Returned {}", key);
key_list.push(key);
index += strlen;
}
if response.end {
break;
}
} }
Ok(key_list) Ok(key_list)
} }

View file

@ -390,7 +390,7 @@ use xous_ipc::Buffer;
use core::cell::RefCell; use core::cell::RefCell;
use std::rc::Rc; use std::rc::Rc;
use std::thread; use std::thread;
use std::collections::HashMap; use std::collections::{HashMap, BTreeSet};
use std::io::ErrorKind; use std::io::ErrorKind;
use core::fmt::Write; use core::fmt::Write;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
@ -559,7 +559,7 @@ fn wrapped_main() -> ! {
} }
}); });
// main server loop // main server loop
let mut key_list = Vec::<String>::new(); // storage for key lists let mut key_list: Option::<BTreeSet::<String>> = None; // storage for key lists
let mut key_token: Option<[u32; 4]> = None; let mut key_token: Option<[u32; 4]> = None;
let mut dict_list = Vec::<String>::new(); // storage for dict lists let mut dict_list = Vec::<String>::new(); // storage for dict lists
let mut dict_token: Option<[u32; 4]> = None; let mut dict_token: Option<[u32; 4]> = None;
@ -1158,7 +1158,6 @@ fn wrapped_main() -> ! {
continue; continue;
} }
key_token = Some(req.token); key_token = Some(req.token);
key_list.clear();
let bname = if req.basis_specified { let bname = if req.basis_specified {
Some(req.basis.as_str().unwrap()) Some(req.basis.as_str().unwrap())
} else { } else {
@ -1166,58 +1165,64 @@ fn wrapped_main() -> ! {
}; };
let dict = req.dict.as_str().expect("dict utf-8 decode error"); let dict = req.dict.as_str().expect("dict utf-8 decode error");
log::debug!("counting keys in dict {} basis {:?}", dict, bname); log::debug!("counting keys in dict {} basis {:?}", dict, bname);
match basis_cache.key_list(&mut pddb_os, dict, bname) { key_list = match basis_cache.key_list(&mut pddb_os, dict, bname) {
Ok(list) => { Ok(list) => {
log::debug!("count: {}", list.len()); log::debug!("count: {}", list.len());
if list.len() > 0 {
req.index = list.len() as u32;
for key in list {
log::debug!("key list: {}", key);
key_list.push(key);
}
} else {
log::debug!("count is 0, resetting state");
// no keys to list, so reset the state
key_token = None;
key_list.clear();
}
req.code = PddbRequestCode::NoErr; req.code = PddbRequestCode::NoErr;
Some(list)
} }
Err(e) => { Err(e) => {
key_token = None;
key_list.clear();
match e.kind() { match e.kind() {
std::io::ErrorKind::NotFound => req.code = PddbRequestCode::NotFound, std::io::ErrorKind::NotFound => req.code = PddbRequestCode::NotFound,
_ => req.code = PddbRequestCode::InternalError, _ => req.code = PddbRequestCode::InternalError,
} }
None
} }
} };
buffer.replace(req).unwrap(); buffer.replace(req).unwrap();
} }
Opcode::GetKeyNameAtIndex => { Opcode::ListKeyV2 => {
let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) }; let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
let mut req = buffer.to_original::<PddbDictRequest, _>().unwrap(); let mut req = buffer.to_original::<PddbKeyList, _>().unwrap();
if let Some(token) = key_token { if let Some(token) = key_token {
if req.token != token { if req.token != token {
req.code = PddbRequestCode::AccessDenied; req.retcode = PddbRetcode::AccessDenied;
} else { } else {
if req.index >= key_list.len() as u32 { if let Some(klist) = &mut key_list {
req.code = PddbRequestCode::InternalError; let mut index = 0;
} else { loop {
req.key = xous_ipc::String::<KEY_NAME_LEN>::from_str(&key_list[req.index as usize]); let keyname =
req.code = PddbRequestCode::NoErr; if let Some(keyname) = klist.iter().next() {
log::debug!("fetching key at index {}: {}", req.index, req.key); keyname.to_string()
// the last index requested must be the highest one! } else {
if req.index == key_list.len() as u32 - 1 { key_token = None;
log::debug!("last key, resetting state"); req.end = true;
key_token = None; req.retcode = PddbRetcode::Ok;
key_list.clear(); break;
};
if keyname.len() + 1 + index < MAX_PDDBKLISTLEN {
assert!(keyname.len() < u8::MAX as usize); // this should always be true due to other limits in the PDDB
req.data[index] = keyname.len() as u8;
index += 1;
req.data[index..index + keyname.len()].copy_from_slice(keyname.as_bytes());
index += keyname.len();
klist.remove(&keyname);
} else {
// don't remove the item, and indicate there is more to come
req.end = false;
req.retcode = PddbRetcode::Ok;
break;
}
} }
} else {
key_token = None;
req.end = true;
req.retcode = PddbRetcode::Ok;
} }
} }
} else { } else {
log::debug!("multiple concurrent requests detected, returning error"); log::debug!("multiple concurrent requests detected, returning error");
req.code = PddbRequestCode::AccessDenied; req.retcode = PddbRetcode::AccessDenied;
} }
buffer.replace(req).unwrap(); buffer.replace(req).unwrap();
} }