diff --git a/services/pddb/src/api.rs b/services/pddb/src/api.rs index 752b4d152..fbb211581 100644 --- a/services/pddb/src/api.rs +++ b/services/pddb/src/api.rs @@ -125,7 +125,7 @@ pub(crate) enum Opcode { // routines to list available resources KeyCountInDict = 11, - GetKeyNameAtIndex = 12, + // GetKeyNameAtIndex = 12, // superceded by ListKeyV2 DictCountInBasis = 13, GetDictNameAtIndex = 14, @@ -207,6 +207,9 @@ pub(crate) enum Opcode { /// Flush the SpaceUpdate log, restoring deniability FlushSpaceUpdate = 44, + /// Optimized key listing + ListKeyV2 = 45, + /// This key type could not be decoded InvalidOpcode = u32::MAX as _, } @@ -289,9 +292,19 @@ pub struct PddbKeyRequest { pub result: PddbRequestCode, } +pub(crate) const MAX_PDDBKLISTLEN: usize = 4064; +/// A structure for requesting a token to access a particular key/value pair +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub (crate) struct PddbKeyList { + pub token: [u32; 4], + pub data: [u8; MAX_PDDBKLISTLEN], + pub retcode: PddbRetcode, + pub end: bool, +} + /// Return codes for Read/Write API calls to the main server #[repr(u8)] -#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)] +#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub(crate) enum PddbRetcode { Uninit = 0, Ok = 1, diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index 747296bc6..ca14c9c5a 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -143,7 +143,7 @@ use aes_gcm_siv::{ use aes::Aes256; use aes::cipher::generic_array::GenericArray; use std::iter::IntoIterator; -use std::collections::{BinaryHeap, HashMap, HashSet}; +use std::collections::{BinaryHeap, HashMap, HashSet, BTreeSet}; use std::io::{Result, Error, ErrorKind}; use std::cmp::Reverse; use std::cmp::Ordering; @@ -381,8 +381,8 @@ impl BasisCache { } dict_set } - pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result> { - let mut merge_list = HashSet::::new(); + pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result> { + let mut merge_list = BTreeSet::::new(); let mut found_dict = false; if basis_name.is_some() { if let Some(basis_index) = self.select_basis(basis_name) { diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 0858d2c46..daf704aa5 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -5,7 +5,7 @@ use std::num::NonZeroU32; use core::ops::{Deref, DerefMut}; use core::mem::size_of; use aes_gcm_siv::Aes256GcmSiv; -use std::collections::{HashMap, BinaryHeap, HashSet}; +use std::collections::{HashMap, BinaryHeap, BTreeSet}; use std::io::{Result, Error, ErrorKind}; use bitfield::bitfield; use std::cmp::{Ordering, Reverse}; @@ -196,7 +196,7 @@ impl DictCacheEntry { } /// merges the list of keys in this dict cache entry into a merge_list. /// The `merge_list` is used because keys are presented as a union across all open basis. - pub(crate) fn key_list(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv, merge_list: &mut HashSet) { + pub(crate) fn key_list(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv, merge_list: &mut BTreeSet) { // ensure that the key cache is filled if self.keys.len() < self.key_count as usize { self.fill(hw, v2p_map, cipher); diff --git a/services/pddb/src/lib.rs b/services/pddb/src/lib.rs index a8a56941f..e18c269f3 100644 --- a/services/pddb/src/lib.rs +++ b/services/pddb/src/lib.rs @@ -524,34 +524,56 @@ impl Pddb { .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; let response = buf.to_original::().unwrap(); - let count = match response.code { - PddbRequestCode::NoErr => response.index, + match response.code { + PddbRequestCode::NoErr => (), PddbRequestCode::NotFound => return Err(Error::new(ErrorKind::NotFound, "dictionary not found")), _ => return Err(Error::new(ErrorKind::Other, "Internal error")), }; - // very non-optimal, slow way of doing this, but let's just get it working first and optimize later. - // it's absolutely important that you access every entry, and the highest index last, because - // that is how the server knows you've finished with the list-out. + // v2 key listing packs the key list into a larger [u8] field that should cut down on the number of messages + // required to list a large dictionary by about a factor of 50. let mut key_list = Vec::::new(); - for index in 0..count { - let request = PddbDictRequest { - basis_specified: basis_name.is_some(), - basis: xous_ipc::String::::from_str(&bname), - dict: xous_ipc::String::::from_str(dict_name), - key: xous_ipc::String::::new(), - index, - code: PddbRequestCode::Uninit, + // just make sure we didn't screw up the sizing of this record + assert!(core::mem::size_of::() < 4096); + loop { + let request = PddbKeyList { token, + end: false, + retcode: PddbRetcode::Uninit, + data: [0u8; MAX_PDDBKLISTLEN] }; let mut buf = Buffer::into_buf(request) .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; - buf.lend_mut(self.conn, Opcode::GetKeyNameAtIndex.to_u32().unwrap()) + buf.lend_mut(self.conn, Opcode::ListKeyV2.to_u32().unwrap()) .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; - let response = buf.to_original::().unwrap(); - match response.code { - PddbRequestCode::NoErr => key_list.push(String::from(response.key.as_str().expect("utf-8 parse error in key name"))), - _ => return Err(Error::new(ErrorKind::Other, "Internal error")), + let response = buf.as_flat::().unwrap(); + match response.retcode { + ArchivedPddbRetcode::Ok => (), + ArchivedPddbRetcode::AccessDenied => return Err(Error::new(ErrorKind::PermissionDenied, "KeyList facility locked by another process, try again later")), + _ => return Err(Error::new(ErrorKind::Other, "Internal Error")), } + // the [u8] data is structured as a packed list of u8-len + u8 data slice. The max length of + // a PDDB key name is guaranteed to be shorter than a u8. If the length field is 0, then this + // particular response has no more data in it to read. + let mut index = 0; + while response.data[index] != 0 && index < MAX_PDDBKLISTLEN { + let strlen = response.data[index] as usize; + index += 1; + if strlen + index >= MAX_PDDBKLISTLEN { + log::error!("Logic error in key list, index would be out of bounds. Aborting"); + break; + } + if strlen == 0 { // case of an empty dictionary + break; + } + let key = String::from(std::str::from_utf8(&response.data[index..index+strlen]).unwrap_or("UTF8 error")); + log::trace!("Returned {}", key); + key_list.push(key); + index += strlen; + } + if response.end { + break; + } + } Ok(key_list) } diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 71cbfbeb9..49eebf9e0 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -390,7 +390,7 @@ use xous_ipc::Buffer; use core::cell::RefCell; use std::rc::Rc; use std::thread; -use std::collections::HashMap; +use std::collections::{HashMap, BTreeSet}; use std::io::ErrorKind; use core::fmt::Write; use std::sync::atomic::{AtomicBool, Ordering}; @@ -559,7 +559,7 @@ fn wrapped_main() -> ! { } }); // main server loop - let mut key_list = Vec::::new(); // storage for key lists + let mut key_list: Option::> = None; // storage for key lists let mut key_token: Option<[u32; 4]> = None; let mut dict_list = Vec::::new(); // storage for dict lists let mut dict_token: Option<[u32; 4]> = None; @@ -1158,7 +1158,6 @@ fn wrapped_main() -> ! { continue; } key_token = Some(req.token); - key_list.clear(); let bname = if req.basis_specified { Some(req.basis.as_str().unwrap()) } else { @@ -1166,58 +1165,64 @@ fn wrapped_main() -> ! { }; let dict = req.dict.as_str().expect("dict utf-8 decode error"); log::debug!("counting keys in dict {} basis {:?}", dict, bname); - match basis_cache.key_list(&mut pddb_os, dict, bname) { + key_list = match basis_cache.key_list(&mut pddb_os, dict, bname) { Ok(list) => { log::debug!("count: {}", list.len()); - if list.len() > 0 { - req.index = list.len() as u32; - for key in list { - log::debug!("key list: {}", key); - key_list.push(key); - } - } else { - log::debug!("count is 0, resetting state"); - // no keys to list, so reset the state - key_token = None; - key_list.clear(); - } req.code = PddbRequestCode::NoErr; + Some(list) } Err(e) => { - key_token = None; - key_list.clear(); match e.kind() { std::io::ErrorKind::NotFound => req.code = PddbRequestCode::NotFound, _ => req.code = PddbRequestCode::InternalError, } + None } - } + }; buffer.replace(req).unwrap(); } - Opcode::GetKeyNameAtIndex => { + Opcode::ListKeyV2 => { let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) }; - let mut req = buffer.to_original::().unwrap(); + let mut req = buffer.to_original::().unwrap(); if let Some(token) = key_token { if req.token != token { - req.code = PddbRequestCode::AccessDenied; + req.retcode = PddbRetcode::AccessDenied; } else { - if req.index >= key_list.len() as u32 { - req.code = PddbRequestCode::InternalError; - } else { - req.key = xous_ipc::String::::from_str(&key_list[req.index as usize]); - req.code = PddbRequestCode::NoErr; - log::debug!("fetching key at index {}: {}", req.index, req.key); - // the last index requested must be the highest one! - if req.index == key_list.len() as u32 - 1 { - log::debug!("last key, resetting state"); - key_token = None; - key_list.clear(); + if let Some(klist) = &mut key_list { + let mut index = 0; + loop { + let keyname = + if let Some(keyname) = klist.iter().next() { + keyname.to_string() + } else { + key_token = None; + req.end = true; + req.retcode = PddbRetcode::Ok; + break; + }; + if keyname.len() + 1 + index < MAX_PDDBKLISTLEN { + assert!(keyname.len() < u8::MAX as usize); // this should always be true due to other limits in the PDDB + req.data[index] = keyname.len() as u8; + index += 1; + req.data[index..index + keyname.len()].copy_from_slice(keyname.as_bytes()); + index += keyname.len(); + klist.remove(&keyname); + } else { + // don't remove the item, and indicate there is more to come + req.end = false; + req.retcode = PddbRetcode::Ok; + break; + } } + } else { + key_token = None; + req.end = true; + req.retcode = PddbRetcode::Ok; } } } else { log::debug!("multiple concurrent requests detected, returning error"); - req.code = PddbRequestCode::AccessDenied; + req.retcode = PddbRetcode::AccessDenied; } buffer.replace(req).unwrap(); }