From 2a0135c781f7da23d3a0d2d33d968ea6dc3b79da Mon Sep 17 00:00:00 2001 From: bunnie Date: Fri, 27 Jan 2023 01:43:24 +0800 Subject: [PATCH] add bulk deletion of keys from a single dictionary --- services/pddb/src/api.rs | 16 +++++++++- services/pddb/src/backend/basis.rs | 37 ++++++++++++++++++++++ services/pddb/src/lib.rs | 50 ++++++++++++++++++++++++++++++ services/pddb/src/main.rs | 42 +++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 1 deletion(-) diff --git a/services/pddb/src/api.rs b/services/pddb/src/api.rs index df88f3b9b..282103ae4 100644 --- a/services/pddb/src/api.rs +++ b/services/pddb/src/api.rs @@ -242,6 +242,9 @@ pub(crate) enum Opcode { /// Basis monitoring - reports when the basis order has changed BasisMonitor = 54, + /// Bulk delete of keys within a dictionary; if non-existent keys are specified, no error is returned. + DictBulkDelete = 55, + /// This key type could not be decoded InvalidOpcode = u32::MAX as _, } @@ -332,9 +335,20 @@ pub (crate) struct PddbKeyList { pub end: bool, } +/// A structure for bulk deletion of keys +pub(crate) const MAX_PDDB_DELETE_LEN: usize = 3800; // approximate limit, might be a little higher if we cared to calculate it out +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub (crate) struct PddbDeleteList { + pub basis_specified: bool, + pub basis: xous_ipc::String::, + pub dict: xous_ipc::String::, + pub data: [u8; MAX_PDDB_DELETE_LEN], + pub retcode: PddbRetcode, +} + /// Return codes for Read/Write API calls to the main server #[repr(u8)] -#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Eq, PartialEq)] pub(crate) enum PddbRetcode { Uninit = 0, Ok = 1, diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index d112ed5c1..a157505de 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -597,6 +597,43 @@ impl BasisCache { } } + pub(crate) fn key_list_remove(&mut self, + hw: &mut PddbOs, dict: &str, key_list: Vec::, basis_name: Option<&str> + ) -> Result<()> { + if let Some(basis_index) = self.select_basis(basis_name) { + let basis = &mut self.cache[basis_index]; + if !basis.ensure_dict_in_cache(hw, dict) { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); + } + if let Some(dict_entry) = basis.dicts.get_mut(dict) { + basis.age = basis.age.saturating_add(1); + basis.clean = false; + for key in key_list { + if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, &key) { + dict_entry.key_remove(hw, &mut basis.v2p_map, &basis.cipher, &key, false); + assert!(dict_entry.clean == false, "dictionary entry should have been marked unclean"); + } + } + // sync the key pools to disk + if !dict_entry.sync_small_pool(hw, &mut basis.v2p_map, &basis.cipher) { + return Err(Error::new(ErrorKind::OutOfMemory, "Ran out of memory syncing small pool")); + } + dict_entry.sync_large_pool(); + // encrypt and write the dict entry to disk + basis.dict_sync(hw, dict)?; + // sync the root basis structure as well, while we're at it... + basis.basis_sync(hw); + // finally, sync the page tables. + basis.pt_sync(hw); + Ok(()) + } else { + Err(Error::new(ErrorKind::NotFound, "dictionary not found")) + } + } else { + Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) + } + } + /// Updates a key in a dictionary; if it doesn't exist, creates it. User can specify a basis, /// or rely upon the auto-basis select algorithm. pub(crate) fn key_update(&mut self, diff --git a/services/pddb/src/lib.rs b/services/pddb/src/lib.rs index 227de45d3..fd0fdadab 100644 --- a/services/pddb/src/lib.rs +++ b/services/pddb/src/lib.rs @@ -527,6 +527,56 @@ impl Pddb { _ => Err(Error::new(ErrorKind::Other, "Internal error")) } } + /// deletes a list of keys from a dictionary. The list of keys must be less than MAX_PDDB_DELETE_LEN characters long. + pub fn delete_key_list(&self, dict_name: &str, key_list: Vec::, basis_name: Option<&str>) -> Result<()> { + if key_list.len() == 0 { + return Ok(()) + } + if dict_name.len() > (DICT_NAME_LEN - 1) { + return Err(Error::new(ErrorKind::InvalidInput, "dictionary name too long")); + } + let bname = if let Some(bname) = basis_name { + if bname.len() > BASIS_NAME_LEN - 1 { + return Err(Error::new(ErrorKind::InvalidInput, "basis name too long")); + } + xous_ipc::String::::from_str(bname) + } else { + xous_ipc::String::::new() + }; + let mut request = PddbDeleteList { + basis_specified: basis_name.is_some(), + basis: xous_ipc::String::::from_str(&bname), + dict: xous_ipc::String::::from_str(dict_name), + retcode: PddbRetcode::Uninit, + data: [0u8; MAX_PDDB_DELETE_LEN] + }; + let mut index = 0; + for keyname in key_list { + if keyname.len() > (KEY_NAME_LEN - 1) { + return Err(Error::new(ErrorKind::InvalidInput, "one of the key names is too long")); + } + if keyname.len() + 1 + index < MAX_PDDB_DELETE_LEN { + assert!(keyname.len() < u8::MAX as usize); // this should always be true due to other limits in the PDDB + request.data[index] = keyname.len() as u8; + index += 1; + request.data[index..index + keyname.len()].copy_from_slice(keyname.as_bytes()); + index += keyname.len(); + } else { + return Err(Error::new(ErrorKind::OutOfMemory, "Key list total size exceeds MAX_PDDBKLISTLEN")); + } + } + let mut buf = Buffer::into_buf(request) + .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; + buf.lend_mut(self.conn, Opcode::DictBulkDelete.to_u32().unwrap()) + .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; + let response = buf.as_flat::().unwrap(); + match response.retcode { + ArchivedPddbRetcode::Ok => Ok(()), + ArchivedPddbRetcode::AccessDenied => Err(Error::new(ErrorKind::NotFound, "Dictionary not found, or inaccessible")), + ArchivedPddbRetcode::Uninit => Err(Error::new(ErrorKind::ConnectionAborted, "Return code not set processing bulk delete, server aborted?")), + _ => Err(Error::new(ErrorKind::Other, "Internal Error handling bulk delete list")), + } + } /// deletes the entire dictionary pub fn delete_dict(&self, dict_name: &str, basis_name: Option<&str>) -> Result<()> { if dict_name.len() > (DICT_NAME_LEN - 1) { diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 7f4b7e0f4..6546a0ef3 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -1148,6 +1148,48 @@ fn wrapped_main() -> ! { } buffer.replace(req).unwrap(); } + Opcode::DictBulkDelete => { + let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) }; + let mut req = buffer.to_original::().unwrap(); + let mut key_list = Vec::::new(); + // the [u8] data is structured as a packed list of u8-len + u8 data slice. The max length of + // a PDDB key name is guaranteed to be shorter than a u8. If the length field is 0, then this + // particular response has no more data in it to read. + let mut index = 0; + while req.data[index] != 0 && index < MAX_PDDB_DELETE_LEN { + let strlen = req.data[index] as usize; + index += 1; + if strlen + index >= MAX_PDDB_DELETE_LEN { + log::error!("Logic error in key list, index would be out of bounds. Aborting"); + req.retcode = PddbRetcode::InternalError; + break; + } + let key = String::from(std::str::from_utf8(&req.data[index..index+strlen]).unwrap_or("UTF8 error")); + key_list.push(key); + index += strlen; + } + if req.retcode == PddbRetcode::InternalError { + buffer.replace(req).ok(); + continue; + } + log::info!("Deleting key list: {:?}", key_list); + let start = tt.elapsed_ms(); + let bname = if req.basis_specified { + Some(req.basis.as_str().unwrap()) + } else { + None + }; + let dict = req.dict.as_str().expect("dict utf-8 decode error"); + match basis_cache.key_list_remove(&mut pddb_os, dict, key_list, bname) { + Ok(_) => req.retcode = PddbRetcode::Ok, + Err(e) => match e.kind() { + std::io::ErrorKind::NotFound => req.retcode = PddbRetcode::AccessDenied, + _ => req.retcode = PddbRetcode::InternalError, + } + } + log::info!("Bulk delete finished in {}ms", tt.elapsed_ms() - start); + buffer.replace(req).ok(); + } Opcode::DeleteKeyStd => { if let Some(mem) = msg.body.memory_message_mut() { mem.offset = None;