From 7cb98699aa6196900fc451c75f9e7392f7427ba4 Mon Sep 17 00:00:00 2001 From: bunnie Date: Fri, 27 Jan 2023 15:26:46 +0800 Subject: [PATCH] add some consistency check and repair facility to PDDB --- services/pddb/src/api.rs | 4 ++ services/pddb/src/backend/basis.rs | 60 +++++++++++++++++-------- services/pddb/src/backend/dictionary.rs | 11 +++-- services/pddb/src/lib.rs | 42 +++++++++++++++++ services/pddb/src/libstd/mod.rs | 8 ++-- services/pddb/src/main.rs | 14 +++--- services/pddb/src/tests.rs | 27 ++++++----- services/shellchat/src/cmds/pddb_cmd.rs | 20 +++++++++ 8 files changed, 144 insertions(+), 42 deletions(-) diff --git a/services/pddb/src/api.rs b/services/pddb/src/api.rs index 282103ae4..61e85b1db 100644 --- a/services/pddb/src/api.rs +++ b/services/pddb/src/api.rs @@ -308,6 +308,10 @@ pub struct PddbDictRequest { pub code: PddbRequestCode, /// used only to specify a readback size limit for bulk-return requests pub bulk_limit: Option, + /// return value for key_count metadata (informative only) + pub key_count: u32, + /// return value for found_key_count metadata (informative only) + pub found_key_count: u32, } /// A structure for requesting a token to access a particular key/value pair diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index a157505de..ae63a7f7b 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -339,6 +339,7 @@ impl BasisCache { last_disk_key_index: 1, // we know we don't have to search past this in a new dictionary flags: init_flags, key_count: 0, + found_key_count: 0, small_pool: Vec::::new(), small_pool_free: BinaryHeap::::new(), aad: my_aad, @@ -348,7 +349,7 @@ impl BasisCache { basis.dicts.insert(String::from(name), dict_cache); basis.num_dicts += 1; // encrypt and write the dict entry to disk - basis.dict_sync(hw, name)?; + basis.dict_sync(hw, name, false)?; // sync the root basis structure as well, while we're at it... basis.basis_sync(hw); // finally, sync the page tables. @@ -386,9 +387,11 @@ impl BasisCache { } dict_set } - pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result> { + pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result<(BTreeSet::, u32, u32)> { let mut merge_list = BTreeSet::::new(); let mut found_dict = false; + let mut key_count: u32 = 0; + let mut found_key_count: u32 = 0; if basis_name.is_some() { if let Some(basis_index) = self.select_basis(basis_name) { let basis = &mut self.cache[basis_index]; @@ -396,6 +399,8 @@ impl BasisCache { if let Some(dcache) = basis.dicts.get_mut(dict) { dcache.key_list(hw, &basis.v2p_map, &basis.cipher, &mut merge_list); found_dict = true; + key_count += dcache.key_count; + found_key_count += dcache.found_key_count; } } } else { @@ -404,11 +409,13 @@ impl BasisCache { if let Some(dcache) = basis.dicts.get_mut(dict) { dcache.key_list(hw, &basis.v2p_map, &basis.cipher, &mut merge_list); found_dict = true; + key_count += dcache.key_count; + found_key_count += dcache.found_key_count; } } } if found_dict { - Ok(merge_list) + Ok((merge_list, key_count, found_key_count)) } else { return Err(Error::new(ErrorKind::NotFound, "dictionary not found")) } @@ -580,7 +587,7 @@ impl BasisCache { } dict_entry.sync_large_pool(); // encrypt and write the dict entry to disk - basis.dict_sync(hw, dict)?; + basis.dict_sync(hw, dict, false)?; // sync the root basis structure as well, while we're at it... basis.basis_sync(hw); // finally, sync the page tables. @@ -620,7 +627,7 @@ impl BasisCache { } dict_entry.sync_large_pool(); // encrypt and write the dict entry to disk - basis.dict_sync(hw, dict)?; + basis.dict_sync(hw, dict, false)?; // sync the root basis structure as well, while we're at it... basis.basis_sync(hw); // finally, sync the page tables. @@ -765,7 +772,7 @@ impl BasisCache { dict_entry.sync_large_pool(); // encrypt and write the dict entry to disk - basis.dict_sync(hw, dict)?; + basis.dict_sync(hw, dict, false)?; // sync the root basis structure as well, while we're at it... basis.basis_sync(hw); // finally, sync the page tables. @@ -986,7 +993,7 @@ impl BasisCache { pub(crate) fn basis_unmount(&mut self, hw: &mut PddbOs, basis_name: &str) -> Result<()> { if let Some(basis_index) = self.select_basis(Some(basis_name)) { let basis = &mut self.cache[basis_index]; - basis.sync(hw)?; + basis.sync(hw, false)?; self.cache.retain(|x| x.name != basis_name); Ok(()) } else { @@ -1014,22 +1021,25 @@ impl BasisCache { } } - pub(crate) fn sync(&mut self, hw: &mut PddbOs, basis_name: Option<&str>) -> Result<()> { + pub(crate) fn sync(&mut self, hw: &mut PddbOs, basis_name: Option<&str>, cleanup: bool) -> Result<()> { + if cleanup { + log::info!("calling sync with cleanup!"); + } if basis_name.is_some() { if let Some(basis_index) = self.select_basis(basis_name) { - self.cache[basis_index].sync(hw)? + self.cache[basis_index].sync(hw, cleanup)? } } else { for basis in self.cache.iter_mut() { log::debug!("syncing {}", basis.name); - basis.sync(hw)?; + basis.sync(hw, cleanup)?; } } Ok(()) } pub(crate) fn suspend(&mut self, hw: &mut PddbOs) { - self.sync(hw, None).expect("couldn't sync on suspend"); + self.sync(hw, None, false).expect("couldn't sync on suspend"); let mut lock_list = Vec::::new(); for basis in self.cache.iter_mut() { match basis.policy { @@ -1075,7 +1085,7 @@ impl BasisCache { // 1. iterate through all the known keys, creating a sorted heap of fully-specified keys (basis/dict/key) // sorted by access time // 2. evict keys with the oldest access time, until target_bytes is hit. - self.sync(hw, None).unwrap(); + self.sync(hw, None, false).unwrap(); let mut candidates = BinaryHeap::new(); for basis in self.cache.iter() { @@ -1285,7 +1295,7 @@ impl BasisCacheEntry { let mut largest_extent = 0; for dict in self.dicts.values_mut() { if dict.flags.valid() { - let extent = dict.fill(hw, &mut self.v2p_map, &self.cipher).get(); + let extent = dict.fill(hw, &mut self.v2p_map, &self.cipher, false).get(); if extent > largest_extent { largest_extent = extent; } @@ -1309,12 +1319,12 @@ impl BasisCacheEntry { }; if !dict_present_and_valid { let mut dcache = DictCacheEntry::new(dict, try_entry, &self.aad); - let max_large_alloc = dcache.fill(hw, &self.v2p_map, &self.cipher); + let max_large_alloc = dcache.fill(hw, &self.v2p_map, &self.cipher, false); self.dicts.insert(dict_name.to_string(), dcache); self.large_pool_update(max_large_alloc.get()); } else { let dcache = self.dicts.get_mut(&dict_name).expect("dict should be present, as we checked for it already..."); - let extent = dcache.fill(hw, &mut self.v2p_map, &self.cipher).get(); + let extent = dcache.fill(hw, &mut self.v2p_map, &self.cipher, false).get(); self.large_pool_update(extent); } dict_count += 1; @@ -1347,7 +1357,7 @@ impl BasisCacheEntry { let dcache = self.dicts.get_mut(name).expect("entry was ensured, but somehow missing"); log::trace!("dcache {} index {}, key_count {}", name, dcache.index, dcache.key_count); // ensure all the keys are in RAM - dcache.fill(hw, &self.v2p_map, &self.cipher); + dcache.fill(hw, &self.v2p_map, &self.cipher, false); // allocate a copy of the key list, to avoid interior mutability problems with the next remove step let mut key_list = Vec::::new(); @@ -1525,10 +1535,17 @@ impl BasisCacheEntry { /// allocated. You can try calling it without pre-allocating the entries, but if the FastSpace structure /// doesn't have enough space, the routine will return an error indicating we're out of memory. /// You could then try to allocate more FastSpace, and re-try the sync operation. - pub(crate) fn dict_sync(&mut self, hw: &mut PddbOs, name: &str) -> Result<()> { + /// + /// If `cleanup` is true, then the keycount field is set to the number of keys actually found. This + /// "fixes" inconsistencies by just trimming records that got lost. + pub(crate) fn dict_sync(&mut self, hw: &mut PddbOs, name: &str, cleanup: bool) -> Result<()> { self.last_sync = Some(hw.timestamp_now()); if let Some(dict) = self.dicts.get_mut(&String::from(name)) { let dict_offset = VirtAddr::new(dict.index.get() as u64 * DICT_VSIZE).unwrap(); + if cleanup && (dict.key_count != dict.found_key_count) { + log::info!("Cleaning up key_count inconsistency: {} -> {}", dict.key_count, dict.found_key_count); + dict.key_count = dict.found_key_count; + } if !dict.clean { let dict_name = DictName::try_from_str(name).or(Err(Error::new(ErrorKind::InvalidInput, "dictionary name invalid: invalid utf-8 or length")))?; let dict_disk = Dictionary { @@ -1737,7 +1754,7 @@ impl BasisCacheEntry { dict_found } - pub(crate) fn sync(&mut self, hw: &mut PddbOs) -> Result<()> { + pub(crate) fn sync(&mut self, hw: &mut PddbOs, cleanup: bool) -> Result<()> { self.dicts.retain(|_name, entry| entry.flags.valid()); // prune any invalid dictionary entries, as they have been deleted // this is a bit awkward, but we have to make a copy of all the dictionary names // because otherwise we borrow self as immutable to enumerate the names, and then @@ -1754,9 +1771,14 @@ impl BasisCacheEntry { return Err(Error::new(ErrorKind::OutOfMemory, "Ran out of memory syncing small pool")); } dict_entry.sync_large_pool(); + if cleanup { + // this will do a full-scan through all the key entries to discover any extra + // ones that are "way out" in the storage space that could have been lost + dict_entry.fill(hw, &mut self.v2p_map, &self.cipher, cleanup); + } } - match self.dict_sync(hw, &dict) { + match self.dict_sync(hw, &dict, cleanup) { Ok(_) => {}, Err(e) => { log::error!("Error encountered syncing dict {}: {:?}", dict, e); diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 9bb1cb637..dc8397b94 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -34,6 +34,8 @@ pub(crate) struct DictCacheEntry { pub(crate) keys: HashMap::, /// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys` pub(crate) key_count: u32, + /// actual count of keys found -- this drifts from the actual key count due to write errors and ungraceful powerdowns + pub(crate) found_key_count: u32, /// track the pool of free key indices. Wrapped in a refcell so we can work the index mechanism while updating the keys HashMap pub(crate) free_keys: BinaryHeap::>, /// hint for when to stop doing a brute-force search for the existence of a key in the disk records. @@ -77,6 +79,7 @@ impl DictCacheEntry { index: NonZeroU32::new(index as u32).unwrap(), keys: HashMap::::new(), key_count: dict.num_keys, + found_key_count: dict.num_keys, free_keys, last_disk_key_index: dict.free_key_index, clean: false, @@ -94,7 +97,7 @@ impl DictCacheEntry { /// Todo: condense routines in common with ensure_key_entry() to make it easier to maintain. /// /// ASSUMES: all small/large pools have been sync'd prior to calling this - pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv) -> VirtAddr { + pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv, cleanup: bool) -> VirtAddr { let mut try_entry = 1; let mut key_count = 0; let mut alloc_top = VirtAddr::new(LARGE_POOL_START).unwrap(); @@ -133,7 +136,7 @@ impl DictCacheEntry { let mut index_cache = PlaintextCache { data: None, tag: None }; let mut data_cache = PlaintextCache { data: None, tag: None }; let mut errcnt = 0; - while try_entry < KEY_MAXCOUNT && key_count < self.key_count { + while try_entry < KEY_MAXCOUNT && (cleanup || (key_count < self.key_count)) { #[cfg(feature="perfcounter")] hw.perf_entry(FILE_ID_SERVICES_PDDB_SRC_DICTIONARY, PERFMETA_NONE, key_count, std::line!()); // cache our decryption data -- there's about 32 entries per page, and the scan is largely linear/sequential, so this should @@ -198,6 +201,8 @@ impl DictCacheEntry { } // note where the scan left off, so we don't have to brute-force it in the future self.last_disk_key_index = try_entry as u32; + // note the actual number of keys found + self.found_key_count = key_count; // now build the small_pool_free binary heap structure #[cfg(feature="perfcounter")] @@ -215,7 +220,7 @@ impl DictCacheEntry { hw.perf_entry(FILE_ID_SERVICES_PDDB_SRC_DICTIONARY, PERFMETA_STARTBLOCK, 0, std::line!()); // ensure that the key cache is filled if self.keys.len() < self.key_count as usize { - self.fill(hw, v2p_map, cipher); + self.fill(hw, v2p_map, cipher, false); } #[cfg(feature="perfcounter")] hw.perf_entry(FILE_ID_SERVICES_PDDB_SRC_DICTIONARY, PERFMETA_NONE, 0, std::line!()); diff --git a/services/pddb/src/lib.rs b/services/pddb/src/lib.rs index fd0fdadab..50e75342d 100644 --- a/services/pddb/src/lib.rs +++ b/services/pddb/src/lib.rs @@ -99,6 +99,9 @@ pub struct Pddb { /// like this are probably OK. keys: Arc >>>, trng: trng::Trng, + /// These are temporary fields only to be used by the consistency check feature. + key_count: RefCell, + found_key_count: RefCell, } impl Pddb { pub fn new() -> Self { @@ -112,6 +115,9 @@ impl Pddb { cb_handle: RefCell::new(None), keys, trng: trng::Trng::new(&xns).unwrap(), + /// These are record the result of the most recent call to list_keys() + key_count: RefCell::new(0), + found_key_count: RefCell::new(0), } } fn ensure_async_responder(&self) { @@ -640,6 +646,32 @@ impl Pddb { Err(Error::new(ErrorKind::Other, "Xous internal error")) } } + /// cleans up inconsistencies in the PDDB. fsck-like. + pub fn sync_cleanup(&self) -> Result<()> { + let response = send_message( + self.conn, + Message::new_blocking_scalar(Opcode::WriteKeyFlush.to_usize().unwrap(), 1, 0, 0, 0) + ).or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; + if let xous::Result::Scalar1(rcode) = response { + match FromPrimitive::from_u8(rcode as u8) { + Some(PddbRetcode::Ok) => Ok(()), + Some(PddbRetcode::BasisLost) => Err(Error::new(ErrorKind::BrokenPipe, "Basis lost")), + Some(PddbRetcode::DiskFull) => Err(Error::new(ErrorKind::OutOfMemory, "Out of disk space, some data lost on sync")), + _ => Err(Error::new(ErrorKind::Interrupted, "Flush failed for unspecified reasons")), + } + } else { + Err(Error::new(ErrorKind::Other, "Xous internal error")) + } + } + + /// Returns a tuple of (recorded, found) keys. If record != found then the dictionary is inconsistent. + /// Only valid if the most recent call to the pddb object was list_keys() (as called below) + /// This is meant to be used only for certain maintenance routines, so the ergonomics are crap on it. + /// + /// SAFETY: caller must guarantee that list_keys() was called prior to calling this + pub unsafe fn was_listed_dict_consistent(&self) -> (u32, u32) { + (*self.key_count.borrow(), *self.found_key_count.borrow()) + } pub fn list_keys(&self, dict_name: &str, basis_name: Option<&str>) -> Result> { if dict_name.len() > (DICT_NAME_LEN - 1) { @@ -664,6 +696,8 @@ impl Pddb { code: PddbRequestCode::Uninit, token, bulk_limit: None, + key_count: 0, + found_key_count: 0, }; let mut buf = Buffer::into_buf(request) .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; @@ -678,6 +712,8 @@ impl Pddb { PddbRequestCode::Uninit => return Err(Error::new(ErrorKind::ConnectionAborted, "Return code not set getting key count, server aborted?")), _ => return Err(Error::new(ErrorKind::Other, "Internal error generating key count")), }; + *self.key_count.borrow_mut() = response.key_count; + *self.found_key_count.borrow_mut() = response.found_key_count; // v2 key listing packs the key list into a larger [u8] field that should cut down on the number of messages // required to list a large dictionary by about a factor of 50. let mut key_list = Vec::::new(); @@ -749,6 +785,8 @@ impl Pddb { code: PddbRequestCode::Uninit, token, bulk_limit: None, + key_count: 0, + found_key_count: 0, }; let mut buf = Buffer::into_buf(request) .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; @@ -774,6 +812,8 @@ impl Pddb { code: PddbRequestCode::Uninit, token, bulk_limit: None, + key_count: 0, + found_key_count: 0, }; let mut buf = Buffer::into_buf(request) .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?; @@ -939,6 +979,8 @@ impl Pddb { token: [0u32; 4], code: PddbRequestCode::BulkRead, bulk_limit: Some(size_limit.unwrap_or(DEFAULT_LIMIT)), + key_count: 0, + found_key_count: 0, }; // main loop let mut ret = Vec::new(); diff --git a/services/pddb/src/libstd/mod.rs b/services/pddb/src/libstd/mod.rs index db4c3db9a..0cd11956b 100644 --- a/services/pddb/src/libstd/mod.rs +++ b/services/pddb/src/libstd/mod.rs @@ -114,7 +114,7 @@ pub(crate) fn stat_path( // Find all keys that are in this dict. Ignore errors, since sometimes // the dict doesn't exist, which is fine. if let Some((dict_path, key_path)) = stripped_path.rsplit_once(std::path::MAIN_SEPARATOR) { - if let Some(key_list) = basis_cache + if let Some((key_list, _, _)) = basis_cache .key_list(pddb_os, dict_path, basis.as_deref()) .map_err(|e| { // log::error!("unable to get key list: {:?}", e); @@ -196,7 +196,7 @@ pub(crate) fn list_path( let dict_list = basis_cache.dict_list(pddb_os, basis.as_deref()); // Find all keys that are in this dict. Ignore errors, since sometimes // the dict doesn't exist, which is fine. - let key_list = basis_cache + let (key_list, _, _) = basis_cache .key_list(pddb_os, dict, basis.as_deref()) .map_err(|e| { // log::error!("unable to get key list: {:?}", e); @@ -809,7 +809,7 @@ pub(crate) fn list_key( .writer(*b"LiKR") .ok_or(crate::PddbRetcode::InternalError)?; - let key_list = basis_cache + let (key_list, _, _) = basis_cache .key_list(pddb_os, &key, bname.as_deref()) .or_else(|e| { log::error!( @@ -848,7 +848,7 @@ pub(crate) fn delete_dict( .or(Err(crate::PddbRetcode::InternalError))?; let dict = dict.ok_or(crate::PddbRetcode::InternalError)?; - if let Some(key_list) = basis_cache + if let Some((key_list, _, _)) = basis_cache .key_list(pddb_os, &dict, bname.as_deref()) .map_err(|e| { // log::error!("unable to get key list: {:?}", e); diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 6546a0ef3..f2d7ae1ff 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -1306,8 +1306,10 @@ fn wrapped_main() -> ! { #[cfg(feature="perfcounter")] pddb_os.perf_entry(FILE_ID_SERVICES_PDDB_SRC_MAIN, perflib::PERFMETA_NONE, 0, std::line!()); key_list = match basis_cache.key_list(&mut pddb_os, dict, bname) { - Ok(list) => { + Ok((list, key_count, found_key_count)) => { log::debug!("count: {}", list.len()); + req.key_count = key_count; + req.found_key_count = found_key_count; req.code = PddbRequestCode::NoErr; Some(list) } @@ -1528,7 +1530,7 @@ fn wrapped_main() -> ! { None } ) { - Ok(list) => list.into_iter().rev().collect(), // reverse order so Vec can just pop and get "first" item + Ok((list, _, _)) => list.into_iter().rev().collect(), // reverse order so Vec can just pop and get "first" item Err(e) => { match e.kind() { std::io::ErrorKind::NotFound => { @@ -1773,7 +1775,7 @@ fn wrapped_main() -> ! { // we don't need a "replace" operation because all ops happen in-place // for now, do an expensive sync operation after every write to ensure data integrity - basis_cache.sync(&mut pddb_os, None).expect("couldn't sync basis"); + basis_cache.sync(&mut pddb_os, None, false).expect("couldn't sync basis"); } Opcode::WriteKeyStd => { @@ -1786,8 +1788,8 @@ fn wrapped_main() -> ! { } } - Opcode::WriteKeyFlush => msg_blocking_scalar_unpack!(msg, _, _, _, _, { - match basis_cache.sync(&mut pddb_os, None) { + Opcode::WriteKeyFlush => msg_blocking_scalar_unpack!(msg, cleanup, _, _, _, { + match basis_cache.sync(&mut pddb_os, None, if cleanup == 1 { true } else { false }) { Ok(_) => xous::return_scalar(msg.sender, PddbRetcode::Ok.to_usize().unwrap()).unwrap(), Err(e) => match e.kind() { std::io::ErrorKind::OutOfMemory => xous::return_scalar(msg.sender, PddbRetcode::DiskFull.to_usize().unwrap()).unwrap(), @@ -1898,7 +1900,7 @@ fn wrapped_main() -> ! { xous::return_scalar(msg.sender, 1).unwrap(); // nothing to do, nothing mounted. success! continue; } - basis_cache.sync(&mut pddb_os, None).expect("can't sync for unmount"); + basis_cache.sync(&mut pddb_os, None, false).expect("can't sync for unmount"); // unmount all the open basis first let mut mounted_bases = basis_cache.basis_list(); mounted_bases.retain(|x| x != PDDB_DEFAULT_SYSTEM_BASIS); diff --git a/services/pddb/src/tests.rs b/services/pddb/src/tests.rs index 96560759b..a5d0cae6d 100644 --- a/services/pddb/src/tests.rs +++ b/services/pddb/src/tests.rs @@ -161,7 +161,7 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache, #[cfg(not(feature = "deterministic"))] let dict_list = basis_cache.dict_list(hw, None); for dict in dict_list.iter() { - if let Ok(key_list_unord) = basis_cache.key_list(hw, dict, None) { + if let Ok((key_list_unord, _, _)) = basis_cache.key_list(hw, dict, None) { #[cfg(feature = "deterministic")] let mut key_list = BTreeSet::::new(); #[cfg(feature = "deterministic")] @@ -244,7 +244,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, let mut evict_iter = 0; for (evicted, evict_dict) in dict_list.iter().enumerate() { if evicted < evict_count { - let key_list_unord = basis_cache.key_list(hw, evict_dict, None).unwrap(); + let (key_list_unord, _, _) = basis_cache.key_list(hw, evict_dict, None).unwrap(); let mut key_list = BTreeSet::::new(); for s in key_list_unord { key_list.insert(s); @@ -320,7 +320,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, } log::info!("doing all-basis sync"); - basis_cache.sync(hw, None).unwrap(); + basis_cache.sync(hw, None, false).unwrap(); log::info!("all-basis sync done"); list_all(hw, basis_cache); @@ -366,7 +366,8 @@ pub(crate) fn list_all(hw: &mut PddbOs, basis_cache: &mut BasisCache) { }; log::debug!("{:?}", da); let mut sanity_count = 0; - for key in basis_cache.key_list(hw, dict, None).unwrap().iter() { + let (key_list, _, _) = basis_cache.key_list(hw, dict, None).unwrap(); + for key in key_list.iter() { let attrs = match basis_cache.key_attributes(hw, dict, key, None) { Ok(a) => a, Err(e) => {log::debug!("key not in basis, searching another basis, {:?}", e); continue}, @@ -475,7 +476,8 @@ pub(crate) fn ci_tests(pddb_os: &mut PddbOs) -> Result<()> { let mut pre_list = HashSet::::new(); for dict in basis_cache.dict_list(pddb_os, None).iter() { - for key in basis_cache.key_list(pddb_os, dict, None).unwrap().iter() { + let (key_list, _, _) = basis_cache.key_list(pddb_os, dict, None).unwrap(); + for key in key_list.iter() { pre_list.insert(key.to_string()); } } @@ -504,7 +506,8 @@ pub(crate) fn ci_tests(pddb_os: &mut PddbOs) -> Result<()> { let mut merge_list = HashSet::::new(); for dict in basis_cache.dict_list(pddb_os, None).iter() { - for key in basis_cache.key_list(pddb_os, dict, None).unwrap().iter() { + let (key_list, _, _) = basis_cache.key_list(pddb_os, dict, None).unwrap(); + for key in key_list.iter() { merge_list.insert(key.to_string()); } } @@ -512,7 +515,8 @@ pub(crate) fn ci_tests(pddb_os: &mut PddbOs) -> Result<()> { let mut b2_list = HashSet::::new(); for dict in basis_cache.dict_list(pddb_os, Some(EXTRA_BASIS)).iter() { - for key in basis_cache.key_list(pddb_os, dict, Some(EXTRA_BASIS)).unwrap().iter() { + let (key_list, _, _) = basis_cache.key_list(pddb_os, dict, Some(EXTRA_BASIS)).unwrap(); + for key in key_list.iter() { b2_list.insert(key.to_string()); } } @@ -521,7 +525,8 @@ pub(crate) fn ci_tests(pddb_os: &mut PddbOs) -> Result<()> { basis_cache.basis_unmount(pddb_os, EXTRA_BASIS).unwrap(); let mut post_list = HashSet::::new(); for dict in basis_cache.dict_list(pddb_os, None).iter() { - for key in basis_cache.key_list(pddb_os, dict, None).unwrap().iter() { + let (key_list, _, _) = basis_cache.key_list(pddb_os, dict, None).unwrap(); + for key in key_list.iter() { post_list.insert(key.to_string()); } } @@ -533,7 +538,8 @@ pub(crate) fn ci_tests(pddb_os: &mut PddbOs) -> Result<()> { } let mut remount_list = HashSet::::new(); for dict in basis_cache.dict_list(pddb_os, None).iter() { - for key in basis_cache.key_list(pddb_os, dict, None).unwrap().iter() { + let (key_list, _, _) = basis_cache.key_list(pddb_os, dict, None).unwrap(); + for key in key_list.iter() { remount_list.insert(key.to_string()); } } @@ -546,7 +552,8 @@ pub(crate) fn ci_tests(pddb_os: &mut PddbOs) -> Result<()> { } let mut merge2_list = HashSet::::new(); for dict in basis_cache.dict_list(pddb_os, None).iter() { - for key in basis_cache.key_list(pddb_os, dict, None).unwrap().iter() { + let (key_list, _, _) = basis_cache.key_list(pddb_os, dict, None).unwrap(); + for key in key_list.iter() { merge2_list.insert(key.to_string()); } } diff --git a/services/shellchat/src/cmds/pddb_cmd.rs b/services/shellchat/src/cmds/pddb_cmd.rs index 328816a6f..671dcaf15 100644 --- a/services/shellchat/src/cmds/pddb_cmd.rs +++ b/services/shellchat/src/cmds/pddb_cmd.rs @@ -393,6 +393,26 @@ impl<'a> ShellCmdApi<'a> for PddbCmd { } } } + "consistency" => { + if let Some(dict) = tokens.next() { + match self.pddb.list_keys(dict, None) { + Ok(_list) => { + unsafe{ + let (keys, found_keys) = self.pddb.was_listed_dict_consistent(); + write!(ret, "reported keys: {}, found keys: {}", keys, found_keys).ok(); + log::info!("reported keys: {}, found keys: {}", keys, found_keys); + }; + } + Err(_) => write!(ret, "{} does not exist or other error", dict).ok().unwrap_or(()), + } + } else { + write!(ret, "Missing dictionary name").unwrap(); + } + } + // Warning: this is experimental, it's advised you don't call this unless you know what you're doing. + "cleanup" => { + write!(ret, "Cleanup result code: {:?}\n", self.pddb.sync_cleanup()).ok(); + } #[cfg(feature="pddbtest")] "deltest" => { const JUNK_MIN: usize = 512;