From c84ed61ecd856ed506d079868112b0c616d3a83c Mon Sep 17 00:00:00 2001 From: bunnie Date: Tue, 30 Nov 2021 00:38:29 +0800 Subject: [PATCH] basis overlay test done & passing the formatting of the listing output is a bit messed up but i'm too tired and too late to deal with that. at least the core logic seems functionally correct! --- services/pddb/src/backend/basis.rs | 151 +++++++++++++++++++++-------- services/pddb/src/main.rs | 63 +++++++++++- services/pddb/src/tests.rs | 34 ++++--- 3 files changed, 192 insertions(+), 56 deletions(-) diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index b0f5c307e..ea3cb640c 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -347,26 +347,49 @@ impl BasisCache { /// Returns a list of all the known dictionaries, across all the basis. A HashSet is returned /// because you can have the same-named dictionary in multiple basis, and what we're asking for /// is the union of all the dictionary names, without duplicates. - pub(crate) fn dict_list(&mut self, hw: &mut PddbOs) -> HashSet:: { + pub(crate) fn dict_list(&mut self, hw: &mut PddbOs, basis_name: Option<&str>) -> HashSet:: { let mut dict_set = HashSet::::new(); - for basis in self.cache.iter_mut() { - basis.populate_caches(hw); - for (key, dcache) in basis.dicts.iter() { - if dcache.flags.valid() { - dict_set.insert(String::from(key)); + if basis_name.is_some() { + if let Some(basis_index) = self.select_basis(basis_name) { + let basis = &mut self.cache[basis_index]; + basis.populate_caches(hw); + for (key, dcache) in basis.dicts.iter() { + if dcache.flags.valid() { + dict_set.insert(String::from(key)); + } + } + } + } else { + for basis in self.cache.iter_mut() { + basis.populate_caches(hw); + for (key, dcache) in basis.dicts.iter() { + if dcache.flags.valid() { + dict_set.insert(String::from(key)); + } } } } dict_set } - pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str) -> Result> { + pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result> { let mut merge_list = HashSet::::new(); let mut found_dict = false; - for basis in self.cache.iter_mut() { - basis.populate_caches(hw); - if let Some(dcache) = basis.dicts.get_mut(dict) { - dcache.key_list(hw, &basis.v2p_map, &basis.cipher, &mut merge_list); - found_dict = true; + if basis_name.is_some() { + if let Some(basis_index) = self.select_basis(basis_name) { + let basis = &mut self.cache[basis_index]; + basis.populate_caches(hw); + if let Some(dcache) = basis.dicts.get_mut(dict) { + dcache.key_list(hw, &basis.v2p_map, &basis.cipher, &mut merge_list); + found_dict = true; + } + } + } else { + for basis in self.cache.iter_mut() { + basis.populate_caches(hw); + if let Some(dcache) = basis.dicts.get_mut(dict) { + dcache.key_list(hw, &basis.v2p_map, &basis.cipher, &mut merge_list); + found_dict = true; + } } } if found_dict { @@ -680,42 +703,81 @@ impl BasisCache { } pub(crate) fn key_attributes(&mut self, hw: &mut PddbOs, dict: &str, key: &str, basis_name: Option<&str>) -> Result { - if let Some(basis_index) = self.select_basis(basis_name) { - let basis = &mut self.cache[basis_index]; - if !basis.ensure_dict_in_cache(hw, dict) { - return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); - } else { - let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!"); - if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) { - let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!"); - Ok(KeyAttributes { - len: kcache.len as usize, - reserved: kcache.reserved as usize, - age: kcache.age as usize, - dict: dict.to_string(), - basis: (&basis.name).to_string(), - flags: kcache.flags, - }) + if basis_name.is_none() { + for basis in self.cache.iter_mut().rev() { + if !basis.ensure_dict_in_cache(hw, dict) { + continue; } else { - return Err(Error::new(ErrorKind::NotFound, "key not found")); + let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!"); + if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) { + let kcache = match dict_entry.keys.get_mut(key) { + Some(kc) => kc, + None => continue, + }; + return Ok(KeyAttributes { + len: kcache.len as usize, + reserved: kcache.reserved as usize, + age: kcache.age as usize, + dict: dict.to_string(), + basis: (&basis.name).to_string(), + flags: kcache.flags, + }) + } else { + return Err(Error::new(ErrorKind::NotFound, "key not found")); + } } } + Err(Error::new(ErrorKind::NotFound, "key not found")) } else { - Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) + if let Some(basis_index) = self.select_basis(basis_name) { + let basis = &mut self.cache[basis_index]; + if !basis.ensure_dict_in_cache(hw, dict) { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); + } else { + let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!"); + if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) { + let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!"); + Ok(KeyAttributes { + len: kcache.len as usize, + reserved: kcache.reserved as usize, + age: kcache.age as usize, + dict: dict.to_string(), + basis: (&basis.name).to_string(), + flags: kcache.flags, + }) + } else { + return Err(Error::new(ErrorKind::NotFound, "key not found")); + } + } + } else { + Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) + } } } pub(crate) fn dict_attributes(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result { - if let Some(basis_index) = self.select_basis(basis_name) { - let basis = &mut self.cache[basis_index]; - if !basis.ensure_dict_in_cache(hw, dict) { - return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); - } else { - let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!"); - Ok(dict_entry.to_dict_attributes(dict, &basis.name)) + if basis_name.is_none() { + for basis in self.cache.iter_mut().rev() { + if !basis.ensure_dict_in_cache(hw, dict) { + continue; + } else { + let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!"); + return Ok(dict_entry.to_dict_attributes(dict, &basis.name)); + } } + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); } else { - Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) + if let Some(basis_index) = self.select_basis(basis_name) { + let basis = &mut self.cache[basis_index]; + if !basis.ensure_dict_in_cache(hw, dict) { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); + } else { + let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!"); + Ok(dict_entry.to_dict_attributes(dict, &basis.name)) + } + } else { + Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) + } } } @@ -828,10 +890,15 @@ impl BasisCache { self.cache.push(basis); } - pub(crate) fn basis_unmount(&mut self, hw: &mut PddbOs, mut basis: BasisCacheEntry) -> Result<()> { - basis.sync(hw)?; - self.cache.retain(|x| x.name != basis.name); - Ok(()) + pub(crate) fn basis_unmount(&mut self, hw: &mut PddbOs, basis_name: &str) -> Result<()> { + if let Some(basis_index) = self.select_basis(Some(basis_name)) { + let basis = &mut self.cache[basis_index]; + basis.sync(hw)?; + self.cache.retain(|x| x.name != basis_name); + Ok(()) + } else { + Err(Error::new(ErrorKind::NotFound, "Basis not found")) + } } /// note: you can "delete" a basis simply by forgetting its password, but this is more thorough. diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 832dbfcd8..08d7ad83c 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -379,6 +379,8 @@ use xous::{msg_blocking_scalar_unpack, msg_scalar_unpack}; use core::cell::RefCell; use std::rc::Rc; +use std::collections::HashSet; + #[xous::xous_main] fn xmain() -> ! { log_server::init_wait().unwrap(); @@ -455,7 +457,7 @@ fn xmain() -> ! { log::info!("Doing delete pattern test"); delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None); pddb_os.dbg_dump(Some("patterne".to_string()), None); - +/* // extended tests. // allocation space curtailed to force resource exhaustion faster. // note to self: FSCB_PAGES revert to 16 (hw.rs), FASTSPACE_PAGES revert to 2 (fastspace.rs) @@ -484,6 +486,13 @@ fn xmain() -> ! { Some(50), None, None, None); log::info!("Saving `dachecke4` to local host"); pddb_os.dbg_dump(Some("dachecke4".to_string()), None); +*/ + let mut pre_list = HashSet::::new(); + for dict in basis_cache.dict_list(&mut pddb_os, None).iter() { + for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() { + pre_list.insert(key.to_string()); + } + } log::info!("Doing remount disk test"); let mut basis_cache = BasisCache::new(); @@ -504,6 +513,56 @@ fn xmain() -> ! { log::info!("Saving `basis2` to local host"); pddb_os.dbg_dump(Some("basis2".to_string()), Some(&export)); + let mut merge_list = HashSet::::new(); + for dict in basis_cache.dict_list(&mut pddb_os, None).iter() { + for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() { + merge_list.insert(key.to_string()); + } + } + assert!(pre_list.is_subset(&merge_list), "pre-merge list is not a subset of the merged basis"); + + let mut b2_list = HashSet::::new(); + for dict in basis_cache.dict_list(&mut pddb_os, Some(EXTRA_BASIS)).iter() { + for key in basis_cache.key_list(&mut pddb_os, dict, Some(EXTRA_BASIS)).unwrap().iter() { + b2_list.insert(key.to_string()); + } + } + assert!(b2_list.is_subset(&merge_list), "basis 2 is not a subset of the merged lists"); + + basis_cache.basis_unmount(&mut pddb_os, EXTRA_BASIS).unwrap(); + let mut post_list = HashSet::::new(); + for dict in basis_cache.dict_list(&mut pddb_os, None).iter() { + for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() { + post_list.insert(key.to_string()); + } + } + + log::info!("Doing remount disk test part 2"); + let mut basis_cache = BasisCache::new(); + if let Some(sys_basis) = pddb_os.pddb_mount() { + basis_cache.basis_add(sys_basis); + } + let mut remount_list = HashSet::::new(); + for dict in basis_cache.dict_list(&mut pddb_os, None).iter() { + for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() { + remount_list.insert(key.to_string()); + } + } + assert!(remount_list.difference(&pre_list).count() == 0, "remounted list is not identical to the original list"); + + log::info!("Mounting the second basis again"); + if let Some(basis2) = basis_cache.basis_unlock(&mut pddb_os, + EXTRA_BASIS, EXTRA_BASIS_PW, BasisRetentionPolicy::Persist) { + basis_cache.basis_add(basis2); + } + let mut merge2_list = HashSet::::new(); + for dict in basis_cache.dict_list(&mut pddb_os, None).iter() { + for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() { + merge2_list.insert(key.to_string()); + } + } + assert!(merge2_list.difference(&merge_list).count() == 0, "merged list is different from the original list after remount"); + list_all(&mut pddb_os, &mut basis_cache); } log::info!("CI done"); @@ -547,7 +606,7 @@ fn xmain() -> ! { - [done] key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data. - [done] fast space exhaustion test: allocate and delete a bunch of stuff. trigger a fast-space regenerate. note: for faster stress-testing, we dialed the FSCB_PAGES to 4 and the FASTSPACE_PAGES to 1. - - basis search: create basis A, populate with general integrity. create basis B, add test entries. + - [done] basis search: create basis A, populate with general integrity. create basis B, add test entries. hide basis B, confirm original A; mount basis B, confirm B overlay. */ // register a suspend/resume listener diff --git a/services/pddb/src/tests.rs b/services/pddb/src/tests.rs index 3050d3abc..bdcaadd43 100644 --- a/services/pddb/src/tests.rs +++ b/services/pddb/src/tests.rs @@ -85,7 +85,7 @@ pub(crate) fn delete_add_dict_consistency(hw: &mut PddbOs, basis_cache: &mut Bas let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((LOWER_BOUND, UPPER_BOUND - 4)); let extra_reserved = maybe_extra_reserved.unwrap_or(0); - let dict_list = basis_cache.dict_list(hw); + let dict_list = basis_cache.dict_list(hw, None); let dict_start_index = dict_list.len(); for (evicted, evict_dict) in dict_list.iter().enumerate() { if evicted < evict_count { @@ -124,9 +124,9 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache, let patch_offset = maybe_patch_offset.unwrap_or(5); let patch_data = maybe_patch_data.unwrap_or("patched!".to_string()); - let dict_list = basis_cache.dict_list(hw); + let dict_list = basis_cache.dict_list(hw, None); for dict in dict_list.iter() { - if let Ok(key_list) = basis_cache.key_list(hw, dict) { + if let Ok(key_list) = basis_cache.key_list(hw, dict, None) { for key in key_list.iter() { // this actually does something a bit more complicated on a multi-basis system than you'd think: // it will get the union of all key names, and then patch the *latest open basis* only with new data. @@ -185,11 +185,11 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, let extra_reserved = maybe_extra_reserved.unwrap_or(0); let mut evicted_dicts = Vec::::new(); - let dict_list = basis_cache.dict_list(hw); + let dict_list = basis_cache.dict_list(hw, None); let mut evict_iter = 0; for (evicted, evict_dict) in dict_list.iter().enumerate() { if evicted < evict_count { - for (index, key) in basis_cache.key_list(hw, evict_dict).unwrap().iter().enumerate() { + for (index, key) in basis_cache.key_list(hw, evict_dict, None).unwrap().iter().enumerate() { if index % 2 == (evict_iter % 2) { // exercise odd/even patterns log::info!("deleting {}:{}", evict_dict, key); basis_cache.key_remove(hw, evict_dict, key,None, false).unwrap(); @@ -203,7 +203,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, evicted_dicts.push(evict_dict.to_string()); evict_iter += 1; } - for dict in basis_cache.dict_list(hw).iter() { + for dict in basis_cache.dict_list(hw, None).iter() { let da = basis_cache.dict_attributes(hw, dict, None).unwrap(); log::info!("{:?}", da); } @@ -224,7 +224,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, } } } - for dict in basis_cache.dict_list(hw).iter() { + for dict in basis_cache.dict_list(hw, None).iter() { let da = basis_cache.dict_attributes(hw, dict, None).unwrap(); log::info!("{:?}", da); } @@ -234,7 +234,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, log::info!("all-basis sync done"); list_all(hw, basis_cache); - for dict in basis_cache.dict_list(hw).iter() { + for dict in basis_cache.dict_list(hw, None).iter() { let da = basis_cache.dict_attributes(hw, dict, None).unwrap(); log::info!("{:?}", da); } @@ -242,12 +242,22 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, pub(crate) fn list_all(hw: &mut PddbOs, basis_cache: &mut BasisCache) { // now list all the attributes of the basis - for dict in basis_cache.dict_list(hw).iter() { - let da = basis_cache.dict_attributes(hw, &dict, None).unwrap(); + for dict in basis_cache.dict_list(hw, None).iter() { + let da = match basis_cache.dict_attributes(hw, &dict, None) { + Ok(da) => da, + Err(e) => { + log::warn!("dictionary {} not in sub-basis (this is normal for multi-basis systems, {:?}", dict, e); + continue; + } + }; log::info!("{:?}", da); let mut sanity_count = 0; - for key in basis_cache.key_list(hw, dict).unwrap().iter() { - log::info!("{}:{}=>{:?}", dict, key, basis_cache.key_attributes(hw, dict, key, None)); + for key in basis_cache.key_list(hw, dict, None).unwrap().iter() { + let attrs = match basis_cache.key_attributes(hw, dict, key, None) { + Ok(a) => a, + Err(e) => {log::debug!("key not in basis, searching another basis, {:?}", e); continue}, + }; + log::info!("{}:{}=>{:?}", dict, key, attrs); sanity_count += 1; } log::info!("sanity check count: {}", sanity_count);