basis overlay test done & passing

the formatting of the listing output is a bit messed up but
i'm too tired and too late to deal with that. at least the
core logic seems functionally correct!
This commit is contained in:
bunnie 2021-11-30 00:38:29 +08:00
parent 8066b8fb5e
commit c84ed61ecd
3 changed files with 192 additions and 56 deletions

View file

@ -347,26 +347,49 @@ impl BasisCache {
/// Returns a list of all the known dictionaries, across all the basis. A HashSet is returned /// Returns a list of all the known dictionaries, across all the basis. A HashSet is returned
/// because you can have the same-named dictionary in multiple basis, and what we're asking for /// because you can have the same-named dictionary in multiple basis, and what we're asking for
/// is the union of all the dictionary names, without duplicates. /// is the union of all the dictionary names, without duplicates.
pub(crate) fn dict_list(&mut self, hw: &mut PddbOs) -> HashSet::<String> { pub(crate) fn dict_list(&mut self, hw: &mut PddbOs, basis_name: Option<&str>) -> HashSet::<String> {
let mut dict_set = HashSet::<String>::new(); let mut dict_set = HashSet::<String>::new();
for basis in self.cache.iter_mut() { if basis_name.is_some() {
basis.populate_caches(hw); if let Some(basis_index) = self.select_basis(basis_name) {
for (key, dcache) in basis.dicts.iter() { let basis = &mut self.cache[basis_index];
if dcache.flags.valid() { basis.populate_caches(hw);
dict_set.insert(String::from(key)); for (key, dcache) in basis.dicts.iter() {
if dcache.flags.valid() {
dict_set.insert(String::from(key));
}
}
}
} else {
for basis in self.cache.iter_mut() {
basis.populate_caches(hw);
for (key, dcache) in basis.dicts.iter() {
if dcache.flags.valid() {
dict_set.insert(String::from(key));
}
} }
} }
} }
dict_set dict_set
} }
pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str) -> Result<HashSet::<String>> { pub(crate) fn key_list(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result<HashSet::<String>> {
let mut merge_list = HashSet::<String>::new(); let mut merge_list = HashSet::<String>::new();
let mut found_dict = false; let mut found_dict = false;
for basis in self.cache.iter_mut() { if basis_name.is_some() {
basis.populate_caches(hw); if let Some(basis_index) = self.select_basis(basis_name) {
if let Some(dcache) = basis.dicts.get_mut(dict) { let basis = &mut self.cache[basis_index];
dcache.key_list(hw, &basis.v2p_map, &basis.cipher, &mut merge_list); basis.populate_caches(hw);
found_dict = true; if let Some(dcache) = basis.dicts.get_mut(dict) {
dcache.key_list(hw, &basis.v2p_map, &basis.cipher, &mut merge_list);
found_dict = true;
}
}
} else {
for basis in self.cache.iter_mut() {
basis.populate_caches(hw);
if let Some(dcache) = basis.dicts.get_mut(dict) {
dcache.key_list(hw, &basis.v2p_map, &basis.cipher, &mut merge_list);
found_dict = true;
}
} }
} }
if found_dict { if found_dict {
@ -680,42 +703,81 @@ impl BasisCache {
} }
pub(crate) fn key_attributes(&mut self, hw: &mut PddbOs, dict: &str, key: &str, basis_name: Option<&str>) -> Result<KeyAttributes> { pub(crate) fn key_attributes(&mut self, hw: &mut PddbOs, dict: &str, key: &str, basis_name: Option<&str>) -> Result<KeyAttributes> {
if let Some(basis_index) = self.select_basis(basis_name) { if basis_name.is_none() {
let basis = &mut self.cache[basis_index]; for basis in self.cache.iter_mut().rev() {
if !basis.ensure_dict_in_cache(hw, dict) { if !basis.ensure_dict_in_cache(hw, dict) {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); continue;
} else {
let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!");
if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) {
let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!");
Ok(KeyAttributes {
len: kcache.len as usize,
reserved: kcache.reserved as usize,
age: kcache.age as usize,
dict: dict.to_string(),
basis: (&basis.name).to_string(),
flags: kcache.flags,
})
} else { } else {
return Err(Error::new(ErrorKind::NotFound, "key not found")); let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!");
if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) {
let kcache = match dict_entry.keys.get_mut(key) {
Some(kc) => kc,
None => continue,
};
return Ok(KeyAttributes {
len: kcache.len as usize,
reserved: kcache.reserved as usize,
age: kcache.age as usize,
dict: dict.to_string(),
basis: (&basis.name).to_string(),
flags: kcache.flags,
})
} else {
return Err(Error::new(ErrorKind::NotFound, "key not found"));
}
} }
} }
Err(Error::new(ErrorKind::NotFound, "key not found"))
} else { } else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.ensure_dict_in_cache(hw, dict) {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
} else {
let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!");
if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) {
let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!");
Ok(KeyAttributes {
len: kcache.len as usize,
reserved: kcache.reserved as usize,
age: kcache.age as usize,
dict: dict.to_string(),
basis: (&basis.name).to_string(),
flags: kcache.flags,
})
} else {
return Err(Error::new(ErrorKind::NotFound, "key not found"));
}
}
} else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
}
} }
} }
pub(crate) fn dict_attributes(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result<DictAttributes> { pub(crate) fn dict_attributes(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>) -> Result<DictAttributes> {
if let Some(basis_index) = self.select_basis(basis_name) { if basis_name.is_none() {
let basis = &mut self.cache[basis_index]; for basis in self.cache.iter_mut().rev() {
if !basis.ensure_dict_in_cache(hw, dict) { if !basis.ensure_dict_in_cache(hw, dict) {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); continue;
} else { } else {
let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!"); let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!");
Ok(dict_entry.to_dict_attributes(dict, &basis.name)) return Ok(dict_entry.to_dict_attributes(dict, &basis.name));
}
} }
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
} else { } else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.ensure_dict_in_cache(hw, dict) {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
} else {
let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!");
Ok(dict_entry.to_dict_attributes(dict, &basis.name))
}
} else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
}
} }
} }
@ -828,10 +890,15 @@ impl BasisCache {
self.cache.push(basis); self.cache.push(basis);
} }
pub(crate) fn basis_unmount(&mut self, hw: &mut PddbOs, mut basis: BasisCacheEntry) -> Result<()> { pub(crate) fn basis_unmount(&mut self, hw: &mut PddbOs, basis_name: &str) -> Result<()> {
basis.sync(hw)?; if let Some(basis_index) = self.select_basis(Some(basis_name)) {
self.cache.retain(|x| x.name != basis.name); let basis = &mut self.cache[basis_index];
Ok(()) basis.sync(hw)?;
self.cache.retain(|x| x.name != basis_name);
Ok(())
} else {
Err(Error::new(ErrorKind::NotFound, "Basis not found"))
}
} }
/// note: you can "delete" a basis simply by forgetting its password, but this is more thorough. /// note: you can "delete" a basis simply by forgetting its password, but this is more thorough.

View file

@ -379,6 +379,8 @@ use xous::{msg_blocking_scalar_unpack, msg_scalar_unpack};
use core::cell::RefCell; use core::cell::RefCell;
use std::rc::Rc; use std::rc::Rc;
use std::collections::HashSet;
#[xous::xous_main] #[xous::xous_main]
fn xmain() -> ! { fn xmain() -> ! {
log_server::init_wait().unwrap(); log_server::init_wait().unwrap();
@ -455,7 +457,7 @@ fn xmain() -> ! {
log::info!("Doing delete pattern test"); log::info!("Doing delete pattern test");
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None); delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
pddb_os.dbg_dump(Some("patterne".to_string()), None); pddb_os.dbg_dump(Some("patterne".to_string()), None);
/*
// extended tests. // extended tests.
// allocation space curtailed to force resource exhaustion faster. // allocation space curtailed to force resource exhaustion faster.
// note to self: FSCB_PAGES revert to 16 (hw.rs), FASTSPACE_PAGES revert to 2 (fastspace.rs) // note to self: FSCB_PAGES revert to 16 (hw.rs), FASTSPACE_PAGES revert to 2 (fastspace.rs)
@ -484,6 +486,13 @@ fn xmain() -> ! {
Some(50), None, None, None); Some(50), None, None, None);
log::info!("Saving `dachecke4` to local host"); log::info!("Saving `dachecke4` to local host");
pddb_os.dbg_dump(Some("dachecke4".to_string()), None); pddb_os.dbg_dump(Some("dachecke4".to_string()), None);
*/
let mut pre_list = HashSet::<String>::new();
for dict in basis_cache.dict_list(&mut pddb_os, None).iter() {
for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() {
pre_list.insert(key.to_string());
}
}
log::info!("Doing remount disk test"); log::info!("Doing remount disk test");
let mut basis_cache = BasisCache::new(); let mut basis_cache = BasisCache::new();
@ -504,6 +513,56 @@ fn xmain() -> ! {
log::info!("Saving `basis2` to local host"); log::info!("Saving `basis2` to local host");
pddb_os.dbg_dump(Some("basis2".to_string()), Some(&export)); pddb_os.dbg_dump(Some("basis2".to_string()), Some(&export));
let mut merge_list = HashSet::<String>::new();
for dict in basis_cache.dict_list(&mut pddb_os, None).iter() {
for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() {
merge_list.insert(key.to_string());
}
}
assert!(pre_list.is_subset(&merge_list), "pre-merge list is not a subset of the merged basis");
let mut b2_list = HashSet::<String>::new();
for dict in basis_cache.dict_list(&mut pddb_os, Some(EXTRA_BASIS)).iter() {
for key in basis_cache.key_list(&mut pddb_os, dict, Some(EXTRA_BASIS)).unwrap().iter() {
b2_list.insert(key.to_string());
}
}
assert!(b2_list.is_subset(&merge_list), "basis 2 is not a subset of the merged lists");
basis_cache.basis_unmount(&mut pddb_os, EXTRA_BASIS).unwrap();
let mut post_list = HashSet::<String>::new();
for dict in basis_cache.dict_list(&mut pddb_os, None).iter() {
for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() {
post_list.insert(key.to_string());
}
}
log::info!("Doing remount disk test part 2");
let mut basis_cache = BasisCache::new();
if let Some(sys_basis) = pddb_os.pddb_mount() {
basis_cache.basis_add(sys_basis);
}
let mut remount_list = HashSet::<String>::new();
for dict in basis_cache.dict_list(&mut pddb_os, None).iter() {
for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() {
remount_list.insert(key.to_string());
}
}
assert!(remount_list.difference(&pre_list).count() == 0, "remounted list is not identical to the original list");
log::info!("Mounting the second basis again");
if let Some(basis2) = basis_cache.basis_unlock(&mut pddb_os,
EXTRA_BASIS, EXTRA_BASIS_PW, BasisRetentionPolicy::Persist) {
basis_cache.basis_add(basis2);
}
let mut merge2_list = HashSet::<String>::new();
for dict in basis_cache.dict_list(&mut pddb_os, None).iter() {
for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() {
merge2_list.insert(key.to_string());
}
}
assert!(merge2_list.difference(&merge_list).count() == 0, "merged list is different from the original list after remount");
list_all(&mut pddb_os, &mut basis_cache);
} }
log::info!("CI done"); log::info!("CI done");
@ -547,7 +606,7 @@ fn xmain() -> ! {
- [done] key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data. - [done] key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data.
- [done] fast space exhaustion test: allocate and delete a bunch of stuff. trigger a fast-space regenerate. - [done] fast space exhaustion test: allocate and delete a bunch of stuff. trigger a fast-space regenerate.
note: for faster stress-testing, we dialed the FSCB_PAGES to 4 and the FASTSPACE_PAGES to 1. note: for faster stress-testing, we dialed the FSCB_PAGES to 4 and the FASTSPACE_PAGES to 1.
- basis search: create basis A, populate with general integrity. create basis B, add test entries. - [done] basis search: create basis A, populate with general integrity. create basis B, add test entries.
hide basis B, confirm original A; mount basis B, confirm B overlay. hide basis B, confirm original A; mount basis B, confirm B overlay.
*/ */
// register a suspend/resume listener // register a suspend/resume listener

View file

@ -85,7 +85,7 @@ pub(crate) fn delete_add_dict_consistency(hw: &mut PddbOs, basis_cache: &mut Bas
let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((LOWER_BOUND, UPPER_BOUND - 4)); let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((LOWER_BOUND, UPPER_BOUND - 4));
let extra_reserved = maybe_extra_reserved.unwrap_or(0); let extra_reserved = maybe_extra_reserved.unwrap_or(0);
let dict_list = basis_cache.dict_list(hw); let dict_list = basis_cache.dict_list(hw, None);
let dict_start_index = dict_list.len(); let dict_start_index = dict_list.len();
for (evicted, evict_dict) in dict_list.iter().enumerate() { for (evicted, evict_dict) in dict_list.iter().enumerate() {
if evicted < evict_count { if evicted < evict_count {
@ -124,9 +124,9 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache,
let patch_offset = maybe_patch_offset.unwrap_or(5); let patch_offset = maybe_patch_offset.unwrap_or(5);
let patch_data = maybe_patch_data.unwrap_or("patched!".to_string()); let patch_data = maybe_patch_data.unwrap_or("patched!".to_string());
let dict_list = basis_cache.dict_list(hw); let dict_list = basis_cache.dict_list(hw, None);
for dict in dict_list.iter() { for dict in dict_list.iter() {
if let Ok(key_list) = basis_cache.key_list(hw, dict) { if let Ok(key_list) = basis_cache.key_list(hw, dict, None) {
for key in key_list.iter() { for key in key_list.iter() {
// this actually does something a bit more complicated on a multi-basis system than you'd think: // this actually does something a bit more complicated on a multi-basis system than you'd think:
// it will get the union of all key names, and then patch the *latest open basis* only with new data. // it will get the union of all key names, and then patch the *latest open basis* only with new data.
@ -185,11 +185,11 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache,
let extra_reserved = maybe_extra_reserved.unwrap_or(0); let extra_reserved = maybe_extra_reserved.unwrap_or(0);
let mut evicted_dicts = Vec::<String>::new(); let mut evicted_dicts = Vec::<String>::new();
let dict_list = basis_cache.dict_list(hw); let dict_list = basis_cache.dict_list(hw, None);
let mut evict_iter = 0; let mut evict_iter = 0;
for (evicted, evict_dict) in dict_list.iter().enumerate() { for (evicted, evict_dict) in dict_list.iter().enumerate() {
if evicted < evict_count { if evicted < evict_count {
for (index, key) in basis_cache.key_list(hw, evict_dict).unwrap().iter().enumerate() { for (index, key) in basis_cache.key_list(hw, evict_dict, None).unwrap().iter().enumerate() {
if index % 2 == (evict_iter % 2) { // exercise odd/even patterns if index % 2 == (evict_iter % 2) { // exercise odd/even patterns
log::info!("deleting {}:{}", evict_dict, key); log::info!("deleting {}:{}", evict_dict, key);
basis_cache.key_remove(hw, evict_dict, key,None, false).unwrap(); basis_cache.key_remove(hw, evict_dict, key,None, false).unwrap();
@ -203,7 +203,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache,
evicted_dicts.push(evict_dict.to_string()); evicted_dicts.push(evict_dict.to_string());
evict_iter += 1; evict_iter += 1;
} }
for dict in basis_cache.dict_list(hw).iter() { for dict in basis_cache.dict_list(hw, None).iter() {
let da = basis_cache.dict_attributes(hw, dict, None).unwrap(); let da = basis_cache.dict_attributes(hw, dict, None).unwrap();
log::info!("{:?}", da); log::info!("{:?}", da);
} }
@ -224,7 +224,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache,
} }
} }
} }
for dict in basis_cache.dict_list(hw).iter() { for dict in basis_cache.dict_list(hw, None).iter() {
let da = basis_cache.dict_attributes(hw, dict, None).unwrap(); let da = basis_cache.dict_attributes(hw, dict, None).unwrap();
log::info!("{:?}", da); log::info!("{:?}", da);
} }
@ -234,7 +234,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache,
log::info!("all-basis sync done"); log::info!("all-basis sync done");
list_all(hw, basis_cache); list_all(hw, basis_cache);
for dict in basis_cache.dict_list(hw).iter() { for dict in basis_cache.dict_list(hw, None).iter() {
let da = basis_cache.dict_attributes(hw, dict, None).unwrap(); let da = basis_cache.dict_attributes(hw, dict, None).unwrap();
log::info!("{:?}", da); log::info!("{:?}", da);
} }
@ -242,12 +242,22 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache,
pub(crate) fn list_all(hw: &mut PddbOs, basis_cache: &mut BasisCache) { pub(crate) fn list_all(hw: &mut PddbOs, basis_cache: &mut BasisCache) {
// now list all the attributes of the basis // now list all the attributes of the basis
for dict in basis_cache.dict_list(hw).iter() { for dict in basis_cache.dict_list(hw, None).iter() {
let da = basis_cache.dict_attributes(hw, &dict, None).unwrap(); let da = match basis_cache.dict_attributes(hw, &dict, None) {
Ok(da) => da,
Err(e) => {
log::warn!("dictionary {} not in sub-basis (this is normal for multi-basis systems, {:?}", dict, e);
continue;
}
};
log::info!("{:?}", da); log::info!("{:?}", da);
let mut sanity_count = 0; let mut sanity_count = 0;
for key in basis_cache.key_list(hw, dict).unwrap().iter() { for key in basis_cache.key_list(hw, dict, None).unwrap().iter() {
log::info!("{}:{}=>{:?}", dict, key, basis_cache.key_attributes(hw, dict, key, None)); let attrs = match basis_cache.key_attributes(hw, dict, key, None) {
Ok(a) => a,
Err(e) => {log::debug!("key not in basis, searching another basis, {:?}", e); continue},
};
log::info!("{}:{}=>{:?}", dict, key, attrs);
sanity_count += 1; sanity_count += 1;
} }
log::info!("sanity check count: {}", sanity_count); log::info!("sanity check count: {}", sanity_count);