debugging the patch-extend test.

probably should make a bunch of unit tests for the free dictionary
key tracking logic, it's too tricky to test out.
This commit is contained in:
bunnie 2021-11-24 03:57:50 +08:00
parent 800fcbe3bf
commit 8de0d70f43
5 changed files with 132 additions and 61 deletions

View file

@ -531,7 +531,7 @@ impl BasisCache {
// index exists, see if the page exists
let key_index = (((kcache.start - SMALL_POOL_START as u64) - (dict_entry.index as u64 * SMALL_POOL_STRIDE as u64)) / SMALL_CAPACITY as u64) as usize;
if dict_entry.small_pool.len() > key_index {
log::info!("resolved key index {}, small pool len: {}", key_index, dict_entry.small_pool.len());
log::trace!("resolved key index {}, small pool len: {}", key_index, dict_entry.small_pool.len());
// see if the pool's address exists in the page table
let pool_vaddr = VirtAddr::new(dict_entry.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START).unwrap();
if !basis.v2p_map.contains_key(&pool_vaddr) {
@ -1033,10 +1033,10 @@ impl BasisCacheEntry {
// 3. merge the vpage modifications into the disk
let mut page = if let Some(data) = hw.data_decrypt_page(&self.cipher, &self.aad, &pp) {
log::info!("merging dictionary data into existing page");
log::trace!("merging dictionary data into existing page");
data
} else {
log::info!("existing data invalid, creating a new page");
log::trace!("existing data invalid, creating a new page");
// the existing data was invalid (this happens e.g. on the first time a dict is created). Just overwrite the whole page.
vec![0u8; VPAGE_SIZE + size_of::<JournalType>()]
};

View file

@ -66,7 +66,7 @@ impl DictCacheEntry {
my_aad.push(b);
}
let mut free_keys = BinaryHeap::<FreeKeyRange>::new();
free_keys.push(FreeKeyRange{start: dict.free_key_index, run: KEY_MAXCOUNT as u32 - 1});
free_keys.push(FreeKeyRange{start: dict.free_key_index, run: KEY_MAXCOUNT as u32 - 1 - dict.free_key_index});
DictCacheEntry {
index: index as u32,
keys: HashMap::<String, KeyCacheEntry>::new(),
@ -102,7 +102,7 @@ impl DictCacheEntry {
if index_cache.data.is_none() || index_cache.tag.is_none() {
// somehow we hit a page where nothing was allocated (perhaps it was previously deleted?), or less likely, the data was corrupted. Note the isuse, skip past it.
log::warn!("Dictionary fill op encountered an unallocated page checking entry {} in the dictionary map. Marking it for re-use.", try_entry);
log::warn!("Dictionary fill: encountered unallocated page at {} in the dictionary map. {}/{}", try_entry, key_count, self.key_count);
try_entry += DK_PER_VPAGE;
} else {
let cache_pp = index_cache.tag.as_ref().expect("PP should be in existence, it was already checked...");
@ -321,10 +321,15 @@ impl DictCacheEntry {
for (&src, dst) in data.iter().zip(cache_data.data[offset..].iter_mut()) {
*dst = src;
}
// for now, we ignore "truncate" on a small key
} else {
panic!("Key allocated to small area but its cache data was not of the small type");
}
// check if we grew the length
if kcache.len < (data.len() + offset) as u64 {
kcache.len = (data.len() + offset) as u64;
} else if truncate {
kcache.len = (data.len() + offset) as u64;
}
// mark the storage pool entry as dirty, too.
let pool_index = small_storage_index_from_key(&kcache, self.index).expect("index missing");
self.small_pool[pool_index].clean = false;
@ -396,10 +401,13 @@ impl DictCacheEntry {
}
}
}
log::info!("data written: {}, data requested to write: {}", written, data.len());
log::trace!("data written: {}, data requested to write: {}", written, data.len());
assert!(written == data.len(), "algorithm problem -- didn't write all the data we thought we would");
// 3. truncate.
if truncate {
// 3. truncate or extend
// check if we grew the length; extend the length by exactly enough if so.
if kcache.len < (data.len() + offset) as u64 {
kcache.len = (data.len() + offset) as u64;
} else if truncate {
// discard all whole pages after written+offset, and reset the reserved field to the smaller size.
let vpage_end_offset = PageAlignedVa::from((written + offset) as u64);
if (vpage_end_offset.as_u64() - kcache.start) > kcache.reserved {
@ -410,6 +418,7 @@ impl DictCacheEntry {
}
kcache.reserved = vpage_end_offset.as_u64() - kcache.start;
kcache.clean = false;
kcache.len = (data.len() + offset) as u64;
}
}
}
@ -462,6 +471,9 @@ impl DictCacheEntry {
for &b in data {
alloc_data.push(b);
}
for fk in self.free_keys.iter() {
log::info!("fk start: {} run: {}", fk.start, fk.run);
}
let descriptor_index = if let Some(di) = self.get_free_key_index() {
di
} else {
@ -715,41 +727,51 @@ impl DictCacheEntry {
// - the new key is adjacent to exactly once element, in which case we put it either on the top or bottom (merge into existing record)
// - the new key is adjacent to exactly two elements, in which case we merge the new key and other two elements together, add its length to the new overall run
let mut skip = false;
let mut placed = false;
log::info!("inserting free key {}", index);
for i in 0..free_key_vec.len() {
if skip {
// this happens when we merged into the /next/ record, and we reduced the total number of items by one
skip = false;
continue
}
match free_key_vec[i].compare_to(i as u32) {
FreeKeyCases::LessThan => {
self.free_keys.push(FreeKeyRange{start: index as u32, run: 0});
break;
}
FreeKeyCases::LeftAdjacent => {
self.free_keys.push(FreeKeyRange{start: index as u32, run: free_key_vec[i].run + 1});
}
FreeKeyCases::Within => {
log::error!("Double-free error in free_keys()");
panic!("Double-free error in free_keys()");
}
FreeKeyCases::RightAdjacent => {
// see if we should merge to the right
if i + 1 < free_key_vec.len() {
if free_key_vec[i+1].compare_to(i as u32) == FreeKeyCases::LeftAdjacent {
self.free_keys.push(FreeKeyRange{
start: free_key_vec[i].start,
run: free_key_vec[i].run + free_key_vec[i+1].run + 2
});
skip = true
if !placed {
match free_key_vec[i].arg_compared_to_self(index) {
FreeKeyCases::LessThan => {
self.free_keys.push(FreeKeyRange{start: index as u32, run: 0});
placed = true;
self.free_keys.push(free_key_vec[i]);
}
FreeKeyCases::LeftAdjacent => {
self.free_keys.push(FreeKeyRange{start: index as u32, run: free_key_vec[i].run + 1});
placed = true;
}
FreeKeyCases::Within => {
log::error!("Double-free error in free_keys()");
panic!("Double-free error in free_keys()");
}
FreeKeyCases::RightAdjacent => {
// see if we should merge to the right
if i + 1 < free_key_vec.len() {
if free_key_vec[i+1].arg_compared_to_self(i as u32) == FreeKeyCases::LeftAdjacent {
self.free_keys.push(FreeKeyRange{
start: free_key_vec[i].start,
run: free_key_vec[i].run + free_key_vec[i+1].run + 2
});
skip = true;
placed = true;
}
} else {
self.free_keys.push(FreeKeyRange { start: free_key_vec[i].start, run: free_key_vec[i].run + 1 });
placed = true;
}
} else {
self.free_keys.push(FreeKeyRange { start: free_key_vec[i].start, run: free_key_vec[i].run + 1 })
}
FreeKeyCases::GreaterThan => {
self.free_keys.push(free_key_vec[i]);
}
}
FreeKeyCases::GreaterThan => {
self.free_keys.push(free_key_vec[i]);
}
} else {
self.free_keys.push(free_key_vec[i]);
}
}
}
@ -853,14 +875,16 @@ pub(crate) struct FreeKeyRange {
pub(crate) run: u32,
}
impl FreeKeyRange {
pub(crate) fn compare_to(&self, index: u32) -> FreeKeyCases {
if self.start > 1 && index < self.start - 1 {
/// returns the result of "index compared to self", so,
/// if the index is smaller than me, the result is LessThan.
pub(crate) fn arg_compared_to_self(&self, index: u32) -> FreeKeyCases {
if self.start > 1 && (index < (self.start - 1)) {
FreeKeyCases::LessThan
} else if self.start > 0 && index == self.start - 1 {
} else if self.start > 0 && (index == self.start - 1) {
FreeKeyCases::LeftAdjacent
} else if index >= self.start && index <= self.start + self.run {
} else if (index >= self.start) && (index <= (self.start + self.run)) {
FreeKeyCases::Within
} else if index == self.start + self.run + 1 {
} else if index == (self.start + self.run + 1) {
FreeKeyCases::RightAdjacent
} else {
FreeKeyCases::GreaterThan

View file

@ -397,11 +397,13 @@ fn xmain() -> ! {
// OS-specific PDDB driver
let mut pddb_os = PddbOs::new(Rc::clone(&entropy));
/*
#[cfg(not(any(target_os = "none", target_os = "xous")))]
{
log::info!("Creating `basecase1`");
let mut basis_cache = BasisCache::new();
create_basis_testcase(&mut pddb_os, &mut basis_cache, None, None, None);
create_basis_testcase(&mut pddb_os, &mut basis_cache, None,
None, None, None);
log::info!("Saving `basecase1` to local host");
pddb_os.dbg_dump(Some("basecase1".to_string()));
@ -411,10 +413,30 @@ fn xmain() -> ! {
pddb_os.dbg_dump(Some("dacheck".to_string()));
log::info!("Doing patch test");
patch_test(&mut pddb_os, &mut basis_cache, None, None);
patch_test(&mut pddb_os, &mut basis_cache, None, None, false);
pddb_os.dbg_dump(Some("patch".to_string()));
log::info!("CI done");
}
*/
#[cfg(not(any(target_os = "none", target_os = "xous")))]
{
pddb_os.test_reset();
log::info!("Creating `basecase1e`");
let mut basis_cache = BasisCache::new();
create_basis_testcase(&mut pddb_os, &mut basis_cache, None,
None, None, Some(4));
log::info!("Saving `basecase1e` to local host");
pddb_os.dbg_dump(Some("basecase1e".to_string()));
log::info!("Doing delete/add consistency");
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, None, None, None);
log::info!("Saving `dachecke` to local host");
pddb_os.dbg_dump(Some("dachecke".to_string()));
log::info!("Doing patch test");
patch_test(&mut pddb_os, &mut basis_cache, None, None, true);
pddb_os.dbg_dump(Some("patche".to_string()));
}
log::info!("CI done");
/*
{ // a simple case that could be run directly on the hardware
log::info!("Running `manual` test case");
@ -450,7 +472,7 @@ fn xmain() -> ! {
/* list of test cases:
[done] genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
[done] delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
in-place update consistency: general integrity then patch all keys with a new test pattern
[done] in-place update consistency: general integrity then patch all keys with a new test pattern
extend update consistency: general integrity then patch all keys with a longer test pattern
key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data.
basis search: create basis A, populate with general integrity. create basis B, add test entries.

View file

@ -34,7 +34,9 @@ fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bo
}
pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCache,
maybe_num_dicts: Option<usize>, maybe_num_keys: Option<usize>, maybe_key_sizes: Option<(usize, usize)>) {
maybe_num_dicts: Option<usize>, maybe_num_keys: Option<usize>, maybe_key_sizes: Option<(usize, usize)>,
maybe_extra_reserved: Option<usize>,
) {
hw.pddb_format(false).unwrap();
let sys_basis = hw.pddb_mount().expect("couldn't mount system basis");
@ -43,6 +45,7 @@ pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCach
let num_dicts = maybe_num_dicts.unwrap_or(4);
let num_keys = maybe_num_keys.unwrap_or(34);
let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, UPPER_BOUND - 4)); // 4 bytes for the CI checksum
let extra_reserved = maybe_extra_reserved.unwrap_or(0);
// make all the directories first
for dictnum in 1..=num_dicts {
@ -57,7 +60,16 @@ pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCach
let dictname = format!("dict{}", dictnum);
let (keyname, keydata) = gen_key(&dictname, keynum, key_lower_bound, key_upper_bound);
// now we're ready to write it out
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None, None, None, false).unwrap();
if maybe_extra_reserved.is_none() {
// we do this slightly funky way instead of just passing Some(0) because we want to test the "None" path explicitly
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None,
None,
None, false).unwrap();
} else {
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None,
Some(keydata.len() + extra_reserved),
None, false).unwrap();
}
}
}
}
@ -94,7 +106,7 @@ pub(crate) fn delete_add_dict_consistency(hw: &mut PddbOs, basis_cache: &mut Bas
/// patch data check
pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache,
maybe_patch_offset: Option<usize>, maybe_patch_data: Option<String>) {
maybe_patch_offset: Option<usize>, maybe_patch_data: Option<String>, extend: bool) {
let patch_offset = maybe_patch_offset.unwrap_or(5);
let patch_data = maybe_patch_data.unwrap_or("patched!".to_string());
@ -114,19 +126,32 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache,
// now fix the CI checksum. structured as two separate patches. not because it's efficient,
// but because it exercises the code harder.
let readlen = basis_cache.key_read(hw, dict, key, &mut patchbuf, Some(0), None).unwrap();
// now extend and compute the checksum
let mut checkdata = Vec::<u8>::new();
for &b in &patchbuf[..readlen-4] {
checkdata.push(b);
if !extend {
// now re-compute the checksum
let mut checkdata = Vec::<u8>::new();
for &b in &patchbuf[..readlen-4] {
checkdata.push(b);
}
log::trace!("checkdata len: {}", checkdata.len());
while checkdata.len() % 4 != 0 {
checkdata.push(0);
}
let checksum = murmur3_32(&checkdata, 0);
basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen-4), None, None, false).unwrap();
} else {
// this gloms a new checksum onto the existing record, adding 4 new bytes.
let mut checkdata = Vec::<u8>::new();
for &b in &patchbuf {
checkdata.push(b);
}
log::trace!("checkdata len: {}", checkdata.len());
while checkdata.len() % 4 != 0 {
checkdata.push(0);
}
let checksum = murmur3_32(&checkdata, 0);
basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen), None, None, false).unwrap();
}
log::trace!("checkdata len: {}", checkdata.len());
while checkdata.len() % 4 != 0 {
checkdata.push(0);
}
let checksum = murmur3_32(&checkdata, 0);
basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen-4), None, None, false).unwrap();
}
}
}
}
}

View file

@ -387,10 +387,10 @@ class BasisDicts:
for (name, key) in self.keys.items():
namecheck = name.split('|')
if namecheck[1] != self.name:
print(" Key/dict mismatch: {}/{}", key.name, self.name)
print(" Key/dict mismatch: {}/{}".format(key.name, self.name))
keylen = int(namecheck[3][3:])
if keylen != key.len:
print(" Key length mismatch: {}/{}", keylen, key.len)
print(" Key named len vs disk len mismatch: {}/{}".format(keylen, key.len))
if key.ci_ok == False:
print(" CI failed on key:")
print(key.as_str(' '))