mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-17 08:40:52 +00:00
debugging the patch-extend test.
probably should make a bunch of unit tests for the free dictionary key tracking logic, it's too tricky to test out.
This commit is contained in:
parent
800fcbe3bf
commit
8de0d70f43
5 changed files with 132 additions and 61 deletions
|
|
@ -531,7 +531,7 @@ impl BasisCache {
|
|||
// index exists, see if the page exists
|
||||
let key_index = (((kcache.start - SMALL_POOL_START as u64) - (dict_entry.index as u64 * SMALL_POOL_STRIDE as u64)) / SMALL_CAPACITY as u64) as usize;
|
||||
if dict_entry.small_pool.len() > key_index {
|
||||
log::info!("resolved key index {}, small pool len: {}", key_index, dict_entry.small_pool.len());
|
||||
log::trace!("resolved key index {}, small pool len: {}", key_index, dict_entry.small_pool.len());
|
||||
// see if the pool's address exists in the page table
|
||||
let pool_vaddr = VirtAddr::new(dict_entry.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START).unwrap();
|
||||
if !basis.v2p_map.contains_key(&pool_vaddr) {
|
||||
|
|
@ -1033,10 +1033,10 @@ impl BasisCacheEntry {
|
|||
|
||||
// 3. merge the vpage modifications into the disk
|
||||
let mut page = if let Some(data) = hw.data_decrypt_page(&self.cipher, &self.aad, &pp) {
|
||||
log::info!("merging dictionary data into existing page");
|
||||
log::trace!("merging dictionary data into existing page");
|
||||
data
|
||||
} else {
|
||||
log::info!("existing data invalid, creating a new page");
|
||||
log::trace!("existing data invalid, creating a new page");
|
||||
// the existing data was invalid (this happens e.g. on the first time a dict is created). Just overwrite the whole page.
|
||||
vec![0u8; VPAGE_SIZE + size_of::<JournalType>()]
|
||||
};
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ impl DictCacheEntry {
|
|||
my_aad.push(b);
|
||||
}
|
||||
let mut free_keys = BinaryHeap::<FreeKeyRange>::new();
|
||||
free_keys.push(FreeKeyRange{start: dict.free_key_index, run: KEY_MAXCOUNT as u32 - 1});
|
||||
free_keys.push(FreeKeyRange{start: dict.free_key_index, run: KEY_MAXCOUNT as u32 - 1 - dict.free_key_index});
|
||||
DictCacheEntry {
|
||||
index: index as u32,
|
||||
keys: HashMap::<String, KeyCacheEntry>::new(),
|
||||
|
|
@ -102,7 +102,7 @@ impl DictCacheEntry {
|
|||
|
||||
if index_cache.data.is_none() || index_cache.tag.is_none() {
|
||||
// somehow we hit a page where nothing was allocated (perhaps it was previously deleted?), or less likely, the data was corrupted. Note the isuse, skip past it.
|
||||
log::warn!("Dictionary fill op encountered an unallocated page checking entry {} in the dictionary map. Marking it for re-use.", try_entry);
|
||||
log::warn!("Dictionary fill: encountered unallocated page at {} in the dictionary map. {}/{}", try_entry, key_count, self.key_count);
|
||||
try_entry += DK_PER_VPAGE;
|
||||
} else {
|
||||
let cache_pp = index_cache.tag.as_ref().expect("PP should be in existence, it was already checked...");
|
||||
|
|
@ -321,10 +321,15 @@ impl DictCacheEntry {
|
|||
for (&src, dst) in data.iter().zip(cache_data.data[offset..].iter_mut()) {
|
||||
*dst = src;
|
||||
}
|
||||
// for now, we ignore "truncate" on a small key
|
||||
} else {
|
||||
panic!("Key allocated to small area but its cache data was not of the small type");
|
||||
}
|
||||
// check if we grew the length
|
||||
if kcache.len < (data.len() + offset) as u64 {
|
||||
kcache.len = (data.len() + offset) as u64;
|
||||
} else if truncate {
|
||||
kcache.len = (data.len() + offset) as u64;
|
||||
}
|
||||
// mark the storage pool entry as dirty, too.
|
||||
let pool_index = small_storage_index_from_key(&kcache, self.index).expect("index missing");
|
||||
self.small_pool[pool_index].clean = false;
|
||||
|
|
@ -396,10 +401,13 @@ impl DictCacheEntry {
|
|||
}
|
||||
}
|
||||
}
|
||||
log::info!("data written: {}, data requested to write: {}", written, data.len());
|
||||
log::trace!("data written: {}, data requested to write: {}", written, data.len());
|
||||
assert!(written == data.len(), "algorithm problem -- didn't write all the data we thought we would");
|
||||
// 3. truncate.
|
||||
if truncate {
|
||||
// 3. truncate or extend
|
||||
// check if we grew the length; extend the length by exactly enough if so.
|
||||
if kcache.len < (data.len() + offset) as u64 {
|
||||
kcache.len = (data.len() + offset) as u64;
|
||||
} else if truncate {
|
||||
// discard all whole pages after written+offset, and reset the reserved field to the smaller size.
|
||||
let vpage_end_offset = PageAlignedVa::from((written + offset) as u64);
|
||||
if (vpage_end_offset.as_u64() - kcache.start) > kcache.reserved {
|
||||
|
|
@ -410,6 +418,7 @@ impl DictCacheEntry {
|
|||
}
|
||||
kcache.reserved = vpage_end_offset.as_u64() - kcache.start;
|
||||
kcache.clean = false;
|
||||
kcache.len = (data.len() + offset) as u64;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -462,6 +471,9 @@ impl DictCacheEntry {
|
|||
for &b in data {
|
||||
alloc_data.push(b);
|
||||
}
|
||||
for fk in self.free_keys.iter() {
|
||||
log::info!("fk start: {} run: {}", fk.start, fk.run);
|
||||
}
|
||||
let descriptor_index = if let Some(di) = self.get_free_key_index() {
|
||||
di
|
||||
} else {
|
||||
|
|
@ -715,41 +727,51 @@ impl DictCacheEntry {
|
|||
// - the new key is adjacent to exactly once element, in which case we put it either on the top or bottom (merge into existing record)
|
||||
// - the new key is adjacent to exactly two elements, in which case we merge the new key and other two elements together, add its length to the new overall run
|
||||
let mut skip = false;
|
||||
let mut placed = false;
|
||||
log::info!("inserting free key {}", index);
|
||||
for i in 0..free_key_vec.len() {
|
||||
if skip {
|
||||
// this happens when we merged into the /next/ record, and we reduced the total number of items by one
|
||||
skip = false;
|
||||
continue
|
||||
}
|
||||
match free_key_vec[i].compare_to(i as u32) {
|
||||
FreeKeyCases::LessThan => {
|
||||
self.free_keys.push(FreeKeyRange{start: index as u32, run: 0});
|
||||
break;
|
||||
}
|
||||
FreeKeyCases::LeftAdjacent => {
|
||||
self.free_keys.push(FreeKeyRange{start: index as u32, run: free_key_vec[i].run + 1});
|
||||
}
|
||||
FreeKeyCases::Within => {
|
||||
log::error!("Double-free error in free_keys()");
|
||||
panic!("Double-free error in free_keys()");
|
||||
}
|
||||
FreeKeyCases::RightAdjacent => {
|
||||
// see if we should merge to the right
|
||||
if i + 1 < free_key_vec.len() {
|
||||
if free_key_vec[i+1].compare_to(i as u32) == FreeKeyCases::LeftAdjacent {
|
||||
self.free_keys.push(FreeKeyRange{
|
||||
start: free_key_vec[i].start,
|
||||
run: free_key_vec[i].run + free_key_vec[i+1].run + 2
|
||||
});
|
||||
skip = true
|
||||
if !placed {
|
||||
match free_key_vec[i].arg_compared_to_self(index) {
|
||||
FreeKeyCases::LessThan => {
|
||||
self.free_keys.push(FreeKeyRange{start: index as u32, run: 0});
|
||||
placed = true;
|
||||
self.free_keys.push(free_key_vec[i]);
|
||||
}
|
||||
FreeKeyCases::LeftAdjacent => {
|
||||
self.free_keys.push(FreeKeyRange{start: index as u32, run: free_key_vec[i].run + 1});
|
||||
placed = true;
|
||||
}
|
||||
FreeKeyCases::Within => {
|
||||
log::error!("Double-free error in free_keys()");
|
||||
panic!("Double-free error in free_keys()");
|
||||
}
|
||||
FreeKeyCases::RightAdjacent => {
|
||||
// see if we should merge to the right
|
||||
if i + 1 < free_key_vec.len() {
|
||||
if free_key_vec[i+1].arg_compared_to_self(i as u32) == FreeKeyCases::LeftAdjacent {
|
||||
self.free_keys.push(FreeKeyRange{
|
||||
start: free_key_vec[i].start,
|
||||
run: free_key_vec[i].run + free_key_vec[i+1].run + 2
|
||||
});
|
||||
skip = true;
|
||||
placed = true;
|
||||
}
|
||||
} else {
|
||||
self.free_keys.push(FreeKeyRange { start: free_key_vec[i].start, run: free_key_vec[i].run + 1 });
|
||||
placed = true;
|
||||
}
|
||||
} else {
|
||||
self.free_keys.push(FreeKeyRange { start: free_key_vec[i].start, run: free_key_vec[i].run + 1 })
|
||||
}
|
||||
FreeKeyCases::GreaterThan => {
|
||||
self.free_keys.push(free_key_vec[i]);
|
||||
}
|
||||
}
|
||||
FreeKeyCases::GreaterThan => {
|
||||
self.free_keys.push(free_key_vec[i]);
|
||||
}
|
||||
} else {
|
||||
self.free_keys.push(free_key_vec[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -853,14 +875,16 @@ pub(crate) struct FreeKeyRange {
|
|||
pub(crate) run: u32,
|
||||
}
|
||||
impl FreeKeyRange {
|
||||
pub(crate) fn compare_to(&self, index: u32) -> FreeKeyCases {
|
||||
if self.start > 1 && index < self.start - 1 {
|
||||
/// returns the result of "index compared to self", so,
|
||||
/// if the index is smaller than me, the result is LessThan.
|
||||
pub(crate) fn arg_compared_to_self(&self, index: u32) -> FreeKeyCases {
|
||||
if self.start > 1 && (index < (self.start - 1)) {
|
||||
FreeKeyCases::LessThan
|
||||
} else if self.start > 0 && index == self.start - 1 {
|
||||
} else if self.start > 0 && (index == self.start - 1) {
|
||||
FreeKeyCases::LeftAdjacent
|
||||
} else if index >= self.start && index <= self.start + self.run {
|
||||
} else if (index >= self.start) && (index <= (self.start + self.run)) {
|
||||
FreeKeyCases::Within
|
||||
} else if index == self.start + self.run + 1 {
|
||||
} else if index == (self.start + self.run + 1) {
|
||||
FreeKeyCases::RightAdjacent
|
||||
} else {
|
||||
FreeKeyCases::GreaterThan
|
||||
|
|
|
|||
|
|
@ -397,11 +397,13 @@ fn xmain() -> ! {
|
|||
|
||||
// OS-specific PDDB driver
|
||||
let mut pddb_os = PddbOs::new(Rc::clone(&entropy));
|
||||
/*
|
||||
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
||||
{
|
||||
log::info!("Creating `basecase1`");
|
||||
let mut basis_cache = BasisCache::new();
|
||||
create_basis_testcase(&mut pddb_os, &mut basis_cache, None, None, None);
|
||||
create_basis_testcase(&mut pddb_os, &mut basis_cache, None,
|
||||
None, None, None);
|
||||
log::info!("Saving `basecase1` to local host");
|
||||
pddb_os.dbg_dump(Some("basecase1".to_string()));
|
||||
|
||||
|
|
@ -411,10 +413,30 @@ fn xmain() -> ! {
|
|||
pddb_os.dbg_dump(Some("dacheck".to_string()));
|
||||
|
||||
log::info!("Doing patch test");
|
||||
patch_test(&mut pddb_os, &mut basis_cache, None, None);
|
||||
patch_test(&mut pddb_os, &mut basis_cache, None, None, false);
|
||||
pddb_os.dbg_dump(Some("patch".to_string()));
|
||||
log::info!("CI done");
|
||||
}
|
||||
*/
|
||||
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
||||
{
|
||||
pddb_os.test_reset();
|
||||
log::info!("Creating `basecase1e`");
|
||||
let mut basis_cache = BasisCache::new();
|
||||
create_basis_testcase(&mut pddb_os, &mut basis_cache, None,
|
||||
None, None, Some(4));
|
||||
log::info!("Saving `basecase1e` to local host");
|
||||
pddb_os.dbg_dump(Some("basecase1e".to_string()));
|
||||
|
||||
log::info!("Doing delete/add consistency");
|
||||
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, None, None, None);
|
||||
log::info!("Saving `dachecke` to local host");
|
||||
pddb_os.dbg_dump(Some("dachecke".to_string()));
|
||||
|
||||
log::info!("Doing patch test");
|
||||
patch_test(&mut pddb_os, &mut basis_cache, None, None, true);
|
||||
pddb_os.dbg_dump(Some("patche".to_string()));
|
||||
}
|
||||
log::info!("CI done");
|
||||
/*
|
||||
{ // a simple case that could be run directly on the hardware
|
||||
log::info!("Running `manual` test case");
|
||||
|
|
@ -450,7 +472,7 @@ fn xmain() -> ! {
|
|||
/* list of test cases:
|
||||
[done] genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
|
||||
[done] delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
|
||||
in-place update consistency: general integrity then patch all keys with a new test pattern
|
||||
[done] in-place update consistency: general integrity then patch all keys with a new test pattern
|
||||
extend update consistency: general integrity then patch all keys with a longer test pattern
|
||||
key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data.
|
||||
basis search: create basis A, populate with general integrity. create basis B, add test entries.
|
||||
|
|
|
|||
|
|
@ -34,7 +34,9 @@ fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bo
|
|||
}
|
||||
|
||||
pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCache,
|
||||
maybe_num_dicts: Option<usize>, maybe_num_keys: Option<usize>, maybe_key_sizes: Option<(usize, usize)>) {
|
||||
maybe_num_dicts: Option<usize>, maybe_num_keys: Option<usize>, maybe_key_sizes: Option<(usize, usize)>,
|
||||
maybe_extra_reserved: Option<usize>,
|
||||
) {
|
||||
|
||||
hw.pddb_format(false).unwrap();
|
||||
let sys_basis = hw.pddb_mount().expect("couldn't mount system basis");
|
||||
|
|
@ -43,6 +45,7 @@ pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCach
|
|||
let num_dicts = maybe_num_dicts.unwrap_or(4);
|
||||
let num_keys = maybe_num_keys.unwrap_or(34);
|
||||
let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, UPPER_BOUND - 4)); // 4 bytes for the CI checksum
|
||||
let extra_reserved = maybe_extra_reserved.unwrap_or(0);
|
||||
|
||||
// make all the directories first
|
||||
for dictnum in 1..=num_dicts {
|
||||
|
|
@ -57,7 +60,16 @@ pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCach
|
|||
let dictname = format!("dict{}", dictnum);
|
||||
let (keyname, keydata) = gen_key(&dictname, keynum, key_lower_bound, key_upper_bound);
|
||||
// now we're ready to write it out
|
||||
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None, None, None, false).unwrap();
|
||||
if maybe_extra_reserved.is_none() {
|
||||
// we do this slightly funky way instead of just passing Some(0) because we want to test the "None" path explicitly
|
||||
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None,
|
||||
None,
|
||||
None, false).unwrap();
|
||||
} else {
|
||||
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None,
|
||||
Some(keydata.len() + extra_reserved),
|
||||
None, false).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -94,7 +106,7 @@ pub(crate) fn delete_add_dict_consistency(hw: &mut PddbOs, basis_cache: &mut Bas
|
|||
|
||||
/// patch data check
|
||||
pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache,
|
||||
maybe_patch_offset: Option<usize>, maybe_patch_data: Option<String>) {
|
||||
maybe_patch_offset: Option<usize>, maybe_patch_data: Option<String>, extend: bool) {
|
||||
|
||||
let patch_offset = maybe_patch_offset.unwrap_or(5);
|
||||
let patch_data = maybe_patch_data.unwrap_or("patched!".to_string());
|
||||
|
|
@ -114,19 +126,32 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache,
|
|||
// now fix the CI checksum. structured as two separate patches. not because it's efficient,
|
||||
// but because it exercises the code harder.
|
||||
let readlen = basis_cache.key_read(hw, dict, key, &mut patchbuf, Some(0), None).unwrap();
|
||||
// now extend and compute the checksum
|
||||
let mut checkdata = Vec::<u8>::new();
|
||||
for &b in &patchbuf[..readlen-4] {
|
||||
checkdata.push(b);
|
||||
if !extend {
|
||||
// now re-compute the checksum
|
||||
let mut checkdata = Vec::<u8>::new();
|
||||
for &b in &patchbuf[..readlen-4] {
|
||||
checkdata.push(b);
|
||||
}
|
||||
log::trace!("checkdata len: {}", checkdata.len());
|
||||
while checkdata.len() % 4 != 0 {
|
||||
checkdata.push(0);
|
||||
}
|
||||
let checksum = murmur3_32(&checkdata, 0);
|
||||
basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen-4), None, None, false).unwrap();
|
||||
} else {
|
||||
// this gloms a new checksum onto the existing record, adding 4 new bytes.
|
||||
let mut checkdata = Vec::<u8>::new();
|
||||
for &b in &patchbuf {
|
||||
checkdata.push(b);
|
||||
}
|
||||
log::trace!("checkdata len: {}", checkdata.len());
|
||||
while checkdata.len() % 4 != 0 {
|
||||
checkdata.push(0);
|
||||
}
|
||||
let checksum = murmur3_32(&checkdata, 0);
|
||||
basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen), None, None, false).unwrap();
|
||||
}
|
||||
log::trace!("checkdata len: {}", checkdata.len());
|
||||
while checkdata.len() % 4 != 0 {
|
||||
checkdata.push(0);
|
||||
}
|
||||
let checksum = murmur3_32(&checkdata, 0);
|
||||
basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen-4), None, None, false).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -387,10 +387,10 @@ class BasisDicts:
|
|||
for (name, key) in self.keys.items():
|
||||
namecheck = name.split('|')
|
||||
if namecheck[1] != self.name:
|
||||
print(" Key/dict mismatch: {}/{}", key.name, self.name)
|
||||
print(" Key/dict mismatch: {}/{}".format(key.name, self.name))
|
||||
keylen = int(namecheck[3][3:])
|
||||
if keylen != key.len:
|
||||
print(" Key length mismatch: {}/{}", keylen, key.len)
|
||||
print(" Key named len vs disk len mismatch: {}/{}".format(keylen, key.len))
|
||||
if key.ci_ok == False:
|
||||
print(" CI failed on key:")
|
||||
print(key.as_str(' '))
|
||||
|
|
|
|||
Loading…
Reference in a new issue