fixed a subtle bug in the pte sync algorithm

was mixing deletions and updates and because they are in arbitrary
order due to the hashset structure, sometimes i'd delete an entry
after it was updated.

this is now found and fixed.
This commit is contained in:
bunnie 2021-12-28 06:47:33 +08:00
parent 713c0d6e73
commit 6990513278
6 changed files with 77 additions and 21 deletions

View file

@ -721,6 +721,7 @@ impl BasisCache {
dict: dict.to_string(),
basis: (&basis.name).to_string(),
flags: kcache.flags,
index: kcache.descriptor_index,
})
} else {
return Err(Error::new(ErrorKind::NotFound, "key not found"));
@ -744,6 +745,7 @@ impl BasisCache {
dict: dict.to_string(),
basis: (&basis.name).to_string(),
flags: kcache.flags,
index: kcache.descriptor_index,
})
} else {
return Err(Error::new(ErrorKind::NotFound, "key not found"));
@ -1257,16 +1259,13 @@ impl BasisCacheEntry {
pub(crate) fn pt_sync(&mut self, hw: &mut PddbOs) {
self.last_sync = Some(hw.timestamp_now());
let mut kill_list = Vec::<VirtAddr>::new();
// iterate once to delete old entries
for (&virt, phys) in self.v2p_map.iter_mut() {
if !phys.valid() {
// erase the entry
log::debug!("deleting pte va: {:x?} pa: {:x?}", virt, phys);
kill_list.push(virt);
hw.pt_erase(phys.page_number());
} else if !phys.clean() {
log::debug!("syncing dirty pte va: {:x?} pa: {:x?}", virt, phys);
hw.pt_patch_mapping(virt, phys.page_number(), &self.cipher_ecb);
phys.set_clean(true);
}
}
// have to do this in a second phase due to interior mutability problems doing it inside the first iterator
@ -1275,6 +1274,16 @@ impl BasisCacheEntry {
log::warn!("went to remove PTE from v2p map but it wasn't there: {:x}", kill);
}
}
// iterate a second time to write new entries -- can't do this in a single loop because the
// order of visitation is arbitrary and we can delete after writing an entry if we put these
// two in the same loop!
for (&virt, phys) in self.v2p_map.iter_mut() {
if !phys.clean() {
log::debug!("syncing dirty pte va: {:x?} pa: {:x?}", virt, phys);
hw.pt_patch_mapping(virt, phys.page_number(), &self.cipher_ecb);
phys.set_clean(true);
}
}
}
/// This will sync the named Dictionary header + dirty *key descriptors*. It does not
@ -1309,6 +1318,7 @@ impl BasisCacheEntry {
// this is the virtual page within the dictionary region that we're currently serializing
let mut vpage_num = 0;
let mut loopcheck= 0;
let mut sync_count = 0;
loop {
loopcheck += 1;
if loopcheck == 256 {
@ -1321,6 +1331,9 @@ impl BasisCacheEntry {
ap.set_valid(true);
ap
});
//if name.contains("dict2") {
// log::debug!("TRACING: {}/{} | {:x?}", name, vpage_num, pp);
//}
assert!(pp.valid(), "v2p returned an invalid page");
// 2(a). fill in the target vpage with data: header special case
@ -1343,6 +1356,12 @@ impl BasisCacheEntry {
// the assumption that the KeyCacheEntry doesn't ever get to a very large N.
let next_vpage = VirtAddr::new(cur_vpage.get() + VPAGE_SIZE as u64).unwrap();
for (key_name, key) in dict.keys.iter_mut() {
/*if key_name.contains("dict2|key6|len2347") {
log::warn!("TRACING: {}", key_name);
log::warn!("start: {:x}, clean: {}, flags: {:?}, desciptor: {:?}",
key.start, key.clean, key.flags, key.descriptor_index,
);
}*/
if !key.clean && key.flags.valid() {
if key.descriptor_vaddr(dict_offset) >= cur_vpage &&
key.descriptor_vaddr(dict_offset) < next_vpage {
@ -1403,8 +1422,10 @@ impl BasisCacheEntry {
}
}
if !found_next {
log::debug!("breaking after syncing {} keys", sync_count);
break;
}
sync_count += 1;
}
log::debug!("done syncing dict");
dict.clean = true;
@ -1479,8 +1500,10 @@ impl BasisCacheEntry {
// because otherwise we borrow self as immutable to enumerate the names, and then
// we borrow it as mutable to do the sync. This deep-copy works around the issue.
let mut dictnames = Vec::<String>::new();
for dict in self.dicts.keys() {
dictnames.push(dict.to_string());
for (dict, entry) in self.dicts.iter() {
if entry.flags.valid() {
dictnames.push(dict.to_string());
}
}
for dict in dictnames {
match self.dict_sync(hw, &dict) {

View file

@ -633,6 +633,8 @@ impl DictCacheEntry {
// 1. allocate all the pages up to the reservation limit
for vpage_addr in (large_alloc_ptr.as_u64()..large_alloc_ptr.as_u64() + reservation.as_u64()).step_by(VPAGE_SIZE) {
let pp = hw.try_fast_space_alloc().expect("out of disk space");
assert!(pp.valid(), "didn't receive a valid page in large space alloc");
log::debug!("pp alloc v{:x}->p{:x?}", vpage_addr, pp);
v2p_map.insert(VirtAddr::new(vpage_addr).unwrap(), pp);
}
// 2. Recurse. Now, the key should exist, and it should go through the "write the data out" section of the algorithm.

View file

@ -191,4 +191,6 @@ pub(crate) struct KeyAttributes {
pub(crate) basis: String,
/// flags
pub(crate) flags: KeyFlags,
/// descriptor index
pub(crate) index: NonZeroU32,
}

View file

@ -463,12 +463,14 @@ fn xmain() -> ! {
log::info!("Doing delete/add consistency with data extension");
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, None,
None, None, None, None);
log::set_max_level(log::LevelFilter::Debug);
log::info!("Saving `dachecke` to local host");
pddb_os.dbg_dump(Some("dachecke".to_string()), None);
log::info!("Doing patch test");
patch_test(&mut pddb_os, &mut basis_cache, None, None, true);
pddb_os.dbg_dump(Some("patche".to_string()), None);
log::set_max_level(log::LevelFilter::Info);
log::info!("Doing delete pattern test");
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
@ -485,14 +487,11 @@ fn xmain() -> ! {
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
pddb_os.dbg_dump(Some("patterne2".to_string()), None);
log::set_max_level(log::LevelFilter::Debug);
// dict2|key6|len794
log::info!("Doing delete/add consistency with data extension 2");
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, Some(3),
Some(50), None, None, None);
log::info!("Saving `dachecke2` to local host");
pddb_os.dbg_dump(Some("dachecke2".to_string()), None);
log::set_max_level(log::LevelFilter::Info);
log::info!("Doing delete/add consistency with data extension 3");
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, Some(3),

View file

@ -176,6 +176,7 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache,
}
}
log::debug!("bef readback");
// now fix the CI checksum. structured as two separate patches. not because it's efficient,
// but because it exercises the code harder.
let mut patchbuf = [0u8; UPPER_BOUND];

View file

@ -206,6 +206,11 @@ def main():
for bdict in basis_dicts.values():
logging.debug(bdict.as_str())
#d2 = basis_dicts["dict2"]
#logging.info("listing {} keys".format(len(d2.keys)))
#for key in d2.keys:
# logging.info("{}".format(key))
logging.info("CI checks:")
for bdict in basis_dicts.values():
bdict.ci_check()
@ -250,7 +255,12 @@ class KeyDescriptor:
read_start = self.start % VPAGE_SIZE # reads can start not page-aligned
while page_addr < self.start + self.len:
data_base = (page_addr // VPAGE_SIZE) * VPAGE_SIZE
pp_start = v2p[data_base]
try:
pp_start = v2p[data_base]
except KeyError:
logging.error("key {} is missing data allocation at va {:x}".format(self.name, data_base))
self.ci_ok = False
raise KeyError
pp_data = disk[pp_start:pp_start + PAGE_SIZE]
try:
cipher = AES_GCM_SIV(key, pp_data[:12])
@ -362,31 +372,50 @@ class BasisDicts:
keys_tried = 1
while (keys_found < self.num_keys) and keys_tried < 131071:
keyindex_start_vaddr = self.DICT_VSTRIDE * (index + 1) + (keys_tried * self.KV_STRIDE)
# print('ki_vaddr {:x}'.format(keyindex_start_vaddr))
#if self.name == 'dict2' and keyindex_start_vaddr < (0x1fc2fa0 + 0xfe0):
# try:
# print('keys_tried {}'.format(keys_tried))
# print('ki_vaddr {:x}'.format((keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE))
# pp = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE]
# print('ki_paddr {:x}'.format(pp))
# except KeyError:
# pass
try:
pp_start = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE]
pp_data = disk[pp_start:pp_start + PAGE_SIZE]
except KeyError:
# the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried"
keys_tried += VPAGE_SIZE // self.KV_STRIDE
continue
try:
cipher = AES_GCM_SIV(key, pp_data[:12])
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE
#if self.name == 'dict2':
# print('key_index {:x}/{}/{:x}/{:x}'.format(key_index, keys_tried, pp, (keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE))
# print('{:x}'.format(v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE]))
try:
cipher = AES_GCM_SIV(key, pp_data[:12])
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE
#print('key_index {:x}'.format(key_index))
maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name)
if maybe_key.valid:
self.keys[maybe_key.name] = maybe_key
keys_found += 1
#if self.name == 'dict2':
# print('found {}: {}'.format(keys_found, maybe_key.name))
except KeyError:
logging.error("Key @ offset {} is missing data allocation".format(key_index))
except ValueError:
logging.error("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr, pp_start))
keys_tried += 1
except KeyError:
# the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried"
keys_tried += VPAGE_SIZE // self.KV_STRIDE
except ValueError:
logging.error("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr, pp_start))
keys_tried += 1
if keys_found < self.num_keys:
logging.error("Expected {} keys but only found {}".format(self.num_keys, keys_found))
self.found_all_keys = False
else:
self.found_all_keys = True
#if self.name == 'dict2':
# for v, p in v2p.items():
# print('map: v{:16x} | p{:8x}'.format(v, p))
else:
self.valid = False