fix another edge case in key deletion

invalidated keys that were not dirty were not being removed
also added routine to fill keys that are evicted from the cache
during the update routine
This commit is contained in:
bunnie 2023-01-28 19:08:01 +08:00
parent 7cb98699aa
commit 145e0469bc
6 changed files with 86 additions and 16 deletions

View file

@ -245,6 +245,9 @@ pub(crate) enum Opcode {
/// Bulk delete of keys within a dictionary; if non-existent keys are specified, no error is returned.
DictBulkDelete = 55,
/// Prune the cache. Used mainly for diagnostics.
Prune = 56,
/// This key type could not be decoded
InvalidOpcode = u32::MAX as _,
}

View file

@ -1601,6 +1601,7 @@ impl BasisCacheEntry {
// make, but for now, let's do it with a dumb O(N) scan through the KeyCacheEntry, running under
// the assumption that the KeyCacheEntry doesn't ever get to a very large N.
let next_vpage = VirtAddr::new(cur_vpage.get() + VPAGE_SIZE as u64).unwrap();
let mut invalidate = Vec::<String>::new(); // remember which keys to invalidate
for (key_name, key) in dict.keys.iter_mut() {
/*if key_name.contains("dict2|key6|len2347") {
log::warn!("TRACING: {}", key_name);
@ -1631,8 +1632,14 @@ impl BasisCacheEntry {
} else {
log::debug!("proposed key fell outside of our vpage: {} vpage{:x}/vaddr{:x}", key_name, cur_vpage.get(), key.descriptor_vaddr(dict_offset));
}
} else {
invalidate.push(key_name.to_string());
}
}
// remove the invalidated keys from the cache
for key in invalidate {
dict.keys.remove(&key);
}
// 3. merge the vpage modifications into the disk
let mut page = if let Some(data) = hw.data_decrypt_page(&self.cipher, &self.aad, &pp) {
@ -1663,7 +1670,7 @@ impl BasisCacheEntry {
// exit the loop
let mut found_next = false;
for key in dict.keys.values() {
if !key.clean && key.flags.valid() {
if !key.clean && key.flags.valid() || !key.flags.valid() {
found_next = true;
// note: we don't care *which* vpage we do next -- so we just break after finding the first one
vpage_num = key.descriptor_vpage_num();
@ -1676,8 +1683,6 @@ impl BasisCacheEntry {
}
sync_count += 1;
}
// remove invalid keys from the dictionary cache
dict.keys.retain(|_name, entry| entry.flags.valid());
log::debug!("done syncing dict");
dict.clean = true;

View file

@ -239,10 +239,25 @@ impl DictCacheEntry {
/// if cache is hot or key was found on search.
pub(crate) fn ensure_key_entry(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv,
name_str: &str) -> bool {
// only fill if the key isn't in the cache.
if !self.keys.contains_key(name_str) {
let mut data_cache = PlaintextCache { data: None, tag: None };
// if the key isn't valid, evict it now
self.keys.retain(|name, value| (name != name_str) || ((name == name_str) && value.flags.valid()));
// only fill if the key isn't in the cache, or if the data section has been evicted
let needs_fill = if let Some(entry) = self.keys.get(name_str) {
if entry.data.is_none() {
self.try_fill_small_key(hw, v2p_map, cipher, &mut data_cache, name_str);
// it should be filled, so no further processing is needed
false
} else {
// data is already there
false
}
} else {
// entry doesn't exist
true
};
if needs_fill {
log::debug!("searching for key {}", name_str);
let mut data_cache = PlaintextCache { data: None, tag: None };
let mut try_entry = 1;
let mut key_count = 0;
let mut index_cache = PlaintextCache { data: None, tag: None };
@ -496,17 +511,45 @@ impl DictCacheEntry {
kcache.clean = false;
// it's a small key; note that we didn't consult the *size* of the key to determine its pool type:
// small-sized keys can still end up in the "large" space if the small pool allocation is exhausted.
if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") {
cache_data.clean = false;
// grow the data cache to accommodate the necessary length; this should be efficient because we reserved space when the vector was allocated
while cache_data.data.len() < data.len() + offset {
cache_data.data.push(0);
match kcache.data.as_mut() {
Some(KeyCacheData::Small(cache_data)) => {
cache_data.clean = false;
// grow the data cache to accommodate the necessary length; this should be efficient because we reserved space when the vector was allocated
while cache_data.data.len() < data.len() + offset {
cache_data.data.push(0);
}
for (&src, dst) in data.iter().zip(cache_data.data[offset..].iter_mut()) {
*dst = src;
}
}
for (&src, dst) in data.iter().zip(cache_data.data[offset..].iter_mut()) {
*dst = src;
Some(KeyCacheData::Large(_)) => {
panic!("Key allocated to small area but its cache data was not of the small type");
}
None => {
// cache data was evicted, fill it
assert!(kcache.flags.valid(), "Entry to be filled should be valid");
let pool_index = small_storage_index_from_key(&kcache, self.index).expect("pool should be valid");
let ksp = &mut self.small_pool[pool_index];
assert!(ksp.contents.contains(&name.to_string()), "Entry to fill isn't in the expected pool");
let data_vaddr = small_storage_base_vaddr_from_indices(self.index, pool_index);
let mut data_cache = PlaintextCache { data: None, tag: None };
data_cache.fill(hw, v2p_map, cipher, &self.aad, VirtAddr::new(data_vaddr).unwrap());
if let Some(page) = data_cache.data.as_ref() {
let start_offset = size_of::<JournalType>() + (kcache.start % VPAGE_SIZE as u64) as usize;
let mut data = page[start_offset..start_offset + kcache.len as usize].to_vec();
data.reserve_exact((kcache.reserved - kcache.len) as usize);
kcache.data = Some(KeyCacheData::Small(
KeySmallData {
clean: true,
data
}
));
} else {
let error =
format!("Key {}'s data region at pp: {:x?} va: {:x} is unreadable", name, data_cache.tag, kcache.start);
panic!("{}", error);
}
}
} else {
panic!("Key allocated to small area but its cache data was not of the small type");
}
// check if we grew the length
if kcache.len < (data.len() + offset) as u64 {
@ -1090,6 +1133,9 @@ impl DictCacheEntry {
if kcache.data.is_some() {
let pruned = kcache.size();
kcache.data.take(); // this effectively frees up the key cache data
// mark the key's pool as unclean, so it is processed for filling
let pool_index = small_storage_index_from_key(&kcache, self.index).expect("index missing");
self.small_pool[pool_index].clean = false;
pruned
} else {
0

View file

@ -856,6 +856,13 @@ impl Pddb {
Message::new_blocking_scalar(Opcode::FlushSpaceUpdate.to_usize().unwrap(), 0, 0, 0, 0)
).expect("couldn't send FlushSpaceUpdate");
}
/// Manually prune the PDDB cache.
/// Mostly provided for force-triggering for testing; normally this is done automatically
pub fn manual_prune(&self) {
send_message(self.conn,
Message::new_blocking_scalar(Opcode::Prune.to_usize().unwrap(), 0, 0, 0, 0)
).expect("couldn't send FlushSpaceUpdate");
}
/// Rekey the PDDB. This can be a very long-running blocking operation that will definitely.
/// interrupt normal user flow.
pub fn rekey_pddb(&self, op: PddbRekeyOp) -> Result<()> {

View file

@ -1838,6 +1838,13 @@ fn wrapped_main() -> ! {
pddb_os.reset_dont_ask_init();
xous::return_scalar(msg.sender, 1).ok();
}
Opcode::Prune => {
log::info!("PDDB prune manual request: {} heap, {} cache", latest_heap, basis_cache.cache_size());
let pruned = basis_cache.cache_prune(&mut pddb_os, HEAP_GC_TARGET);
latest_heap = heap_usage();
log::info!("{} pruned, now: {} heap, {} cache", pruned, latest_heap, basis_cache.cache_size());
xous::return_scalar(msg.sender, 1).ok();
}
#[cfg(not(target_os = "xous"))]
Opcode::DangerousDebug => {
let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) };

View file

@ -1135,9 +1135,11 @@ impl<'a> ShellCmdApi<'a> for PddbCmd {
}
}
}
#[cfg(not(target_os = "xous"))]
"prune" => {
#[cfg(not(target_os = "xous"))]
self.pddb.dbg_prune().ok();
#[cfg(target_os = "xous")]
self.pddb.manual_prune();
write!(ret, "Prune finished").ok();
}
#[cfg(not(target_os = "xous"))]