From f71278b422f4b6e47036b1dd265506786bab8afe Mon Sep 17 00:00:00 2001 From: bunnie Date: Thu, 29 Dec 2022 22:04:55 +0800 Subject: [PATCH] fix issue #109 - refactor key fill/scan to not crash after pruning After a prune, key fill/scan can crash because there is an assumption that data is always resident in the cache for a small key. This code handles the case where the small key data was evicted during a prune, and reads the data back in just for that key on demand. Furthermore, during a `sync` call, it will pull in the key data that corresponds to the page containing just the small key, allowing that page to be re-composed and replaced in memory. caching is used on the decryption of the small key page so the expensive bit of the call should only happen once per small key bank. tested with pddb-ci and manually in hosted mode. A portion of code was re-written to make it clear that only the length of allocated data is copied to/from disk and memory, instead of the reserved space as the previous code would indicate. The reserved space is guaranteed to be zero because the RAM-based page for composing the small key block is zero-ized prior to operations; left comments reminding me of this in the future so I don't accidentally optimize something out and cause data leakage between small keys because I'm allowing the reserved areas to contain stale data. --- services/pddb/src/backend/basis.rs | 162 +++++++++++++----------- services/pddb/src/backend/dictionary.rs | 71 +++++++++-- 2 files changed, 152 insertions(+), 81 deletions(-) diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index 42681d0ad..33af420f2 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -254,12 +254,15 @@ pub(crate) struct BasisCache { cache: Vec::, /// ticktimer reference, for managing atimes pub(crate) tt: ticktimer_server::Ticktimer, + /// data cache - stores the most recently decrypted pages of data + data_cache: PlaintextCache, } impl BasisCache { pub(crate) fn new() -> Self { BasisCache { cache: Vec::new(), tt: ticktimer_server::Ticktimer::new().unwrap(), + data_cache: PlaintextCache { data: None, tag: None }, } } /// Returns a Vec which is a list of Bases to visit, in order of visitation, to create the union view. @@ -443,88 +446,103 @@ impl BasisCache { } if let Some(dict_entry) = basis.dicts.get_mut(dict) { if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) { - let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!"); - kcache.set_atime(self.tt.elapsed_ms()); - // the key exists, *and* there's sufficient space for the data - if kcache.start < SMALL_POOL_END { - // small pool fetch - // for now, the rule is, if we have a small key, we also have its data in cache. - if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") { - let mut bytes_read = 0; + let mut needs_fill = false; + loop { // this weird loop works around an interior mutability issue with filling key data. It's awful, like a `goto`. + if needs_fill { + // fetch the data from disk + log::debug!("Small key evicted: filling {}:{}", dict, key); + // this call does a mutable borrow of dict_entry, which interferes with getting kcache below. + dict_entry.refill_small_key(hw, &basis.v2p_map, &basis.cipher, &mut self.data_cache, key); + } + let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!"); + kcache.set_atime(self.tt.elapsed_ms()); + // the key exists, *and* there's sufficient space for the data + if kcache.start < SMALL_POOL_END { + // small pool fetch + if kcache.data.is_none() { + needs_fill = true; + // loop starts again at the top, but this time filling the key first. + continue; + } + // at this point, if we have a small key, we also have its data in cache. + if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") { + let mut bytes_read = 0; + if offset.unwrap_or(0) as u64 > kcache.len { + return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length")); + } + for (&src, dst) in cache_data.data[offset.unwrap_or(0)..kcache.len as usize].iter().zip(data.iter_mut()) { + *dst = src; + bytes_read += 1; + } + if bytes_read != data.len() { + log::debug!("Key shorter than read buffer {}:{} ({}/{})", dict, key, bytes_read, data.len()); + } + return Ok(bytes_read) + } else { + panic!("Key allocated to small area but its cache data was not of the small type"); + } + } else { if offset.unwrap_or(0) as u64 > kcache.len { return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length")); } - for (&src, dst) in cache_data.data[offset.unwrap_or(0)..kcache.len as usize].iter().zip(data.iter_mut()) { - *dst = src; - bytes_read += 1; + if data.len() == 0 { // mostly because i don't want to have to think about this case in the later logic. + return Ok(0) } - if bytes_read != data.len() { - log::debug!("Key shorter than read buffer {}:{} ({}/{})", dict, key, bytes_read, data.len()); - } - return Ok(bytes_read) - } else { - panic!("Key allocated to small area but its cache data was not of the small type"); - } - } else { - if offset.unwrap_or(0) as u64 > kcache.len { - return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length")); - } - if data.len() == 0 { // mostly because i don't want to have to think about this case in the later logic. - return Ok(0) - } - // large pool fetch - let mut abs_cursor = offset.unwrap_or(0) as u64; - let mut blocks_read = 0; - let mut bytes_read = 0; - loop { - let start_vpage_addr = ((kcache.start + abs_cursor) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64; + // large pool fetch + let mut abs_cursor = offset.unwrap_or(0) as u64; + let mut blocks_read = 0; + let mut bytes_read = 0; + loop { + let start_vpage_addr = ((kcache.start + abs_cursor) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64; - if let Some(pp) = basis.v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()) { - let block_start_pos = (abs_cursor % VPAGE_SIZE as u64) as usize; - assert!(pp.valid(), "v2p returned an invalid page"); - let pt_data = hw.data_decrypt_page(&basis.cipher, &basis.aad, pp).expect("Decryption auth error"); - if blocks_read != 0 { - assert!(block_start_pos == 0, "algorithm error in handling offset data"); - } - if blocks_read == 0 { - log::debug!("reading {} abs: {}, block_start: {}, block: {}, data.len:{} kcache.len:{}", - key, abs_cursor, block_start_pos, blocks_read, data.len(), kcache.len); - } else { - log::debug!(" reading {} abs: {}, block_start: {}, block: {}, remaining.data:{} kcache.len:{}", - key, abs_cursor, block_start_pos, blocks_read, data.len() - bytes_read, kcache.len); - } - let data_offset = bytes_read; - for (&src, dst) in - pt_data[ - size_of::() // always this fixed offset per block - + block_start_pos - .. - ].iter().zip(data[data_offset..].iter_mut()) { - *dst = src; - // it'd be computationally more efficient to figure out what this should be going into - // every copy loop, but it's logically easier to think about in this form. Without this - // check, a user could read past the allocated space for a block... - if abs_cursor >= kcache.len { - break; + if let Some(pp) = basis.v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()) { + let block_start_pos = (abs_cursor % VPAGE_SIZE as u64) as usize; + assert!(pp.valid(), "v2p returned an invalid page"); + let pt_data = hw.data_decrypt_page(&basis.cipher, &basis.aad, pp).expect("Decryption auth error"); + if blocks_read != 0 { + assert!(block_start_pos == 0, "algorithm error in handling offset data"); } - abs_cursor += 1; - bytes_read += 1; + if blocks_read == 0 { + log::debug!("reading {} abs: {}, block_start: {}, block: {}, data.len:{} kcache.len:{}", + key, abs_cursor, block_start_pos, blocks_read, data.len(), kcache.len); + } else { + log::debug!(" reading {} abs: {}, block_start: {}, block: {}, remaining.data:{} kcache.len:{}", + key, abs_cursor, block_start_pos, blocks_read, data.len() - bytes_read, kcache.len); + } + let data_offset = bytes_read; + for (&src, dst) in + pt_data[ + size_of::() // always this fixed offset per block + + block_start_pos + .. + ].iter().zip(data[data_offset..].iter_mut()) { + *dst = src; + // it'd be computationally more efficient to figure out what this should be going into + // every copy loop, but it's logically easier to think about in this form. Without this + // check, a user could read past the allocated space for a block... + if abs_cursor >= kcache.len { + break; + } + abs_cursor += 1; + bytes_read += 1; + } + blocks_read += 1; + } else { + log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, abs_cursor, data.len()); + return Ok(bytes_read as usize) + } + if abs_cursor >= kcache.len { + break; + } + if bytes_read >= data.len() { + break; } - blocks_read += 1; - } else { - log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, abs_cursor, data.len()); - return Ok(bytes_read as usize) - } - if abs_cursor >= kcache.len { - break; - } - if bytes_read >= data.len() { - break; } + return Ok(bytes_read as usize) } - return Ok(bytes_read as usize) + // note that from this point forward, either the "if" or the "else" branch returns; + // the end of this loop is never seen, unless a `continue` statement is hit within the loop. } - } else { return Err(Error::new(ErrorKind::NotFound, "key not found")); } diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 8156a7c90..28e45b81d 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -368,6 +368,37 @@ impl DictCacheEntry { panic!("consistency error"); } } + /// To be called only to re-fill key entries whose data have been evicted by a prune. + pub (crate) fn refill_small_key(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv, + data_cache: &mut PlaintextCache, key_name: &str) { + if let Some(kcache) = self.keys.get_mut(key_name) { + if let Some(pool_index) = small_storage_index_from_key(&kcache, self.index) { + let ksp = &mut self.small_pool[pool_index]; + assert!(ksp.contents.contains(&key_name.to_string()), "refill called on a non-existent key. This is an illegal state."); + // now grab the *data* referred to by this key. Maybe this is a "bad" idea -- this can really eat up RAM fast to hold + // all the small pool data right away, but let's give it a try and see how it works. Later on we can always skip this. + // manage a separate small cache for data blocks, under the theory that small keys tend to be packed together + let data_vaddr = small_storage_base_vaddr_from_indices(self.index, pool_index); + data_cache.fill(hw, v2p_map, cipher, &self.aad, VirtAddr::new(data_vaddr).unwrap()); + if let Some(page) = data_cache.data.as_ref() { + let start_offset = size_of::() + (kcache.start % VPAGE_SIZE as u64) as usize; + let mut data = page[start_offset..start_offset + kcache.len as usize].to_vec(); + data.reserve_exact((kcache.reserved - kcache.len) as usize); + kcache.data = Some(KeyCacheData::Small( + KeySmallData { + clean: true, + data + } + )); + } else { + log::error!("Key {}'s data region at pp: {:x?} va: {:x} is unreadable", key_name, data_cache.tag, kcache.start); + } + } + } else { + log::error!("refill_small_key() can only be called after the key cache entry has been allocated"); + panic!("consistency error"); + } + } /// Update a key entry. If the key does not already exist, it will create a new one. /// /// Assume: the caller has called ensure_fast_space_alloc() to make sure there is sufficient space for the inserted key before calling. @@ -835,6 +866,7 @@ impl DictCacheEntry { /// Observation: given the dictionary index and the small key pool index, we know exactly /// the virtual address of the data pool. pub(crate) fn sync_small_pool(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::, cipher: &Aes256GcmSiv) -> bool { + let mut data_cache = PlaintextCache { data: None, tag: None }; for (index, entry) in self.small_pool.iter_mut().enumerate() { if !entry.clean { let pool_vaddr = VirtAddr::new(small_storage_base_vaddr_from_indices(self.index, index)).unwrap(); @@ -845,7 +877,7 @@ impl DictCacheEntry { let mut ap = hw.try_fast_space_alloc().expect("No free space to allocate small key storage"); ap.set_valid(true); ap - }); + }).clone(); assert!(pp.valid(), "v2p returned an invalid page"); // WARNING - we don't read back the journal number before loading data into the page! @@ -876,14 +908,33 @@ impl DictCacheEntry { kcache.clean = false; kcache.flags.set_unresolved(false); kcache.flags.set_valid(true); - if let Some(KeyCacheData::Small(data)) = kcache.data.as_mut() { - data.clean = true; - for (&src, dst) in data.data.iter() - .zip(page[size_of::() + pool_offset..size_of::() + pool_offset + kcache.reserved as usize].iter_mut()) - {*dst = src;} - } else { - // we have a rule that all small keys, when cached, also carry their data: there should not be an index without data. - panic!("Incorrect data cache type for small key entry."); + match kcache.data.as_mut() { + Some(KeyCacheData::Small(data)) => { + data.clean = true; + page[size_of::() + pool_offset .. + size_of::() + pool_offset + kcache.len as usize] + .copy_from_slice(&data.data[..kcache.len as usize]); + } + None => { + // the entry was pruned and must be loaded + // fill() will only do the expensive decryption operation once per page, and refer to the cached value on other calls + log::debug!("Small key sync filling {}", key_name); + data_cache.fill(hw, v2p_map, cipher, &self.aad, VirtAddr::new(pool_vaddr.get()).unwrap()); + if let Some(old_page) = data_cache.data.as_ref() { + let start_offset = size_of::() + (kcache.start % VPAGE_SIZE as u64) as usize; + // the delta of len-reserved bytes is assumed zeroed by the original space allocator; we just copy that + // here. However, if this assumption is broken, we could have leakage between keys in the reserved area!!! + page[size_of::() + pool_offset .. + size_of::() + pool_offset + kcache.len as usize] + .copy_from_slice(&old_page[start_offset..start_offset + kcache.len as usize]); + } else { + log::error!("Key {}'s data region at pp: {:x?} va: {:x} is unreadable", key_name, data_cache.tag, kcache.start); + } + } + _ => { + // the type returned was large, which is patently incorrect + panic!("Incorrect data cache type for small key entry."); + } } pool_offset += kcache.reserved as usize; } @@ -891,6 +942,8 @@ impl DictCacheEntry { // now commit the sector to disk hw.data_encrypt_and_patch_page(cipher, &self.aad, &mut page, &pp); entry.clean = true; + } else { + log::debug!("Key pool entry was clean, no need to sync: {:?}", entry.contents); } } // we now have a bunch of dirty kcache entries. You should call `dict_sync` shortly after this to synchronize those entries to disk.