diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index 42681d0ad..33af420f2 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -254,12 +254,15 @@ pub(crate) struct BasisCache { cache: Vec::, /// ticktimer reference, for managing atimes pub(crate) tt: ticktimer_server::Ticktimer, + /// data cache - stores the most recently decrypted pages of data + data_cache: PlaintextCache, } impl BasisCache { pub(crate) fn new() -> Self { BasisCache { cache: Vec::new(), tt: ticktimer_server::Ticktimer::new().unwrap(), + data_cache: PlaintextCache { data: None, tag: None }, } } /// Returns a Vec which is a list of Bases to visit, in order of visitation, to create the union view. @@ -443,88 +446,103 @@ impl BasisCache { } if let Some(dict_entry) = basis.dicts.get_mut(dict) { if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) { - let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!"); - kcache.set_atime(self.tt.elapsed_ms()); - // the key exists, *and* there's sufficient space for the data - if kcache.start < SMALL_POOL_END { - // small pool fetch - // for now, the rule is, if we have a small key, we also have its data in cache. - if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") { - let mut bytes_read = 0; + let mut needs_fill = false; + loop { // this weird loop works around an interior mutability issue with filling key data. It's awful, like a `goto`. + if needs_fill { + // fetch the data from disk + log::debug!("Small key evicted: filling {}:{}", dict, key); + // this call does a mutable borrow of dict_entry, which interferes with getting kcache below. + dict_entry.refill_small_key(hw, &basis.v2p_map, &basis.cipher, &mut self.data_cache, key); + } + let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!"); + kcache.set_atime(self.tt.elapsed_ms()); + // the key exists, *and* there's sufficient space for the data + if kcache.start < SMALL_POOL_END { + // small pool fetch + if kcache.data.is_none() { + needs_fill = true; + // loop starts again at the top, but this time filling the key first. + continue; + } + // at this point, if we have a small key, we also have its data in cache. + if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") { + let mut bytes_read = 0; + if offset.unwrap_or(0) as u64 > kcache.len { + return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length")); + } + for (&src, dst) in cache_data.data[offset.unwrap_or(0)..kcache.len as usize].iter().zip(data.iter_mut()) { + *dst = src; + bytes_read += 1; + } + if bytes_read != data.len() { + log::debug!("Key shorter than read buffer {}:{} ({}/{})", dict, key, bytes_read, data.len()); + } + return Ok(bytes_read) + } else { + panic!("Key allocated to small area but its cache data was not of the small type"); + } + } else { if offset.unwrap_or(0) as u64 > kcache.len { return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length")); } - for (&src, dst) in cache_data.data[offset.unwrap_or(0)..kcache.len as usize].iter().zip(data.iter_mut()) { - *dst = src; - bytes_read += 1; + if data.len() == 0 { // mostly because i don't want to have to think about this case in the later logic. + return Ok(0) } - if bytes_read != data.len() { - log::debug!("Key shorter than read buffer {}:{} ({}/{})", dict, key, bytes_read, data.len()); - } - return Ok(bytes_read) - } else { - panic!("Key allocated to small area but its cache data was not of the small type"); - } - } else { - if offset.unwrap_or(0) as u64 > kcache.len { - return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length")); - } - if data.len() == 0 { // mostly because i don't want to have to think about this case in the later logic. - return Ok(0) - } - // large pool fetch - let mut abs_cursor = offset.unwrap_or(0) as u64; - let mut blocks_read = 0; - let mut bytes_read = 0; - loop { - let start_vpage_addr = ((kcache.start + abs_cursor) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64; + // large pool fetch + let mut abs_cursor = offset.unwrap_or(0) as u64; + let mut blocks_read = 0; + let mut bytes_read = 0; + loop { + let start_vpage_addr = ((kcache.start + abs_cursor) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64; - if let Some(pp) = basis.v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()) { - let block_start_pos = (abs_cursor % VPAGE_SIZE as u64) as usize; - assert!(pp.valid(), "v2p returned an invalid page"); - let pt_data = hw.data_decrypt_page(&basis.cipher, &basis.aad, pp).expect("Decryption auth error"); - if blocks_read != 0 { - assert!(block_start_pos == 0, "algorithm error in handling offset data"); - } - if blocks_read == 0 { - log::debug!("reading {} abs: {}, block_start: {}, block: {}, data.len:{} kcache.len:{}", - key, abs_cursor, block_start_pos, blocks_read, data.len(), kcache.len); - } else { - log::debug!(" reading {} abs: {}, block_start: {}, block: {}, remaining.data:{} kcache.len:{}", - key, abs_cursor, block_start_pos, blocks_read, data.len() - bytes_read, kcache.len); - } - let data_offset = bytes_read; - for (&src, dst) in - pt_data[ - size_of::() // always this fixed offset per block - + block_start_pos - .. - ].iter().zip(data[data_offset..].iter_mut()) { - *dst = src; - // it'd be computationally more efficient to figure out what this should be going into - // every copy loop, but it's logically easier to think about in this form. Without this - // check, a user could read past the allocated space for a block... - if abs_cursor >= kcache.len { - break; + if let Some(pp) = basis.v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()) { + let block_start_pos = (abs_cursor % VPAGE_SIZE as u64) as usize; + assert!(pp.valid(), "v2p returned an invalid page"); + let pt_data = hw.data_decrypt_page(&basis.cipher, &basis.aad, pp).expect("Decryption auth error"); + if blocks_read != 0 { + assert!(block_start_pos == 0, "algorithm error in handling offset data"); } - abs_cursor += 1; - bytes_read += 1; + if blocks_read == 0 { + log::debug!("reading {} abs: {}, block_start: {}, block: {}, data.len:{} kcache.len:{}", + key, abs_cursor, block_start_pos, blocks_read, data.len(), kcache.len); + } else { + log::debug!(" reading {} abs: {}, block_start: {}, block: {}, remaining.data:{} kcache.len:{}", + key, abs_cursor, block_start_pos, blocks_read, data.len() - bytes_read, kcache.len); + } + let data_offset = bytes_read; + for (&src, dst) in + pt_data[ + size_of::() // always this fixed offset per block + + block_start_pos + .. + ].iter().zip(data[data_offset..].iter_mut()) { + *dst = src; + // it'd be computationally more efficient to figure out what this should be going into + // every copy loop, but it's logically easier to think about in this form. Without this + // check, a user could read past the allocated space for a block... + if abs_cursor >= kcache.len { + break; + } + abs_cursor += 1; + bytes_read += 1; + } + blocks_read += 1; + } else { + log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, abs_cursor, data.len()); + return Ok(bytes_read as usize) + } + if abs_cursor >= kcache.len { + break; + } + if bytes_read >= data.len() { + break; } - blocks_read += 1; - } else { - log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, abs_cursor, data.len()); - return Ok(bytes_read as usize) - } - if abs_cursor >= kcache.len { - break; - } - if bytes_read >= data.len() { - break; } + return Ok(bytes_read as usize) } - return Ok(bytes_read as usize) + // note that from this point forward, either the "if" or the "else" branch returns; + // the end of this loop is never seen, unless a `continue` statement is hit within the loop. } - } else { return Err(Error::new(ErrorKind::NotFound, "key not found")); } diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 8156a7c90..28e45b81d 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -368,6 +368,37 @@ impl DictCacheEntry { panic!("consistency error"); } } + /// To be called only to re-fill key entries whose data have been evicted by a prune. + pub (crate) fn refill_small_key(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv, + data_cache: &mut PlaintextCache, key_name: &str) { + if let Some(kcache) = self.keys.get_mut(key_name) { + if let Some(pool_index) = small_storage_index_from_key(&kcache, self.index) { + let ksp = &mut self.small_pool[pool_index]; + assert!(ksp.contents.contains(&key_name.to_string()), "refill called on a non-existent key. This is an illegal state."); + // now grab the *data* referred to by this key. Maybe this is a "bad" idea -- this can really eat up RAM fast to hold + // all the small pool data right away, but let's give it a try and see how it works. Later on we can always skip this. + // manage a separate small cache for data blocks, under the theory that small keys tend to be packed together + let data_vaddr = small_storage_base_vaddr_from_indices(self.index, pool_index); + data_cache.fill(hw, v2p_map, cipher, &self.aad, VirtAddr::new(data_vaddr).unwrap()); + if let Some(page) = data_cache.data.as_ref() { + let start_offset = size_of::() + (kcache.start % VPAGE_SIZE as u64) as usize; + let mut data = page[start_offset..start_offset + kcache.len as usize].to_vec(); + data.reserve_exact((kcache.reserved - kcache.len) as usize); + kcache.data = Some(KeyCacheData::Small( + KeySmallData { + clean: true, + data + } + )); + } else { + log::error!("Key {}'s data region at pp: {:x?} va: {:x} is unreadable", key_name, data_cache.tag, kcache.start); + } + } + } else { + log::error!("refill_small_key() can only be called after the key cache entry has been allocated"); + panic!("consistency error"); + } + } /// Update a key entry. If the key does not already exist, it will create a new one. /// /// Assume: the caller has called ensure_fast_space_alloc() to make sure there is sufficient space for the inserted key before calling. @@ -835,6 +866,7 @@ impl DictCacheEntry { /// Observation: given the dictionary index and the small key pool index, we know exactly /// the virtual address of the data pool. pub(crate) fn sync_small_pool(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::, cipher: &Aes256GcmSiv) -> bool { + let mut data_cache = PlaintextCache { data: None, tag: None }; for (index, entry) in self.small_pool.iter_mut().enumerate() { if !entry.clean { let pool_vaddr = VirtAddr::new(small_storage_base_vaddr_from_indices(self.index, index)).unwrap(); @@ -845,7 +877,7 @@ impl DictCacheEntry { let mut ap = hw.try_fast_space_alloc().expect("No free space to allocate small key storage"); ap.set_valid(true); ap - }); + }).clone(); assert!(pp.valid(), "v2p returned an invalid page"); // WARNING - we don't read back the journal number before loading data into the page! @@ -876,14 +908,33 @@ impl DictCacheEntry { kcache.clean = false; kcache.flags.set_unresolved(false); kcache.flags.set_valid(true); - if let Some(KeyCacheData::Small(data)) = kcache.data.as_mut() { - data.clean = true; - for (&src, dst) in data.data.iter() - .zip(page[size_of::() + pool_offset..size_of::() + pool_offset + kcache.reserved as usize].iter_mut()) - {*dst = src;} - } else { - // we have a rule that all small keys, when cached, also carry their data: there should not be an index without data. - panic!("Incorrect data cache type for small key entry."); + match kcache.data.as_mut() { + Some(KeyCacheData::Small(data)) => { + data.clean = true; + page[size_of::() + pool_offset .. + size_of::() + pool_offset + kcache.len as usize] + .copy_from_slice(&data.data[..kcache.len as usize]); + } + None => { + // the entry was pruned and must be loaded + // fill() will only do the expensive decryption operation once per page, and refer to the cached value on other calls + log::debug!("Small key sync filling {}", key_name); + data_cache.fill(hw, v2p_map, cipher, &self.aad, VirtAddr::new(pool_vaddr.get()).unwrap()); + if let Some(old_page) = data_cache.data.as_ref() { + let start_offset = size_of::() + (kcache.start % VPAGE_SIZE as u64) as usize; + // the delta of len-reserved bytes is assumed zeroed by the original space allocator; we just copy that + // here. However, if this assumption is broken, we could have leakage between keys in the reserved area!!! + page[size_of::() + pool_offset .. + size_of::() + pool_offset + kcache.len as usize] + .copy_from_slice(&old_page[start_offset..start_offset + kcache.len as usize]); + } else { + log::error!("Key {}'s data region at pp: {:x?} va: {:x} is unreadable", key_name, data_cache.tag, kcache.start); + } + } + _ => { + // the type returned was large, which is patently incorrect + panic!("Incorrect data cache type for small key entry."); + } } pool_offset += kcache.reserved as usize; } @@ -891,6 +942,8 @@ impl DictCacheEntry { // now commit the sector to disk hw.data_encrypt_and_patch_page(cipher, &self.aad, &mut page, &pp); entry.clean = true; + } else { + log::debug!("Key pool entry was clean, no need to sync: {:?}", entry.contents); } } // we now have a bunch of dirty kcache entries. You should call `dict_sync` shortly after this to synchronize those entries to disk.