From 42e926fcab7e6bb5a54e49b2fa01c886c40af1d8 Mon Sep 17 00:00:00 2001 From: bunnie Date: Tue, 23 Nov 2021 23:33:06 +0800 Subject: [PATCH] fixes for patch update test lots of corner cases found and ifxed --- services/pddb/src/backend/basis.rs | 97 ++++++++++++++++++++++--- services/pddb/src/backend/dictionary.rs | 20 ++++- services/pddb/src/backend/key.rs | 14 ++++ services/pddb/src/tests.rs | 20 ++++- tools/pddbdbg.py | 26 +++++-- 5 files changed, 156 insertions(+), 21 deletions(-) diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index bc194967e..a87546a4f 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -368,40 +368,82 @@ impl BasisCache { // for now, the rule is, if we have a small key, we also have its data in cache. if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") { let mut bytes_read = 0; - for (&src, dst) in cache_data.data[offset.unwrap_or(0)..].iter().zip(data.iter_mut()) { + if offset.unwrap_or(0) as u64 > kcache.len { + return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length")); + } + for (&src, dst) in cache_data.data[offset.unwrap_or(0)..kcache.len as usize].iter().zip(data.iter_mut()) { *dst = src; bytes_read += 1; } if bytes_read != data.len() { - log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, bytes_read, data.len()); + log::debug!("Key shorter than read buffer {}:{} ({}/{})", dict, key, bytes_read, data.len()); } return Ok(bytes_read) } else { panic!("Key allocated to small area but its cache data was not of the small type"); } } else { + if offset.unwrap_or(0) as u64 > kcache.len { + return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length")); + } + if data.len() == 0 { // mostly because i don't want to have to think about this case in the later logic. + return Ok(0) + } // large pool fetch let mut cur_offset = offset.unwrap_or(0) as u64; - let mut first_iter = true; - while cur_offset < data.len() as u64 { + let mut blocks_read = 0; + loop { let start_vpage_addr = ((kcache.start + cur_offset) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64; + // handle the case of terminating the last block read nicely: + // we can either end because we hit the end of the key data (so the data readback buffer is too large) + let key_endstop = if (kcache.len - cur_offset) < VPAGE_SIZE as u64 { + (kcache.len - cur_offset) as usize + size_of::() + } else { + VPAGE_SIZE + size_of::() + }; + // or we can hit the end because we ran out of datareadback buffer + let data_endstop = if (data.len() as u64 - cur_offset) < VPAGE_SIZE as u64 { + (data.len() as u64 - cur_offset) as usize + size_of::() + } else { + VPAGE_SIZE + size_of::() + }; + // of the two possible end criteria, pick the nearest one + let endstop = if key_endstop < data_endstop { + key_endstop + } else { + data_endstop + }; + log::trace!("reading offset {}/{}:{}", cur_offset, endstop, kcache.len); + let mut read_offset = (cur_offset % VPAGE_SIZE as u64) as usize; if let Some(pp) = basis.v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()) { let pt_data = hw.data_decrypt_page(&basis.cipher, &basis.aad, pp).expect("Decryption auth error"); - if !first_iter { - assert!((cur_offset as usize % VPAGE_SIZE) == 0, "algorithm error in handling offset data"); + if blocks_read != 0 { + assert!(read_offset == 0, "algorithm error in handling offset data"); } - // note: cur_offset % VPAGE_SIZE handles mis-aligned starts; cur_offset will increment until start + cur_offset is - // a multiple of a VPAGE_SIZE, so on later iterations, cur_offset % VPAGE_SIZE should be equal to 0. for (&src, dst) in - pt_data[size_of::() + cur_offset as usize % VPAGE_SIZE..].iter().zip(data.iter_mut()) { + pt_data[ + size_of::() // always this fixed offset per block + + read_offset // this should be 0 after the first block + ..endstop + ].iter().zip(data[cur_offset as usize..].iter_mut()) { *dst = src; cur_offset += 1; + read_offset += 1; } - first_iter = false; + blocks_read += 1; } else { log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, cur_offset, data.len()); return Ok(cur_offset as usize) } + // cur_offset won't hit (start + len) if we're hitting an endstop, so the loop won't exit. + // if we hit an endstop, we should exit the read loop. + if endstop != VPAGE_SIZE + size_of::() { + break; + } + // break if we've hit or exceeded our endstop + if read_offset >= endstop { + break; + } } return Ok(cur_offset as usize) } @@ -581,6 +623,31 @@ impl BasisCache { Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) } } + + pub(crate) fn key_attributes(&mut self, hw: &mut PddbOs, dict: &str, key: &str, basis_name: Option<&str>) -> Result { + if let Some(basis_index) = self.select_basis(basis_name) { + let basis = &mut self.cache[basis_index]; + if !basis.ensure_dict_in_cache(hw, dict) { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); + } else { + let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!"); + if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) { + let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!"); + Ok(KeyAttributes { + len: kcache.len as usize, + reserved: kcache.reserved as usize, + age: kcache.age as usize, + dict: dict.to_string(), + basis: (&basis.name).to_string(), + }) + } else { + return Err(Error::new(ErrorKind::NotFound, "key not found")); + } + } + } else { + Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) + } + } } /// This is the RAM cached copy of a basis as maintained in the PDDB. @@ -892,7 +959,12 @@ impl BasisCacheEntry { // this is the virtual page within the dictionary region that we're currently serializing let mut vpage_num = 0; + let mut loopcheck= 0; loop { + loopcheck += 1; + if loopcheck == 256 { + log::warn!("potential infinite loop detected sync dict {}", name); + } // 1. resolve the virtual address to a target page let cur_vpage = VirtAddr::new(dict_offset.get() + (vpage_num as u64 * VPAGE_SIZE as u64)).unwrap(); let pp = self.v2p_map.entry(cur_vpage).or_insert_with(|| @@ -919,9 +991,9 @@ impl BasisCacheEntry { let next_vpage = VirtAddr::new(cur_vpage.get() + VPAGE_SIZE as u64).unwrap(); for (key_name, key) in dict.keys.iter_mut() { if !key.clean { - log::info!("merging in key {}", key_name); if key.descriptor_vaddr(dict_offset) >= cur_vpage && key.descriptor_vaddr(dict_offset) < next_vpage { + log::info!("merging in key {}", key_name); // key is within the current page, add it to the target list let mut dk_entry = DictKeyEntry::default(); let mut kn = [0u8; KEY_NAME_LEN]; @@ -941,6 +1013,8 @@ impl BasisCacheEntry { } dk_vpage.elements[key.descriptor_index.get() as usize % DK_PER_VPAGE] = Some(dk_entry); key.clean = true; + } else { + log::debug!("proposed key fell outside of our vpage: {} vpage{:x}/vaddr{:x}", key_name, cur_vpage.get(), key.descriptor_vaddr(dict_offset)); } } } @@ -979,6 +1053,7 @@ impl BasisCacheEntry { break; } } + log::info!("done syncing dict"); dict.clean = true; } Ok(()) diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 543f5d12c..91423b590 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -336,13 +336,27 @@ impl DictCacheEntry { } else { kcache.age = kcache.age.saturating_add(1); kcache.clean = false; + if data.len() == 4 { + use std::convert::TryInto; + log::info!("patching checksum: {:x} at offset {}", u32::from_le_bytes(data.try_into().unwrap()), offset); + } // 1. handle unaligned start offsets let mut written: usize = 0; - if ((kcache.start + offset as u64) % VPAGE_SIZE as u64) != 0 { + if ((kcache.start + offset as u64 + written as u64) % VPAGE_SIZE as u64) != 0 { let start_vpage_addr = ((kcache.start + offset as u64) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64; let pp = v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()).expect("large key data allocation missing"); - let mut pt_data = hw.data_decrypt_page(&cipher, &self.aad, pp).expect("Decryption auth error"); - for (&src, dst) in data[written..].iter().zip(pt_data[size_of::() + offset..].iter_mut()) { + let mut pt_data = match hw.data_decrypt_page(&cipher, &self.aad, pp) { + Some(data) => data, + None => { + // perhaps we could just "zero out" previously data, but for now, let's make this undefined behavior + log::warn!("Attempt to patch data outside of the previously initialized length. This is an error."); + return Err(Error::new(ErrorKind::UnexpectedEof, "patching outside of previously written data")); + } + }; + if offset > 0 { + log::info!("patching offset {}, total length {}, data length {}", offset % VPAGE_SIZE, kcache.len, data.len()); + } + for (&src, dst) in data[written..].iter().zip(pt_data[size_of::() + (offset % VPAGE_SIZE)..].iter_mut()) { *dst = src; written += 1; } diff --git a/services/pddb/src/backend/key.rs b/services/pddb/src/backend/key.rs index b058383c1..1e0f23904 100644 --- a/services/pddb/src/backend/key.rs +++ b/services/pddb/src/backend/key.rs @@ -175,3 +175,17 @@ impl PartialEq for KeySmallPoolOrd { self.avail == other.avail } } + +/// A structure for passing around key metadata +pub(crate) struct KeyAttributes { + /// actual length of data in the key + pub(crate) len: usize, + /// pre-reserved storage space for the key (growable to this bound "at no cost") + pub(crate) reserved: usize, + /// access count + pub(crate) age: usize, + /// owning dictionary + pub(crate) dict: String, + /// the specific basis from which this key's metadata was found + pub(crate) basis: String, +} \ No newline at end of file diff --git a/services/pddb/src/tests.rs b/services/pddb/src/tests.rs index 5cc6a8fa5..1b1524614 100644 --- a/services/pddb/src/tests.rs +++ b/services/pddb/src/tests.rs @@ -1,6 +1,8 @@ use rand::Rng; use crate::*; +const UPPER_BOUND: usize = 9000; + fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bound: usize) -> (String, Vec::) { let mut rng = rand::thread_rng(); // we want roughly half our keys to be in the small bin, and half in the large bin @@ -40,7 +42,7 @@ pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCach let num_dicts = maybe_num_dicts.unwrap_or(4); let num_keys = maybe_num_keys.unwrap_or(34); - let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, 9000)); + let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, UPPER_BOUND - 4)); // 4 bytes for the CI checksum // make all the directories first for dictnum in 1..=num_dicts { @@ -101,12 +103,28 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache, for dict in dict_list.iter() { if let Ok(key_list) = basis_cache.key_list(hw, dict) { for key in key_list.iter() { + let mut patchbuf = [0u8; UPPER_BOUND]; // this actually does something a bit more complicated on a multi-basis system than you'd think: // it will get the union of all key names, and then patch the *latest open basis* only with new data. // note: if the key didn't already exist in the latest open basis, it's added, with just that patch data in it. // to override this behavior, you'd want to specify a _specific_ basis, but for testing purposes, we only have one // so we're doing this way that would lead to surprising results if it were copied as template code elsewhere. basis_cache.key_update(hw, dict, key, patch_data.as_bytes(), Some(patch_offset), None, None, false).unwrap(); + + // now fix the CI checksum. structured as two separate patches. not because it's efficient, + // but because it exercises the code harder. + let readlen = basis_cache.key_read(hw, dict, key, &mut patchbuf, Some(0), None).unwrap(); + // now extend and compute the checksum + let mut checkdata = Vec::::new(); + for &b in &patchbuf[..readlen-4] { + checkdata.push(b); + } + log::trace!("checkdata len: {}", checkdata.len()); + while checkdata.len() % 4 != 0 { + checkdata.push(0); + } + let checksum = murmur3_32(&checkdata, 0); + basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen-4), None, None, false).unwrap(); } } diff --git a/tools/pddbdbg.py b/tools/pddbdbg.py index 560d6cdf1..d28d37235 100755 --- a/tools/pddbdbg.py +++ b/tools/pddbdbg.py @@ -163,6 +163,7 @@ def main(): print("Basis {}".format(name)) v2p_table = tables[name][0] p2v_table = tables[name][1] + # v2p_table[0xfe0fe0] = 0x1200* 0x100 basis_data = bytearray() pp_start = v2p_table[VPAGE_SIZE] @@ -207,6 +208,7 @@ def main(): except ValueError: print("couldn't decrypt basis root vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE])) +PRINTED_FULL = False class KeyDescriptor: MAX_NAME_LEN = 95 def __init__(self, record, v2p, disk, key, name): @@ -271,6 +273,7 @@ class KeyDescriptor: def as_str(self, indent=''): PRINT_LEN = 64 + global PRINTED_FULL desc = '' desc += indent + 'Start: 0x{:x}\n'.format(self.start) desc += indent + 'Len: {}/0x{:x}\n'.format(self.len, self.reserved) @@ -283,12 +286,17 @@ class KeyDescriptor: desc += ' | Age: {}'.format(self.age) desc += ' | CI OK: {}'.format(self.ci_ok) desc += '\n' - if len(self.data) < 64: + if len(self.data) < PRINT_LEN: print_len = len(self.data) extra = '' else: - print_len = 64 - extra = '...' + if (PRINTED_FULL == False) and (self.ci_ok == False): + print_len = len(self.data) + extra = '' + PRINTED_FULL = True + else: + print_len = PRINT_LEN + extra = '...' desc += indent + 'Data (hex): {}{}\n'.format(self.data[:print_len].hex(), extra) desc += indent + 'Data (txt): {}{}'.format(make_printable(self.data[:print_len].decode('utf-8', errors='ignore')), extra) + '\n' return (desc) @@ -347,7 +355,7 @@ class BasisDicts: keys_found += 1 except ValueError: - print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr), pp_start) + print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr, pp_start)) keys_tried += 1 except KeyError: # the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried" @@ -504,9 +512,15 @@ class Pte: def find_mbbb(mbbb): global MBBB_PAGES pages = [mbbb[i:i+PAGE_SIZE] for i in range(0, MBBB_PAGES * PAGE_SIZE, PAGE_SIZE)] + candidates = [] for page in pages: if bytearray(page[:16]) != bytearray([0xff] * 16): - return page + candidates.append(page) + + if len(candidates) > 1: + print("More than one MBBB found, this is an error!") + else: + return candidates[0] return None def decode_pagetable(img, entries, keys, mbbb): @@ -577,7 +591,7 @@ class PhysPage: return (self.pp >> 24) & 0xF def as_str(self): - return 'a_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal()) + return 'pp_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal()) class Fscb: def __init__(self, i_bytes):