From d716fea17ae0536b3bf428ced2e77c80894087e4 Mon Sep 17 00:00:00 2001 From: bunnie Date: Tue, 23 Nov 2021 02:57:13 +0800 Subject: [PATCH] now passing dictionary add/remove test --- services/pddb/src/backend/basis.rs | 86 +++++++++------- services/pddb/src/backend/dictionary.rs | 19 ++-- services/pddb/src/backend/hw.rs | 9 +- services/pddb/src/main.rs | 14 ++- services/pddb/src/tests.rs | 91 +++++++++++------ tools/pddbdbg.py | 128 ++++++++++++++---------- 6 files changed, 216 insertions(+), 131 deletions(-) diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index d46fe9c8c..5a439893c 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -7,7 +7,7 @@ use std::convert::TryInto; use aes_gcm_siv::{Aes256GcmSiv, Key}; use aes_gcm_siv::aead::NewAead; use std::iter::IntoIterator; -use std::collections::{BinaryHeap, HashMap}; +use std::collections::{BinaryHeap, HashMap, HashSet}; use std::io::{Result, Error, ErrorKind}; @@ -304,32 +304,37 @@ impl BasisCache { } } + /// Returns a list of all the known dictionaries, across all the basis. A HashSet is returned + /// because you can have the same-named dictionary in multiple basis, and what we're asking for + /// is the union of all the dictionary names, without duplicates. + pub(crate) fn dict_list(&mut self, hw: &mut PddbOs) -> HashSet:: { + let mut dict_set = HashSet::::new(); + for basis in self.cache.iter_mut() { + basis.populate_caches(hw); + for key in basis.dicts.keys() { + dict_set.insert(String::from(key)); + } + } + dict_set + } + /// This version of the call only removes one instance of a dictionary from the specified basis. + /// Perhaps there also needs to be a `dict_remove_all` call which iterates through every basis + /// makes sure the dictionary is removed from all the possible known basis. Anyways, that function + /// would be a variant of this targeted version. pub(crate) fn dict_remove(&mut self, hw: &mut PddbOs, dict: &str, basis_name: Option<&str>, paranoid: bool ) -> Result<()> { if let Some(basis_index) = self.select_basis(basis_name) { + log::info!("deleting dict {}", dict); let basis = &mut self.cache[basis_index]; - // make sure the cache is "hot" before checking it. The rule is we only operate on data in cache, then sync it to disk. - if !basis.dicts.contains_key(dict) { - if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) { - if dict_record.flags.valid() { - let dict_name = String::from(cstr_to_string(&dict_record.name)); - let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad); - basis.dicts.insert(dict_name, dcache); - } - } else { - return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); - } - } - // now check the cache. - if basis.dicts.contains_key(dict) { - basis.age = basis.age.saturating_add(1); - basis.clean = false; - return basis.dict_delete(hw, dict, paranoid) - } else { - return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); - } + + basis.age = basis.age.saturating_add(1); + basis.clean = false; + basis.dict_delete(hw, dict, paranoid)?; + basis.basis_sync(hw); + basis.pt_sync(hw); + Ok(()) } else { Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) } @@ -705,6 +710,7 @@ impl BasisCacheEntry { self.large_alloc_ptr = Some(PageAlignedVa::from(maybe_end)); } } + /// allocate a pointer data in the large pool, of length `amount`. "always" succeeds because... /// there's 16 million terabytes of large pool to allocate before you run out? /* @@ -717,6 +723,7 @@ impl BasisCacheEntry { self.large_alloc_ptr = Some(self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)) + PageAlignedVa::from(amount)); return alloc_ptr.as_u64() }*/ + /// do a deep scan of all the dictionaries and keys and attempt to populate all the caches pub(crate) fn populate_caches(&mut self, hw: &mut PddbOs) { let mut try_entry = 1; @@ -758,6 +765,7 @@ impl BasisCacheEntry { pub(crate) fn dict_delete(&mut self, hw: &mut PddbOs, name: &str, paranoid: bool) -> Result<()> { let mut dict_found = false; if !self.dicts.contains_key(name) { + log::info!("dict delete: key not in cache {}", name); // if the dictionary doesn't exist in our cache it doesn't necessarily mean it // doesn't exist. Do a comprehensive search if our cache isn't complete. if let Some((index, dict_record)) = self.dict_deep_search(hw, name) { @@ -771,43 +779,52 @@ impl BasisCacheEntry { } if dict_found { let dcache = self.dicts.get_mut(name).expect("entry was ensured, but somehow missing"); + //log::info!("dcache {} index {}, key_count {}", name, dcache.index, dcache.key_count); // ensure all the keys are in RAM dcache.fill(hw, &self.v2p_map, &self.cipher); + //log::info!("dcache filled"); // allocate a copy of the key list, to avoid interior mutability problems with the next remove step let mut key_list = Vec::::new(); for key in dcache.keys.keys() { key_list.push(key.to_string()); } for key in key_list { + //log::info!("removing {}:{}", name, key); // this will wipe any large pools if paranoid is set dcache.key_remove(hw, &mut self.v2p_map, &self.cipher, &key, paranoid); } // wipe & de-allocate any small pages for index in 0..dcache.small_pool.len() { let pool_vaddr = VirtAddr::new(dcache.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START + index as u64 * SMALL_CAPACITY as u64).unwrap(); - if let Some(pp) = self.v2p_map.get(&pool_vaddr) { + if let Some(pp) = self.v2p_map.remove(&pool_vaddr) { if paranoid { let mut random = [0u8; PAGE_SIZE]; hw.trng_slice(&mut random); hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32); } - hw.fast_space_free(*pp); + hw.fast_space_free(pp); } } - // erase the header by writing over with random data. This makes the dictionary unsearchable, but if you - // have the key, you can of course do a hard-scan and try partially re-assemble the dictionary if you - // did not do the delete in paranoid mode. - if let Some(pp) = self.v2p_map.get(&VirtAddr::new(dcache.index as u64 * DICT_VSIZE as u64).unwrap()) { - let mut random = [0u8; PAGE_SIZE]; - hw.trng_slice(&mut random); - hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32); - hw.fast_space_free(*pp); - } else { - log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table."); + // erase the entire dictionary + key allocation area by writing over with random data. It's important to + // do a comprehensive erase, because if the dictionary slot is re-used, the previously allocated key entries + // decrypt correctly, are interpreted as valid keys, and thus cause consistency errors. + let key_pages = 1 + (dcache.last_disk_key_index + 1) as usize / DK_PER_VPAGE; + for page in 0..key_pages { + let dk_vaddr = VirtAddr::new(dcache.index as u64 * DICT_VSIZE as u64 + page as u64 * VPAGE_SIZE as u64).unwrap(); + if let Some(pp) = self.v2p_map.remove(&dk_vaddr) { + log::info!("erasing dk page 0x{:x}/0x{:x}", dk_vaddr, pp.page_number() as usize * PAGE_SIZE); + let mut random = [0u8; PAGE_SIZE]; + hw.trng_slice(&mut random); + hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32); + hw.fast_space_free(pp); + } else { + log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table."); + } } // mark data for re-use self.free_dict_offset = Some(dcache.index); + self.num_dicts -= 1; // remove the cache entry self.dicts.remove(name); @@ -910,7 +927,6 @@ impl BasisCacheEntry { if let Some(dict) = self.dicts.get_mut(&String::from(name)) { let dict_offset = VirtAddr::new(dict.index as u64 * DICT_VSIZE).unwrap(); if !dict.clean { - log::info!("syncing dictionary {}", name); let mut dict_name = [0u8; DICT_NAME_LEN]; for (src, dst) in name.bytes().into_iter().zip(dict_name.iter_mut()) { *dst = src; @@ -922,7 +938,7 @@ impl BasisCacheEntry { free_key_index: dict.last_disk_key_index, name: dict_name, }; - log::info!("syncing dict: {:?}", dict_disk); + log::info!("syncing dict {}: {:?}", name, dict_disk); // log::info!("raw: {:x?}", dict_disk.deref()); // observation: all keys to be flushed to disk will be in the KeyCacheEntry. Some may be clean, // but definitely all the dirty ones are in there (if they aren't, where else would they be??) diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 13f1922d9..117693264 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -167,6 +167,7 @@ impl DictCacheEntry { } /// Populates cache entries, reporting the maximum extent of large alloc data seen so far. /// Requires a descriptor for the hardware, and our virtual to physical page mapping. + /// Does not overwrite existing cache entries, if they already exist -- only loads in ones that are missing. /// Todo: condense routines in common with ensure_key_entry() to make it easier to maintain. pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv) -> VirtAddr { let mut try_entry = 1; @@ -209,15 +210,19 @@ impl DictCacheEntry { data: None, }; let kname = cstr_to_string(&keydesc.name); - if keydesc.start + keydesc.reserved > alloc_top.get() { - // if the key is within the large pool space, note its allocation for the basis overall - alloc_top = VirtAddr::new(keydesc.start + keydesc.reserved).unwrap(); - // nothing else needs to be done -- we don't pre-cache large key data. + if !self.keys.contains_key(&kname) { + if keydesc.start + keydesc.reserved > alloc_top.get() { + // if the key is within the large pool space, note its allocation for the basis overall + alloc_top = VirtAddr::new(keydesc.start + keydesc.reserved).unwrap(); + // nothing else needs to be done -- we don't pre-cache large key data. + } else { + // try to fill the small key cache entry details + self.try_fill_small_key(hw, v2p_map, cipher, &mut data_cache, &mut kcache, &kname); + } + self.keys.insert(kname, kcache); } else { - // try to fill the small key cache entry details - self.try_fill_small_key(hw, v2p_map, cipher, &mut data_cache, &mut kcache, &kname); + log::info!("fill: entry already present {}", kname); } - self.keys.insert(kname, kcache); key_count += 1; } try_entry += 1; diff --git a/services/pddb/src/backend/hw.rs b/services/pddb/src/backend/hw.rs index 75f91ea38..2681d2354 100644 --- a/services/pddb/src/backend/hw.rs +++ b/services/pddb/src/backend/hw.rs @@ -189,6 +189,7 @@ impl PddbOs { pub fn dbg_dump(&self, _name: Option) { // placeholder } + #[allow(dead_code)] #[cfg(not(any(target_os = "none", target_os = "xous")))] /// used to reset the hardware structure for repeated runs of testing within a single invocation pub fn test_reset(&mut self) { @@ -859,10 +860,10 @@ impl PddbOs { let cipher = self.cipher_ecb.as_ref().expect("Inconsistent internal state - syskey_ensure() failed"); let mut update = SpaceUpdate::new(self.entropy.borrow_mut().get_u64(), ppc); let mut block = Block::from_mut_slice(update.deref_mut()); - log::info!("block: {:x?}", block); + log::trace!("block: {:x?}", block); cipher.encrypt_block(&mut block); let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr; - log::info!("patch: {:x?}", block); + log::trace!("patch: {:x?}", block); self.patch_fscb(&block, log_addr); let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr; if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 { @@ -894,10 +895,10 @@ impl PddbOs { let cipher = self.cipher_ecb.as_ref().expect("Inconsistent internal state - syskey_ensure() failed"); let mut update = SpaceUpdate::new(self.entropy.borrow_mut().get_u64(), pp); let mut block = Block::from_mut_slice(update.deref_mut()); - log::info!("block: {:x?}", block); + log::trace!("block: {:x?}", block); cipher.encrypt_block(&mut block); let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr; - log::info!("patch: {:x?}", block); + log::trace!("patch: {:x?}", block); self.patch_fscb(&block, log_addr); let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr; if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 { diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 10f6ad3a0..785baed64 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -357,7 +357,7 @@ const PDDB_BACKING_SIZE: usize = 0x4000_0000; #[xous::xous_main] fn xmain() -> ! { log_server::init_wait().unwrap(); - log::set_max_level(log::LevelFilter::Trace); + log::set_max_level(log::LevelFilter::Debug); log::info!("my PID is {}", xous::process::id()); let xns = xous_names::XousNames::new().unwrap(); @@ -375,9 +375,18 @@ fn xmain() -> ! { #[cfg(not(any(target_os = "none", target_os = "xous")))] { log::info!("Creating `basecase1`"); - create_testcase(&mut pddb_os, None, None, None); + let mut basis_cache = BasisCache::new(); + create_basis_testcase(&mut pddb_os, &mut basis_cache, None, None, None); + log::info!("Saving `basecase1` to local host"); pddb_os.dbg_dump(Some("basecase1".to_string())); + + log::info!("Doing delete/add consistency"); + delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, None, None, None); + log::info!("Saving `dacheck` to local host"); + pddb_os.dbg_dump(Some("dacheck".to_string())); } + log::info!("CI done"); + /* { // a simple case that could be run directly on the hardware log::info!("Running `manual` test case"); #[cfg(not(any(target_os = "none", target_os = "xous")))] @@ -408,6 +417,7 @@ fn xmain() -> ! { #[cfg(not(any(target_os = "none", target_os = "xous")))] pddb_os.dbg_dump(Some("manual".to_string())); } + */ /* list of test cases: genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k. delete/add consistency: general integrity, delete a dictionary, then add a dictionary. diff --git a/services/pddb/src/tests.rs b/services/pddb/src/tests.rs index 13697bfc2..90d8a4c14 100644 --- a/services/pddb/src/tests.rs +++ b/services/pddb/src/tests.rs @@ -1,12 +1,40 @@ use rand::Rng; use crate::*; -pub(crate) fn create_testcase(hw: &mut PddbOs, - maybe_num_dicts: Option, maybe_num_keys: Option, maybe_key_sizes: Option<(usize, usize)>) { +fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bound: usize) -> (String, Vec::) { let mut rng = rand::thread_rng(); + // we want roughly half our keys to be in the small bin, and half in the large bin + let keylen = if rng.gen_bool(0.5) { + rng.gen_range(lower_size_bound..VPAGE_SIZE) + } else { + rng.gen_range(VPAGE_SIZE..upper_size_bound) + }; + // record the owning dictionary name & length with the key. This isn't mandatory for a key name, + // but it helps the test checking program check things. + let keyname = format!("sanitycheck|{}|key{}|len{}", dictname, keynum, keylen); + let mut keydata = Vec::::new(); + for i in 0..keylen-4 { + // the data starts with a number equal to the key number, and increments from there + keydata.push((keynum + i) as u8); + } + // a checksum is appended to each run of data + // copy the stored data, and pad it out to a multiple of word lengths with 0's so we can compute the checksum + let mut checkdata = Vec::::new(); + for &b in &keydata { + checkdata.push(b); + } + while checkdata.len() % 4 != 0 { + checkdata.push(0); + } + let checksum = murmur3_32(&checkdata, 0); + keydata.append(&mut checksum.to_le_bytes().to_vec()); + (keyname, keydata) +} + +pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCache, + maybe_num_dicts: Option, maybe_num_keys: Option, maybe_key_sizes: Option<(usize, usize)>) { hw.pddb_format().unwrap(); - let mut basis_cache = BasisCache::new(); let sys_basis = hw.pddb_mount().expect("couldn't mount system basis"); basis_cache.basis_add(sys_basis); @@ -25,37 +53,44 @@ pub(crate) fn create_testcase(hw: &mut PddbOs, for keynum in 1..=num_keys { for dictnum in 1..=num_dicts { let dictname = format!("dict{}", dictnum); - // we want roughly half our keys to be in the small bin, and half in the large bin - let keylen = if rng.gen_bool(0.5) { - rng.gen_range(key_lower_bound..VPAGE_SIZE) - } else { - rng.gen_range(VPAGE_SIZE..key_upper_bound) - }; - // record the owning dictionary name & length with the key. This isn't mandatory for a key name, - // but it helps the test checking program check things. - let keyname = format!("sanitycheck|{}|key{}|len{}", dictname, keynum, keylen); - let mut keydata = Vec::::new(); - for i in 0..keylen-4 { - // the data starts with a number equal to the key number, and increments from there - keydata.push((keynum + i) as u8); - } - // a checksum is appended to each run of data - // copy the stored data, and pad it out to a multiple of word lengths with 0's so we can compute the checksum - let mut checkdata = Vec::::new(); - for &b in &keydata { - checkdata.push(b); - } - while checkdata.len() % 4 != 0 { - checkdata.push(0); - } - let checksum = murmur3_32(&checkdata, 0); - keydata.append(&mut checksum.to_le_bytes().to_vec()); + let (keyname, keydata) = gen_key(&dictname, keynum, key_lower_bound, key_upper_bound); // now we're ready to write it out basis_cache.key_update(hw, &dictname, &keyname, &keydata, None, None, None, false).unwrap(); } } } +/// Delete & add dictionary consistency check +pub(crate) fn delete_add_dict_consistency(hw: &mut PddbOs, basis_cache: &mut BasisCache, + maybe_num_dicts: Option, maybe_num_keys: Option, maybe_key_sizes: Option<(usize, usize)>) { + let evict_count = maybe_num_dicts.unwrap_or(1); + let num_keys = maybe_num_keys.unwrap_or(36); + let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, 9000)); + + let dict_list = basis_cache.dict_list(hw); + let dict_start_index = dict_list.len(); + for (evicted, evict_dict) in dict_list.iter().enumerate() { + if evicted < evict_count { + match basis_cache.dict_remove(hw, evict_dict, None, false) { + Ok(_) => {}, + Err(e) => log::error!("Error evicting dictionary {}: {:?}", evict_dict, e), + } + } else { + break; + } + } + + for keynum in 1..=num_keys { + for dictnum in 1..=evict_count { + let dictname = format!("dict{}", dictnum + dict_start_index); + let (keyname, keydata) = gen_key(&dictname, keynum, key_lower_bound, key_upper_bound); + // now we're ready to write it out + basis_cache.key_update(hw, &dictname, &keyname, &keydata, None, None, None, false).unwrap(); + } + } +} + +#[allow(dead_code)] pub(crate) fn manual_testcase(hw: &mut PddbOs) { log::info!("Initializing disk..."); hw.pddb_format().unwrap(); diff --git a/tools/pddbdbg.py b/tools/pddbdbg.py index c810584e5..560d6cdf1 100755 --- a/tools/pddbdbg.py +++ b/tools/pddbdbg.py @@ -115,7 +115,6 @@ def main(): keyfile = './tools/pddb-images/{}.key'.format(args.name) imagefile = './tools/pddb-images/{}.bin'.format(args.name) - keys = {} with open(keyfile, 'rb') as key_f: raw_key = key_f.read() @@ -140,6 +139,7 @@ def main(): KEY_PAGES = 1 global PAGE_SIZE global VPAGE_SIZE + MAX_DICTS = 16384 with open(imagefile, 'rb') as img_f: raw_img = img_f.read() @@ -178,9 +178,19 @@ def main(): print(basis.as_str()) basis_dicts = {} - for dict_index in range(basis.num_dicts): + dicts_found = 0 + dict_index = 0 + while dict_index < MAX_DICTS and dicts_found < basis.num_dicts: bdict = BasisDicts(dict_index, v2p_table, data, key, name) - basis_dicts[bdict.name] = bdict + if bdict.valid: + basis_dicts[bdict.name] = bdict + dicts_found += 1 + dict_index += 1 + if dicts_found != basis.num_dicts: + print("Expected {} dictionaries, only found {}; searched {}".format(basis.num_dicts, dicts_found, dict_index)) + found_all_dicts = False + else: + found_all_dicts = True print(" Dictionaries: ") for bdict in basis_dicts.values(): @@ -189,6 +199,10 @@ def main(): print("CI checks:") for bdict in basis_dicts.values(): bdict.ci_check() + if found_all_dicts: + print("All dicts were found.") + else: + print("Missing dictionaries, something is wrong.") except ValueError: print("couldn't decrypt basis root vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE])) @@ -255,7 +269,7 @@ class KeyDescriptor: print('checksum: {:x}, refchecksum: {:x}\n'.format(checksum, refcheck)) - def as_str(self, indent): + def as_str(self, indent=''): PRINT_LEN = 64 desc = '' desc += indent + 'Start: 0x{:x}\n'.format(self.start) @@ -288,59 +302,63 @@ class BasisDicts: global PAGE_SIZE global VPAGE_SIZE dict_header_vaddr = self.DICT_VSTRIDE * (index + 1) - pp = v2p[dict_header_vaddr] - self.keys = {} - try: - print('dict decrypt @ pp {:x}, nonce: {}'.format(pp, disk[pp:pp+12].hex())) - cipher = AES_GCM_SIV(key, disk[pp:pp+12]) - pt_data = cipher.decrypt(disk[pp+12:pp+PAGE_SIZE], basis_aad(name)) - # print('raw pt_data: {}'.format(pt_data[:127].hex())) - i = 0 - self.journal = int.from_bytes(pt_data[i:i+4], 'little') - i += 4 - self.flags = int.from_bytes(pt_data[i:i+4], 'little') - i += 4 - self.age = int.from_bytes(pt_data[i:i+4], 'little') - i += 4 - self.num_keys = int.from_bytes(pt_data[i:i+4], 'little') - i += 4 - self.free_key_index = int.from_bytes(pt_data[i:i+4], 'little') - i += 4 - self.name = pt_data[i:i+BasisDicts.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore') - i += BasisDicts.MAX_NAME_LEN - print("dict header len: {}".format(i-4)) # subtract 4 because of the journal - except ValueError: - print("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr])) + if dict_header_vaddr in v2p: + self.valid = True + pp = v2p[dict_header_vaddr] + self.keys = {} + try: + print('dict decrypt @ pp {:x}, nonce: {}'.format(pp, disk[pp:pp+12].hex())) + cipher = AES_GCM_SIV(key, disk[pp:pp+12]) + pt_data = cipher.decrypt(disk[pp+12:pp+PAGE_SIZE], basis_aad(name)) + # print('raw pt_data: {}'.format(pt_data[:127].hex())) + i = 0 + self.journal = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.flags = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.age = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.num_keys = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.free_key_index = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.name = pt_data[i:i+BasisDicts.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore') + i += BasisDicts.MAX_NAME_LEN + print("dict header len: {}".format(i-4)) # subtract 4 because of the journal + except ValueError: + print("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr])) - if self.num_keys > 0: - keys_found = 0 - keys_tried = 1 - while (keys_found < self.num_keys) and keys_tried < 131071: - keyindex_start_vaddr = self.DICT_VSTRIDE * (index + 1) + (keys_tried * self.KV_STRIDE) - try: - pp_start = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE] - pp_data = disk[pp_start:pp_start + PAGE_SIZE] + if self.num_keys > 0: + keys_found = 0 + keys_tried = 1 + while (keys_found < self.num_keys) and keys_tried < 131071: + keyindex_start_vaddr = self.DICT_VSTRIDE * (index + 1) + (keys_tried * self.KV_STRIDE) try: - cipher = AES_GCM_SIV(key, pp_data[:12]) - pt_data = cipher.decrypt(pp_data[12:], basis_aad(name)) - key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE - # print(key_index) - maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name) - if maybe_key.valid: - self.keys[maybe_key.name] = maybe_key - keys_found += 1 + pp_start = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE] + pp_data = disk[pp_start:pp_start + PAGE_SIZE] + try: + cipher = AES_GCM_SIV(key, pp_data[:12]) + pt_data = cipher.decrypt(pp_data[12:], basis_aad(name)) + key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE + # print(key_index) + maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name) + if maybe_key.valid: + self.keys[maybe_key.name] = maybe_key + keys_found += 1 - except ValueError: - print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr), pp_start) - keys_tried += 1 - except KeyError: - # the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried" - keys_tried += VPAGE_SIZE // self.KV_STRIDE - if keys_found < self.num_keys: - print("Expected {} keys but only found {}".format(keys_found, self.num_keys)) - self.found_all_keys = False - else: - self.found_all_keys = True + except ValueError: + print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr), pp_start) + keys_tried += 1 + except KeyError: + # the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried" + keys_tried += VPAGE_SIZE // self.KV_STRIDE + if keys_found < self.num_keys: + print("Expected {} keys but only found {}".format(self.num_keys, keys_found)) + self.found_all_keys = False + else: + self.found_all_keys = True + else: + self.valid = False def as_str(self): desc = '' @@ -367,7 +385,7 @@ class BasisDicts: print(" Key length mismatch: {}/{}", keylen, key.len) if key.ci_ok == False: print(" CI failed on key:") - print(key.as_str()) + print(key.as_str(' ')) check_ok = False if self.found_all_keys == False: print(' Dictionary was missing keys')