diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index 8c77ce92f..0f9734f9f 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -306,6 +306,35 @@ impl BasisCache { } } + + pub(crate) fn dict_remove(&mut self, + hw: &mut PddbOs, dict: &str, basis_name: Option<&str>, paranoid: bool + ) -> Result<()> { + if let Some(basis_index) = self.select_basis(basis_name) { + let basis = &mut self.cache[basis_index]; + if !basis.dicts.contains_key(dict) { + if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) { + if dict_record.flags.valid() { + let dict_name = String::from(cstr_to_string(&dict_record.name)); + let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad); + basis.dicts.insert(dict.to_string(), dcache); + } + } else { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); + } + } + if let Some(dict_entry) = basis.dicts.get_mut(dict) { + basis.age = basis.age.saturating_add(1); + basis.clean = false; + return basis.dict_delete(hw, dict, paranoid) + } else { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); + } + } else { + Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) + } + } + pub(crate) fn key_read(&mut self, hw: &mut PddbOs, dict: &str, key: &str, data: &mut [u8], offset: Option, basis_name:Option<&str> ) -> Result { @@ -380,6 +409,51 @@ impl BasisCache { } } + pub(crate) fn key_remove(&mut self, + hw: &mut PddbOs, dict: &str, key: &str, basis_name: Option<&str>, paranoid: bool + ) -> Result<()> { + if let Some(basis_index) = self.select_basis(basis_name) { + let basis = &mut self.cache[basis_index]; + if !basis.dicts.contains_key(dict) { + if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) { + if dict_record.flags.valid() { + let dict_name = String::from(cstr_to_string(&dict_record.name)); + let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad); + basis.dicts.insert(dict.to_string(), dcache); + } + } else { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); + } + } + if let Some(dict_entry) = basis.dicts.get_mut(dict) { + basis.age = basis.age.saturating_add(1); + basis.clean = false; + if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) { + if !paranoid { + dict_entry.key_remove(hw, &mut basis.v2p_map, &basis.cipher, key, false); + } else { + // this implementation is still in progress + dict_entry.key_erase(key); + + // encrypt and write the dict entry to disk + basis.dict_sync(hw, dict)?; + // sync the root basis structure as well, while we're at it... + basis.basis_sync(hw); + // finally, sync the page tables. + basis.pt_sync(hw); + } + return Ok(()) + } else { + return Err(Error::new(ErrorKind::NotFound, "key not found")); + } + } else { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); + } + } else { + Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted.")) + } + } + /// Updates a key in a dictionary; if it doesn't exist, creates it. User can specify a basis, /// or rely upon the auto-basis select algorithm. pub(crate) fn key_update(&mut self, @@ -671,25 +745,61 @@ impl BasisCacheEntry { /// through compounded data and re-writing partias sectors, and because of this, initially, /// the `paranoid` erase is `unimplemented`. pub(crate) fn dict_delete(&mut self, hw: &mut PddbOs, name: &str, paranoid: bool) -> Result<()> { - if let Some((index, _dict)) = self.dict_deep_search(hw, name) { - if !paranoid { - // erase the header by writing over with random data. This makes the dictionary unsearchable, but if you - // have the key, you can of course do a hard-scan and try partially re-assemble the dictionary. - if let Some(pp) = self.v2p_map.get(&VirtAddr::new(index as u64 * DICT_VSIZE as u64).unwrap()) { - let mut random = [0u8; PAGE_SIZE]; - hw.trng_slice(&mut random); - hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32); - } else { - log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table."); + let mut dict_found = false; + if !self.dicts.contains_key(name) { + // if the dictionary doesn't exist in our cache it doesn't necessarily mean it + // doesn't exist. Do a comprehensive search if our cache isn't complete. + if let Some((index, dict_record)) = self.dict_deep_search(hw, name) { + let dict_name = String::from(cstr_to_string(&dict_record.name)); + let dcache = DictCacheEntry::new(dict_record, index as usize, &self.aad); + self.dicts.insert(name.to_string(), dcache); + dict_found = true; + } + } else { + dict_found = true; + } + if dict_found { + let dcache = self.dicts.get_mut(name).expect("entry was ensured, but somehow missing"); + // ensure all the keys are in RAM + dcache.fill(hw, &self.v2p_map, &self.cipher); + // allocate a copy of the key list, to avoid interior mutability problems with the next remove step + let mut key_list = Vec::::new(); + for key in dcache.keys.keys() { + key_list.push(key.to_string()); + } + for key in key_list { + // this will wipe any large pools if paranoid is set + dcache.key_remove(hw, &mut self.v2p_map, &self.cipher, &key, paranoid); + } + // wipe & de-allocate any small pages + for index in 0..dcache.small_pool.len() { + let pool_vaddr = VirtAddr::new(dcache.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START + index as u64 * SMALL_CAPACITY as u64).unwrap(); + if let Some(pp) = self.v2p_map.get(&pool_vaddr) { + if paranoid { + let mut random = [0u8; PAGE_SIZE]; + hw.trng_slice(&mut random); + hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32); + } + hw.fast_space_free(*pp); } + } + // erase the header by writing over with random data. This makes the dictionary unsearchable, but if you + // have the key, you can of course do a hard-scan and try partially re-assemble the dictionary if you + // did not do the delete in paranoid mode. + if let Some(pp) = self.v2p_map.get(&VirtAddr::new(dcache.index as u64 * DICT_VSIZE as u64).unwrap()) { + let mut random = [0u8; PAGE_SIZE]; + hw.trng_slice(&mut random); + hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32); + hw.fast_space_free(*pp); } else { - unimplemented!("For now store sensitive data in its own Basis, and then delete the Basis."); - // this is a bit of an arduous code path, it involves recursing through all the keys and nuking - // them. Let's write this after we've actually _got keys_ (we're just figuring out how to add a dictionary - // in the first place right now!). + log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table."); } - // de-allocate all of the dictionary entries + // mark data for re-use + self.free_dict_offset = Some(dcache.index); + // remove the cache entry + self.dicts.remove(name); + Ok(()) } else { Err(Error::new(ErrorKind::NotFound, "Dictionary not found")) diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 73753fd80..f3d28816a 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -309,7 +309,7 @@ impl DictCacheEntry { if let Some(pool_index) = small_storage_index_from_key(&kcache, self.index) { // if the key is within the small pool space, create a bookkeeping record for it, and pre-cache its data. // generate the index within the small pool based on the address - while self.small_pool.len() < pool_index { + while self.small_pool.len() < pool_index + 1 { // fill in the pool with blank entries. In general, we should have a low amount of blank entries, but // one situation where we could get a leak is if we allocate a large amount of small data, and then delete // all but the most recently allocated one, leaving an orphan at a high index, which is then subsequently @@ -322,7 +322,8 @@ impl DictCacheEntry { ksp.contents.push(key_name.to_string()); assert!(kcache.reserved >= kcache.len, "Reserved amount is less than length, this is an error!"); assert!(kcache.reserved <= VPAGE_SIZE as u64, "Reserved amount is not appropriate for the small pool. Logic error in prior PDDB operation!"); - assert!((ksp.avail as u64) < kcache.reserved, "Total amount allocated to a small pool chunk is incorrect; suspect logic error in prior PDDB operation!"); + log::info!("avail: {} reserved: {}", ksp.avail, kcache.reserved); + assert!((ksp.avail as u64) >= kcache.reserved, "Total amount allocated to a small pool chunk is incorrect; suspect logic error in prior PDDB operation!"); ksp.avail -= kcache.reserved as u16; // note: small_pool_free is updated only after all the entries have been read in @@ -373,11 +374,12 @@ impl DictCacheEntry { let kcache = self.keys.get_mut(name).expect("Entry was assured, but then not there!"); // the update isn't going to fit in the reserved space, remove it, and re-insert it with an entirely new entry. if kcache.reserved < (data.len() + offset) as u64 { - self.key_remove(hw, v2p_map, cipher, name); + self.key_remove(hw, v2p_map, cipher, name, false); return self.key_update(hw, v2p_map, cipher, name, data, offset, alloc_hint, truncate, large_alloc_ptr); } // the key exists, *and* there's sufficient space for the data if kcache.start < SMALL_POOL_END { + log::info!("doing data update"); // it's a small key; note that we didn't consult the *size* of the key to determine its pool type: // small-sized keys can still end up in the "large" space if the small pool allocation is exhausted. if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") { @@ -393,6 +395,9 @@ impl DictCacheEntry { } else { panic!("Key allocated to small area but its cache data was not of the small type"); } + // mark the storage pool entry as dirty, too. + let pool_index = small_storage_index_from_key(&kcache, self.index).expect("index missing"); + self.small_pool[pool_index].clean = false; // note: there is no need to update small_pool_free because the reserved size did not change. } else { // it's a large key @@ -456,6 +461,7 @@ impl DictCacheEntry { } } kcache.reserved = vpage_end_offset.as_u64() - kcache.start; + kcache.clean = false; } } } @@ -573,16 +579,17 @@ impl DictCacheEntry { } } /// Used to remove a key from the dictionary. If you call it with a non-existent key, - /// the routine has no effect, and does not report an error. It will not flush to disk, - /// unless the key isn't in the cache, in which case the disk entry is edited. + /// the routine has no effect, and does not report an error. Small keys are not immediately + /// overwritten in paranoid mode, but large keys are. pub fn key_remove(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::, cipher: &Aes256GcmSiv, - name_str: &str) { + name_str: &str, paranoid: bool) { // this call makes sure we have a cache entry to operate on. self.ensure_key_entry(hw, v2p_map, cipher, name_str); let name = String::from(name_str); let mut need_rebuild = false; let mut need_free_key: Option = None; if let Some(kcache) = self.keys.get_mut(&name) { + self.clean = false; if let Some(small_index) = small_storage_index_from_key(kcache, self.index) { // handle the small pool case let ksp = &mut self.small_pool[small_index]; @@ -605,6 +612,11 @@ impl DictCacheEntry { // ...but we remove the virtual pages from the page pool, effectively reclaiming the physical space. for vpage in kcache.large_pool_vpages() { if let Some(pp) = v2p_map.remove(&vpage) { + if paranoid { + let mut noise = [0u8; PAGE_SIZE]; + hw.trng_slice(&mut noise); + hw.patch_data(&noise, pp.page_number() * PAGE_SIZE as u32); + } hw.fast_space_free(pp); } } @@ -627,6 +639,9 @@ impl DictCacheEntry { // a call to sync is necessary to completely flush things, but, we don't sync every time we remove because it's inefficient. } /// used to remove a key from the dictionary, syncing 0's to the disk in the key's place + /// sort of less relevant now that the large keys have a paranoid mode; probably this routine should actually + /// be a higher-level function that catches the paranoid request and does an "update" of 0's to the key + /// then does a disk sync and then calls remove pub fn key_erase(&mut self, name: &str) { unimplemented!(); } @@ -663,7 +678,7 @@ impl DictCacheEntry { pub(crate) fn sync_small_pool(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::, cipher: &Aes256GcmSiv) { for (index, entry) in self.small_pool.iter_mut().enumerate() { if !entry.clean { - let pool_vaddr = VirtAddr::new(self.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START).unwrap(); + let pool_vaddr = VirtAddr::new(self.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START + index as u64 * SMALL_CAPACITY as u64).unwrap(); let pp= v2p_map.entry(pool_vaddr).or_insert_with(|| hw.try_fast_space_alloc().expect("No free space to allocate small key storage")); pp.set_valid(true); diff --git a/services/pddb/src/backend/key.rs b/services/pddb/src/backend/key.rs index e25ae45df..075fa22ff 100644 --- a/services/pddb/src/backend/key.rs +++ b/services/pddb/src/backend/key.rs @@ -150,26 +150,6 @@ impl KeySmallPool { clean: true, } } - /// Tries to fit the key into the current pool. If there isn't enough space, the function returns false. - /// If `capacity_hint` is provided, it tries to reserve this amount of capacity for the key, even if the - /// data provided is smaller than the hint. - pub(crate) fn try_insert_key(&self, data: &KeyCacheEntry, capacity_hint: Option) -> bool { - // 1. check if the key will fit - // 2. if so, add its size to the used pool and contents - // 3. if not, return false - false - } - /// Tries to update the key in the pool with the new data. If it doesn't fit, - /// returns Some(key_name). Else it returns None. Note that the data is immediately - /// evicted if it doesn't fit -- the caller must then find a new pool to place the - /// evicted key in. - pub(crate) fn try_update_key(&self, data: &KeyCacheEntry, capacity_hint: Option) -> Option { - None - } - pub(crate) fn delete_key(&self, data: &KeyCacheEntry) { - // 1. remove the contents string - // 2. update the used tracking information - } /// Returns the amount of space available in the pool pub(crate) fn free(&self) -> usize { self.avail as usize diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 234a2b2db..c86f257b7 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -406,9 +406,43 @@ fn xmain() -> ! { } basis_cache.key_update(&mut pddb_os, "system settings", "big_pool1", &bigdata, None, None, None, false).expect("couldn't add a key"); + basis_cache.dict_add(&mut pddb_os, "test_dict_2", None).expect("couldn't add test dictionary 2"); + basis_cache.key_update(&mut pddb_os, "test_dict_2", "test key in dict 2", "some data".as_bytes(), None, Some(128), None, false).expect("couldn't add a key to second dict"); + + basis_cache.key_update(&mut pddb_os, "system settings", "wifi/wpa_keys/e4200", "ABC".as_bytes(), Some(2), None, None, false).expect("couldn't update e4200 key"); + + log::info!("test readback of wifi/wpa_keys/e4200"); + match basis_cache.key_read(&mut pddb_os, "system settings", "wifi/wpa_keys/e4200", &mut readback, None, None) { + Ok(readsize) => { + log::info!("read back {} bytes", readsize); + log::info!("read data: {}", String::from_utf8_lossy(&readback)); + }, + Err(e) => { + log::info!("couldn't read data: {:?}", e); + } + } + #[cfg(not(any(target_os = "none", target_os = "xous")))] pddb_os.dbg_dump(); + log::info!("Re-mount the PDDB"); + if let Some(sys_basis) = pddb_os.pddb_mount() { + log::info!("PDDB mount operation finished successfully"); + basis_cache.add_basis(sys_basis); + } else { + log::info!("PDDB did not mount; did you remember to format the PDDB region?"); + } + log::info!("test readback of wifi/wpa_keys/e4200"); + match basis_cache.key_read(&mut pddb_os, "system settings", "wifi/wpa_keys/e4200", &mut readback, None, None) { + Ok(readsize) => { + log::info!("read back {} bytes", readsize); + log::info!("read data: {}", String::from_utf8_lossy(&readback)); + }, + Err(e) => { + log::info!("couldn't read data: {:?}", e); + } + } + // register a suspend/resume listener let sr_cid = xous::connect(pddb_sid).expect("couldn't create suspend callback connection"); let susres = susres::Susres::new(&xns, api::Opcode::SuspendResume as u32, sr_cid).expect("couldn't create suspend/resume object"); diff --git a/tools/pddbdbg.py b/tools/pddbdbg.py index 237a55148..455f90790 100755 --- a/tools/pddbdbg.py +++ b/tools/pddbdbg.py @@ -184,7 +184,7 @@ def main(): class KeyDescriptor: MAX_NAME_LEN = 95 def __init__(self, record, v2p, disk, key, name): - print(record.hex()) + # print(record.hex()) i = 0 self.start = int.from_bytes(record[i:i+8], 'little') i += 8 @@ -245,8 +245,14 @@ class KeyDescriptor: if self.unresolved: desc += indent + 'UNRESOLVED' desc += '\n' - desc += indent + 'Data (hex): {}\n'.format(self.data.hex()) - desc += indent + 'Data (txt): {}\n'.format(self.data.decode('utf8', errors='ignore')) + if len(self.data) < 256: + print_len = len(self.data) + extra = '' + else: + print_len = 256 + extra = '...' + desc += indent + 'Data (hex): {}{}\n'.format(self.data[:print_len].hex(), extra) + desc += indent + 'Data (txt): {}{}'.format(self.data[:print_len].decode('utf-8', errors='ignore'), extra).replace('\n', '').replace('\r', '') + '\n' return (desc) @@ -275,7 +281,7 @@ class BasisDicts: i += 4 self.name = pt_data[i:i+BasisDicts.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore') i += BasisDicts.MAX_NAME_LEN - print("dict header len: {}".format(i-4)) # subtract 4 because of the journal + #print("dict header len: {}".format(i-4)) # subtract 4 because of the journal except ValueError: print("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr])) @@ -290,7 +296,7 @@ class BasisDicts: cipher = AES_GCM_SIV(key, pp_data[:12]) pt_data = cipher.decrypt(pp_data[12:], basis_aad(name)) key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE - print(key_index) + # print(key_index) maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name) if maybe_key.valid: self.keys[maybe_key.name] = maybe_key