most of the core routines are now in place

can create/delete dicts, create/delete/update keys.
basis deletion is...just forgetting the password. that's the point,
at least. could maybe extend the basis AAD to include some small
amount of derived data that could be deleted to facilitate basis
deletion...

anyways. next up is...lots and lots of testing.
This commit is contained in:
bunnie 2021-11-22 04:46:36 +08:00
parent 2859e54bb9
commit 1874c8e72d
5 changed files with 192 additions and 47 deletions

View file

@ -306,6 +306,35 @@ impl BasisCache {
}
}
pub(crate) fn dict_remove(&mut self,
hw: &mut PddbOs, dict: &str, basis_name: Option<&str>, paranoid: bool
) -> Result<()> {
if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.dicts.contains_key(dict) {
if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) {
if dict_record.flags.valid() {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad);
basis.dicts.insert(dict.to_string(), dcache);
}
} else {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
}
if let Some(dict_entry) = basis.dicts.get_mut(dict) {
basis.age = basis.age.saturating_add(1);
basis.clean = false;
return basis.dict_delete(hw, dict, paranoid)
} else {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
} else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
}
}
pub(crate) fn key_read(&mut self, hw: &mut PddbOs, dict: &str, key: &str, data: &mut [u8],
offset: Option<usize>, basis_name:Option<&str>
) -> Result<usize> {
@ -380,6 +409,51 @@ impl BasisCache {
}
}
pub(crate) fn key_remove(&mut self,
hw: &mut PddbOs, dict: &str, key: &str, basis_name: Option<&str>, paranoid: bool
) -> Result<()> {
if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.dicts.contains_key(dict) {
if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) {
if dict_record.flags.valid() {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad);
basis.dicts.insert(dict.to_string(), dcache);
}
} else {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
}
if let Some(dict_entry) = basis.dicts.get_mut(dict) {
basis.age = basis.age.saturating_add(1);
basis.clean = false;
if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) {
if !paranoid {
dict_entry.key_remove(hw, &mut basis.v2p_map, &basis.cipher, key, false);
} else {
// this implementation is still in progress
dict_entry.key_erase(key);
// encrypt and write the dict entry to disk
basis.dict_sync(hw, dict)?;
// sync the root basis structure as well, while we're at it...
basis.basis_sync(hw);
// finally, sync the page tables.
basis.pt_sync(hw);
}
return Ok(())
} else {
return Err(Error::new(ErrorKind::NotFound, "key not found"));
}
} else {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
} else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
}
}
/// Updates a key in a dictionary; if it doesn't exist, creates it. User can specify a basis,
/// or rely upon the auto-basis select algorithm.
pub(crate) fn key_update(&mut self,
@ -671,25 +745,61 @@ impl BasisCacheEntry {
/// through compounded data and re-writing partias sectors, and because of this, initially,
/// the `paranoid` erase is `unimplemented`.
pub(crate) fn dict_delete(&mut self, hw: &mut PddbOs, name: &str, paranoid: bool) -> Result<()> {
if let Some((index, _dict)) = self.dict_deep_search(hw, name) {
if !paranoid {
// erase the header by writing over with random data. This makes the dictionary unsearchable, but if you
// have the key, you can of course do a hard-scan and try partially re-assemble the dictionary.
if let Some(pp) = self.v2p_map.get(&VirtAddr::new(index as u64 * DICT_VSIZE as u64).unwrap()) {
let mut random = [0u8; PAGE_SIZE];
hw.trng_slice(&mut random);
hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
} else {
log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table.");
let mut dict_found = false;
if !self.dicts.contains_key(name) {
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
// doesn't exist. Do a comprehensive search if our cache isn't complete.
if let Some((index, dict_record)) = self.dict_deep_search(hw, name) {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &self.aad);
self.dicts.insert(name.to_string(), dcache);
dict_found = true;
}
} else {
dict_found = true;
}
if dict_found {
let dcache = self.dicts.get_mut(name).expect("entry was ensured, but somehow missing");
// ensure all the keys are in RAM
dcache.fill(hw, &self.v2p_map, &self.cipher);
// allocate a copy of the key list, to avoid interior mutability problems with the next remove step
let mut key_list = Vec::<String>::new();
for key in dcache.keys.keys() {
key_list.push(key.to_string());
}
for key in key_list {
// this will wipe any large pools if paranoid is set
dcache.key_remove(hw, &mut self.v2p_map, &self.cipher, &key, paranoid);
}
// wipe & de-allocate any small pages
for index in 0..dcache.small_pool.len() {
let pool_vaddr = VirtAddr::new(dcache.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START + index as u64 * SMALL_CAPACITY as u64).unwrap();
if let Some(pp) = self.v2p_map.get(&pool_vaddr) {
if paranoid {
let mut random = [0u8; PAGE_SIZE];
hw.trng_slice(&mut random);
hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
}
hw.fast_space_free(*pp);
}
}
// erase the header by writing over with random data. This makes the dictionary unsearchable, but if you
// have the key, you can of course do a hard-scan and try partially re-assemble the dictionary if you
// did not do the delete in paranoid mode.
if let Some(pp) = self.v2p_map.get(&VirtAddr::new(dcache.index as u64 * DICT_VSIZE as u64).unwrap()) {
let mut random = [0u8; PAGE_SIZE];
hw.trng_slice(&mut random);
hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
hw.fast_space_free(*pp);
} else {
unimplemented!("For now store sensitive data in its own Basis, and then delete the Basis.");
// this is a bit of an arduous code path, it involves recursing through all the keys and nuking
// them. Let's write this after we've actually _got keys_ (we're just figuring out how to add a dictionary
// in the first place right now!).
log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table.");
}
// de-allocate all of the dictionary entries
// mark data for re-use
self.free_dict_offset = Some(dcache.index);
// remove the cache entry
self.dicts.remove(name);
Ok(())
} else {
Err(Error::new(ErrorKind::NotFound, "Dictionary not found"))

View file

@ -309,7 +309,7 @@ impl DictCacheEntry {
if let Some(pool_index) = small_storage_index_from_key(&kcache, self.index) {
// if the key is within the small pool space, create a bookkeeping record for it, and pre-cache its data.
// generate the index within the small pool based on the address
while self.small_pool.len() < pool_index {
while self.small_pool.len() < pool_index + 1 {
// fill in the pool with blank entries. In general, we should have a low amount of blank entries, but
// one situation where we could get a leak is if we allocate a large amount of small data, and then delete
// all but the most recently allocated one, leaving an orphan at a high index, which is then subsequently
@ -322,7 +322,8 @@ impl DictCacheEntry {
ksp.contents.push(key_name.to_string());
assert!(kcache.reserved >= kcache.len, "Reserved amount is less than length, this is an error!");
assert!(kcache.reserved <= VPAGE_SIZE as u64, "Reserved amount is not appropriate for the small pool. Logic error in prior PDDB operation!");
assert!((ksp.avail as u64) < kcache.reserved, "Total amount allocated to a small pool chunk is incorrect; suspect logic error in prior PDDB operation!");
log::info!("avail: {} reserved: {}", ksp.avail, kcache.reserved);
assert!((ksp.avail as u64) >= kcache.reserved, "Total amount allocated to a small pool chunk is incorrect; suspect logic error in prior PDDB operation!");
ksp.avail -= kcache.reserved as u16;
// note: small_pool_free is updated only after all the entries have been read in
@ -373,11 +374,12 @@ impl DictCacheEntry {
let kcache = self.keys.get_mut(name).expect("Entry was assured, but then not there!");
// the update isn't going to fit in the reserved space, remove it, and re-insert it with an entirely new entry.
if kcache.reserved < (data.len() + offset) as u64 {
self.key_remove(hw, v2p_map, cipher, name);
self.key_remove(hw, v2p_map, cipher, name, false);
return self.key_update(hw, v2p_map, cipher, name, data, offset, alloc_hint, truncate, large_alloc_ptr);
}
// the key exists, *and* there's sufficient space for the data
if kcache.start < SMALL_POOL_END {
log::info!("doing data update");
// it's a small key; note that we didn't consult the *size* of the key to determine its pool type:
// small-sized keys can still end up in the "large" space if the small pool allocation is exhausted.
if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") {
@ -393,6 +395,9 @@ impl DictCacheEntry {
} else {
panic!("Key allocated to small area but its cache data was not of the small type");
}
// mark the storage pool entry as dirty, too.
let pool_index = small_storage_index_from_key(&kcache, self.index).expect("index missing");
self.small_pool[pool_index].clean = false;
// note: there is no need to update small_pool_free because the reserved size did not change.
} else {
// it's a large key
@ -456,6 +461,7 @@ impl DictCacheEntry {
}
}
kcache.reserved = vpage_end_offset.as_u64() - kcache.start;
kcache.clean = false;
}
}
}
@ -573,16 +579,17 @@ impl DictCacheEntry {
}
}
/// Used to remove a key from the dictionary. If you call it with a non-existent key,
/// the routine has no effect, and does not report an error. It will not flush to disk,
/// unless the key isn't in the cache, in which case the disk entry is edited.
/// the routine has no effect, and does not report an error. Small keys are not immediately
/// overwritten in paranoid mode, but large keys are.
pub fn key_remove(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv,
name_str: &str) {
name_str: &str, paranoid: bool) {
// this call makes sure we have a cache entry to operate on.
self.ensure_key_entry(hw, v2p_map, cipher, name_str);
let name = String::from(name_str);
let mut need_rebuild = false;
let mut need_free_key: Option<u32> = None;
if let Some(kcache) = self.keys.get_mut(&name) {
self.clean = false;
if let Some(small_index) = small_storage_index_from_key(kcache, self.index) {
// handle the small pool case
let ksp = &mut self.small_pool[small_index];
@ -605,6 +612,11 @@ impl DictCacheEntry {
// ...but we remove the virtual pages from the page pool, effectively reclaiming the physical space.
for vpage in kcache.large_pool_vpages() {
if let Some(pp) = v2p_map.remove(&vpage) {
if paranoid {
let mut noise = [0u8; PAGE_SIZE];
hw.trng_slice(&mut noise);
hw.patch_data(&noise, pp.page_number() * PAGE_SIZE as u32);
}
hw.fast_space_free(pp);
}
}
@ -627,6 +639,9 @@ impl DictCacheEntry {
// a call to sync is necessary to completely flush things, but, we don't sync every time we remove because it's inefficient.
}
/// used to remove a key from the dictionary, syncing 0's to the disk in the key's place
/// sort of less relevant now that the large keys have a paranoid mode; probably this routine should actually
/// be a higher-level function that catches the paranoid request and does an "update" of 0's to the key
/// then does a disk sync and then calls remove
pub fn key_erase(&mut self, name: &str) {
unimplemented!();
}
@ -663,7 +678,7 @@ impl DictCacheEntry {
pub(crate) fn sync_small_pool(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv) {
for (index, entry) in self.small_pool.iter_mut().enumerate() {
if !entry.clean {
let pool_vaddr = VirtAddr::new(self.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START).unwrap();
let pool_vaddr = VirtAddr::new(self.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START + index as u64 * SMALL_CAPACITY as u64).unwrap();
let pp= v2p_map.entry(pool_vaddr).or_insert_with(||
hw.try_fast_space_alloc().expect("No free space to allocate small key storage"));
pp.set_valid(true);

View file

@ -150,26 +150,6 @@ impl KeySmallPool {
clean: true,
}
}
/// Tries to fit the key into the current pool. If there isn't enough space, the function returns false.
/// If `capacity_hint` is provided, it tries to reserve this amount of capacity for the key, even if the
/// data provided is smaller than the hint.
pub(crate) fn try_insert_key(&self, data: &KeyCacheEntry, capacity_hint: Option<usize>) -> bool {
// 1. check if the key will fit
// 2. if so, add its size to the used pool and contents
// 3. if not, return false
false
}
/// Tries to update the key in the pool with the new data. If it doesn't fit,
/// returns Some(key_name). Else it returns None. Note that the data is immediately
/// evicted if it doesn't fit -- the caller must then find a new pool to place the
/// evicted key in.
pub(crate) fn try_update_key(&self, data: &KeyCacheEntry, capacity_hint: Option<usize>) -> Option<String> {
None
}
pub(crate) fn delete_key(&self, data: &KeyCacheEntry) {
// 1. remove the contents string
// 2. update the used tracking information
}
/// Returns the amount of space available in the pool
pub(crate) fn free(&self) -> usize {
self.avail as usize

View file

@ -406,9 +406,43 @@ fn xmain() -> ! {
}
basis_cache.key_update(&mut pddb_os, "system settings", "big_pool1", &bigdata, None, None, None, false).expect("couldn't add a key");
basis_cache.dict_add(&mut pddb_os, "test_dict_2", None).expect("couldn't add test dictionary 2");
basis_cache.key_update(&mut pddb_os, "test_dict_2", "test key in dict 2", "some data".as_bytes(), None, Some(128), None, false).expect("couldn't add a key to second dict");
basis_cache.key_update(&mut pddb_os, "system settings", "wifi/wpa_keys/e4200", "ABC".as_bytes(), Some(2), None, None, false).expect("couldn't update e4200 key");
log::info!("test readback of wifi/wpa_keys/e4200");
match basis_cache.key_read(&mut pddb_os, "system settings", "wifi/wpa_keys/e4200", &mut readback, None, None) {
Ok(readsize) => {
log::info!("read back {} bytes", readsize);
log::info!("read data: {}", String::from_utf8_lossy(&readback));
},
Err(e) => {
log::info!("couldn't read data: {:?}", e);
}
}
#[cfg(not(any(target_os = "none", target_os = "xous")))]
pddb_os.dbg_dump();
log::info!("Re-mount the PDDB");
if let Some(sys_basis) = pddb_os.pddb_mount() {
log::info!("PDDB mount operation finished successfully");
basis_cache.add_basis(sys_basis);
} else {
log::info!("PDDB did not mount; did you remember to format the PDDB region?");
}
log::info!("test readback of wifi/wpa_keys/e4200");
match basis_cache.key_read(&mut pddb_os, "system settings", "wifi/wpa_keys/e4200", &mut readback, None, None) {
Ok(readsize) => {
log::info!("read back {} bytes", readsize);
log::info!("read data: {}", String::from_utf8_lossy(&readback));
},
Err(e) => {
log::info!("couldn't read data: {:?}", e);
}
}
// register a suspend/resume listener
let sr_cid = xous::connect(pddb_sid).expect("couldn't create suspend callback connection");
let susres = susres::Susres::new(&xns, api::Opcode::SuspendResume as u32, sr_cid).expect("couldn't create suspend/resume object");

View file

@ -184,7 +184,7 @@ def main():
class KeyDescriptor:
MAX_NAME_LEN = 95
def __init__(self, record, v2p, disk, key, name):
print(record.hex())
# print(record.hex())
i = 0
self.start = int.from_bytes(record[i:i+8], 'little')
i += 8
@ -245,8 +245,14 @@ class KeyDescriptor:
if self.unresolved:
desc += indent + 'UNRESOLVED'
desc += '\n'
desc += indent + 'Data (hex): {}\n'.format(self.data.hex())
desc += indent + 'Data (txt): {}\n'.format(self.data.decode('utf8', errors='ignore'))
if len(self.data) < 256:
print_len = len(self.data)
extra = ''
else:
print_len = 256
extra = '...'
desc += indent + 'Data (hex): {}{}\n'.format(self.data[:print_len].hex(), extra)
desc += indent + 'Data (txt): {}{}'.format(self.data[:print_len].decode('utf-8', errors='ignore'), extra).replace('\n', '').replace('\r', '') + '\n'
return (desc)
@ -275,7 +281,7 @@ class BasisDicts:
i += 4
self.name = pt_data[i:i+BasisDicts.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
i += BasisDicts.MAX_NAME_LEN
print("dict header len: {}".format(i-4)) # subtract 4 because of the journal
#print("dict header len: {}".format(i-4)) # subtract 4 because of the journal
except ValueError:
print("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr]))
@ -290,7 +296,7 @@ class BasisDicts:
cipher = AES_GCM_SIV(key, pp_data[:12])
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE
print(key_index)
# print(key_index)
maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name)
if maybe_key.valid:
self.keys[maybe_key.name] = maybe_key