very basic small key additions working as expected now

This commit is contained in:
bunnie 2021-11-21 02:21:25 +08:00
parent 05b79f066c
commit 193f563782
6 changed files with 250 additions and 17 deletions

View file

@ -284,6 +284,8 @@ impl BasisCache {
flags: init_flags,
key_count: 0,
free_key_offset: Some(0),
free_key_offset_hint: None,
key_max_free_offset: 1,
small_pool: Vec::<KeySmallPool>::new(),
small_pool_free: BinaryHeap::<KeySmallPoolOrd>::new(),
aad: my_aad,
@ -303,6 +305,80 @@ impl BasisCache {
}
}
pub(crate) fn key_read(&mut self, hw: &mut PddbOs, dict: &str, key: &str, data: &mut [u8],
offset: Option<usize>, basis_name:Option<&str>
) -> Result<usize> {
if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.dicts.contains_key(dict) {
if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) {
if dict_record.flags.valid() {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad);
basis.dicts.insert(dict.to_string(), dcache);
}
} else {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
}
if let Some(dict_entry) = basis.dicts.get_mut(dict) {
if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) {
let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!");
// the key exists, *and* there's sufficient space for the data
if kcache.start < SMALL_POOL_END {
// small pool fetch
// for now, the rule is, if we have a small key, we also have its data in cache.
if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") {
let mut bytes_read = 0;
for (&src, dst) in cache_data.data[offset.unwrap_or(0)..].iter().zip(data.iter_mut()) {
*dst = src;
bytes_read += 1;
}
if bytes_read != data.len() {
log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, bytes_read, data.len());
}
return Ok(bytes_read)
} else {
panic!("Key allocated to small area but its cache data was not of the small type");
}
} else {
// large pool fetch
let mut cur_offset = offset.unwrap_or(0) as u64;
let mut first_iter = true;
while cur_offset < data.len() as u64 {
let start_vpage_addr = ((kcache.start + cur_offset) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64;
if let Some(pp) = basis.v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()) {
let pt_data = hw.data_decrypt_page(&basis.cipher, &basis.aad, pp).expect("Decryption auth error");
if !first_iter {
assert!((cur_offset as usize % VPAGE_SIZE) == 0, "algorithm error in handling offset data");
}
// note: cur_offset % VPAGE_SIZE handles mis-aligned starts; cur_offset will increment until start + cur_offset is
// a multiple of a VPAGE_SIZE, so on later iterations, cur_offset % VPAGE_SIZE should be equal to 0.
for (&src, dst) in
pt_data[size_of::<JournalType>() + cur_offset as usize % VPAGE_SIZE..].iter().zip(data.iter_mut()) {
*dst = src;
cur_offset += 1;
}
first_iter = false;
} else {
log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, cur_offset, data.len());
return Ok(cur_offset as usize)
}
}
return Ok(cur_offset as usize)
}
} else {
return Err(Error::new(ErrorKind::NotFound, "key not found"));
}
} else {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
} else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
}
}
/// Updates a key in a dictionary; if it doesn't exist, creates it. User can specify a basis,
/// or rely upon the auto-basis select algorithm.
pub(crate) fn key_update(&mut self,
@ -413,6 +489,8 @@ impl BasisCache {
basis.dicts.insert(dict.to_string(), dcache);
dict_found = true;
}
} else {
dict_found = true;
}
}
if !dict_found { // now that we're clear of the deep search, mutate the basis if we are sure it's not there
@ -430,6 +508,7 @@ impl BasisCache {
basis.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START))
).expect("couldn't add key");
basis.large_alloc_ptr = Some(updated_ptr);
dict_entry.sync_small_pool(hw, &mut basis.v2p_map, &basis.cipher);
// we don't have large pool caches yet, but this is a placeholder to remember to do "something" at this point,
// once we do have them.
@ -734,14 +813,8 @@ impl BasisCacheEntry {
loop {
// 1. resolve the virtual address to a target page
let cur_vpage = VirtAddr::new(dict_offset.get() + (vpage_num as u64 * VPAGE_SIZE as u64)).unwrap();
if !self.v2p_map.contains_key(&cur_vpage) {
if let Some(pp) = hw.try_fast_space_alloc() {
self.v2p_map.insert(cur_vpage, pp);
} else {
return Err(Error::new(ErrorKind::OutOfMemory, "FastSpace empty"));
}
}
let pp = self.v2p_map.get(&cur_vpage).unwrap();
let pp = self.v2p_map.entry(cur_vpage).or_insert_with(||
hw.try_fast_space_alloc().expect("FastSpace empty")).clone();
// 2(a). fill in the target vpage with data: header special case
let mut dk_vpage = DictKeyVpage::default();
@ -784,7 +857,41 @@ impl BasisCacheEntry {
for (&src, dst) in key_desc.deref().iter().zip(dk_entry.data.iter_mut()) {
*dst = src;
}
dk_vpage.elements[key.descriptor_modulus()] = Some(dk_entry);
// ensure that we have a free_key offset with a perhaps slow brute-force search of the key space
if dict.free_key_offset.is_none() {
let mut index_cache = PlaintextCache { data: None, tag: None };
let mut try_entry = dict.free_key_offset_hint.take().unwrap_or(1) as usize;
while try_entry < KEY_MAXCOUNT {
let req_vaddr = dict.index as u64 * DICT_VSIZE + ((try_entry / DK_PER_VPAGE) as u64) * VPAGE_SIZE as u64;
index_cache.fill(hw, &self.v2p_map, &self.cipher, &dict.aad, VirtAddr::new(req_vaddr).unwrap());
if index_cache.data.is_none() || index_cache.tag.is_none() {
dict.free_key_offset = Some(try_entry as u32);
log::warn!("Dictionary fill op encountered an unallocated page checking entry {} in the dictionary map. Marking it for re-use.", try_entry);
break;
} else {
let cache = index_cache.data.as_ref().expect("Cache should be full, it was already checked...");
let pp = index_cache.tag.as_ref().expect("PP should be in existence, it was already checked...");
let mut keydesc = KeyDescriptor::default();
let start = size_of::<JournalType>() + (try_entry % DK_PER_VPAGE) * DK_STRIDE;
for (&src, dst) in cache[start..start + DK_STRIDE].iter().zip(keydesc.deref_mut().iter_mut()) {
*dst = src;
}
if !keydesc.flags.valid() {
dict.free_key_offset = Some(try_entry as u32);
break;
}
try_entry += 1;
}
}
}
let dk_index = dict.free_key_offset.take().expect("ensure_key_offset() failed");
dict.free_key_offset_hint = Some(dk_index + 1);
if dk_index + 1 > dict.key_max_free_offset {
// only reset the free key offset if it so happened that the free key was the last one: sometimes, the key could be in the middle of the array
dict.key_max_free_offset = dk_index + 2; // set the hint one past the end of the pool
}
dk_vpage.elements[dk_index as usize % DK_PER_VPAGE] = Some(dk_entry);
}
key.clean = true;
}

View file

@ -72,7 +72,12 @@ pub(crate) struct DictCacheEntry {
/// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys`
pub(crate) key_count: u32,
/// a cached copy of the next free key slot, expressed in units of DK_STRIDE, with 0 being the first valid key offset
/// when free_key_offset is taken, it's recommended to add 1 to it and store a copy in free_key_offset_hint.
pub(crate) free_key_offset: Option<u32>,
/// optional hint on where to start searching for a free key offset. If None, we start from the beginning.
pub(crate) free_key_offset_hint: Option<u32>,
/// optimization to terminate key space search if we know there are no more new keys beyond this point
pub(crate) key_max_free_offset: u32,
/// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified.
pub(crate) clean: bool,
/// track modification count
@ -108,6 +113,8 @@ impl DictCacheEntry {
keys: HashMap::<String, KeyCacheEntry>::new(),
key_count: dict.num_keys,
free_key_offset: None,
free_key_offset_hint: None,
key_max_free_offset: KEY_MAXCOUNT as u32, // start with the largest count, adjust down later
clean: true,
age: dict.age,
flags: dict.flags,
@ -177,6 +184,8 @@ impl DictCacheEntry {
}
}
if self.free_key_offset.is_none() { self.free_key_offset = Some(try_entry as u32) }
// need to stop one past the empty slot
self.key_max_free_offset = try_entry as u32 + 1;
// now build the small_pool_free binary heap structure
self.rebuild_free_pool();
@ -188,16 +197,17 @@ impl DictCacheEntry {
/// This shares a lot of code with the fill() routine -- we should condense the common routines
/// to make this easier to maintain. Returns false if the disk was searched and no key was found; true
/// if cache is hot or key was found on search.
fn ensure_key_entry(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv,
pub(crate) fn ensure_key_entry(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv,
name_str: &str) -> bool {
// only fill if the key isn't in the cache.
if !self.keys.contains_key(name_str) {
log::info!("searching for key {}", name_str);
let mut data_cache = PlaintextCache { data: None, tag: None };
let dict_vaddr = VirtAddr::new(self.index as u64 * DICT_VSIZE).unwrap();
let mut try_entry = 1;
let mut key_count = 0;
let mut index_cache = PlaintextCache { data: None, tag: None };
while try_entry < KEY_MAXCOUNT && key_count < self.key_count {
while try_entry < KEY_MAXCOUNT && key_count < self.key_count && try_entry < self.key_max_free_offset as usize {
// cache our decryption data -- there's about 32 entries per page, and the scan is largely linear/sequential, so this should
// be a substantial savings in effort.
// Determine the absolute virtual address of the requested entry. It's written a little weird because
@ -236,6 +246,8 @@ impl DictCacheEntry {
return true;
}
key_count += 1;
} else {
if self.free_key_offset.is_none() { self.free_key_offset = Some(try_entry as u32) }
}
try_entry += 1;
}
@ -314,8 +326,10 @@ impl DictCacheEntry {
/// multiple DictCacheEntry are required.
pub fn key_update(&mut self, hw: &mut PddbOs, v2p_map: &mut HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv,
name: &str, data: &[u8], offset: usize, alloc_hint:Option<usize>, truncate: bool, large_alloc_ptr: PageAlignedVa) -> Result <PageAlignedVa> {
self.age = self.age.saturating_add(1);
self.clean = false;
if self.ensure_key_entry(hw, v2p_map, cipher, name) {
let kcache = self.keys.get_mut(&String::from(name)).expect("Entry was assured, but then not there!");
let kcache = self.keys.get_mut(name).expect("Entry was assured, but then not there!");
// the update isn't going to fit in the reserved space, remove it, and re-insert it with an entirely new entry.
if kcache.reserved < (data.len() + offset) as u64 {
self.key_remove(hw, v2p_map, cipher, name);
@ -398,6 +412,7 @@ impl DictCacheEntry {
// key does not exist (or was previously erased) -- create one or replace the erased one.
// try to fit the key in the small pool first
if ((data.len() + offset) < SMALL_CAPACITY) && (alloc_hint.unwrap_or(0) < SMALL_CAPACITY) {
log::info!("creating small key");
// handle the case that we're a brand new dictionary and no small keys have ever been stored before.
if self.small_pool.len() == 0 {
self.small_pool.push(KeySmallPool::new());
@ -450,7 +465,9 @@ impl DictCacheEntry {
}))
};
self.keys.insert(name.to_string(), kcache);
self.key_count += 1;
} else {
log::info!("creating large key");
// it didn't fit in the small pool, stick it in the big pool.
let reservation = PageAlignedVa::from(
if alloc_hint.unwrap_or(0) > data.len() + offset {
@ -471,6 +488,7 @@ impl DictCacheEntry {
data: None, // no caching implemented yet for large keys
};
self.keys.insert(name.to_string(), kcache);
self.key_count += 1;
// 1. allocate all the pages up to the reservation limit
for vpage_addr in (large_alloc_ptr.as_u64()..large_alloc_ptr.as_u64() + reservation.as_u64()).step_by(VPAGE_SIZE) {
let pp = hw.try_fast_space_alloc().expect("out of disk space");

View file

@ -1200,8 +1200,8 @@ impl PddbOs {
for (&src, dst) in slice_iter.zip(block.iter_mut()) {
*dst = src;
}
let key = Key::from_slice(self.system_basis_key.as_ref().unwrap());
self.data_encrypt_and_patch_page(&Aes256GcmSiv::new(key), &aad, &mut block, &pp);
let key = Key::clone_from_slice(self.system_basis_key.as_ref().unwrap());
self.data_encrypt_and_patch_page(&Aes256GcmSiv::new(&key), &aad, &mut block, &pp);
// step 9. generate & write initial page table entries
for (&virt, phys) in basis_v2p_map.iter_mut() {

View file

@ -90,15 +90,15 @@ pub(crate) struct KeyCacheEntry {
impl KeyCacheEntry {
/// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself.
pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr {
VirtAddr::new(dict_offset.get() + ((self.descriptor_index.unwrap().get() as u64 + 1) * DK_STRIDE as u64)).unwrap()
VirtAddr::new(dict_offset.get() + ((self.descriptor_index.unwrap().get() as u64) * DK_STRIDE as u64)).unwrap()
}
/// Computes the modular position of the KeyDescriptor within a vpage.
pub(crate) fn descriptor_modulus(&self) -> usize {
(self.descriptor_index.unwrap().get() as usize + 1) % (VPAGE_SIZE / DK_STRIDE)
(self.descriptor_index.unwrap().get() as usize) % (VPAGE_SIZE / DK_STRIDE)
}
/// Computes the vpage offset as measured from the start of the dictionary storage region
pub(crate) fn descriptor_vpage_num(&self) -> usize {
(self.descriptor_index.unwrap().get() as usize + 1) / (VPAGE_SIZE / DK_STRIDE)
(self.descriptor_index.unwrap().get() as usize) / (VPAGE_SIZE / DK_STRIDE)
}
/// returns the list of large-pool virtual pages belonging to this entry, if any.
pub(crate) fn large_pool_vpages(&self) -> Vec::<VirtAddr> {

View file

@ -387,6 +387,17 @@ fn xmain() -> ! {
log::info!("adding 'system settings' dictionary");
basis_cache.dict_add(&mut pddb_os, "system settings", None).expect("couldn't add system settings dictionary");
basis_cache.key_update(&mut pddb_os, "system settings", "wifi/wpa_keys/Kosagi", "my_wpa_key_here".as_bytes(), None, None, None, false).expect("couldn't add a key");
let mut readback = [0u8; 15];
match basis_cache.key_read(&mut pddb_os, "system settings", "wifi/wpa_keys/Kosagi", &mut readback, None, None) {
Ok(readsize) => {
log::info!("read back {} bytes", readsize);
log::info!("read data: {}", String::from_utf8_lossy(&readback));
},
Err(e) => {
log::info!("couldn't read data: {:?}", e);
}
}
basis_cache.key_update(&mut pddb_os, "system settings", "wifi/wpa_keys/e4200", "12345678".as_bytes(), None, None, None, false).expect("couldn't add a key");
#[cfg(not(any(target_os = "none", target_os = "xous")))]
pddb_os.dbg_dump();

View file

@ -181,15 +181,84 @@ def main():
except ValueError:
print("couldn't decrypt basis root vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE]))
class KeyDescriptor:
MAX_NAME_LEN = 95
def __init__(self, record, v2p, disk, key, name):
print(record.hex())
i = 0
self.start = int.from_bytes(record[i:i+8], 'little')
i += 8
self.len = int.from_bytes(record[i:i+8], 'little')
i += 8
self.reserved = int.from_bytes(record[i:i+8], 'little')
i += 8
self.flags_code = int.from_bytes(record[i:i+4], 'little')
i += 4
self.age = int.from_bytes(record[i:i+4], 'little')
i += 8
self.name = record[i:i+KeyDescriptor.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
if self.flags_code & 1 != 0:
self.valid = True
else:
self.valid = False
if self.flags_code & 2 != 0:
self.unresolved = True
else:
self.unresolved = False
if self.valid:
page_addr = self.start
self.data = bytearray()
remaining = self.len
read_start = self.start % VPAGE_SIZE # reads can start not page-aligned
while page_addr < self.start + self.len:
data_base = (page_addr // VPAGE_SIZE) * VPAGE_SIZE
pp_start = v2p[data_base]
pp_data = disk[pp_start:pp_start + PAGE_SIZE]
try:
cipher = AES_GCM_SIV(key, pp_data[:12])
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))[4:] # skip the journal, first 4 bytes
if read_start + remaining > VPAGE_SIZE:
read_end = VPAGE_SIZE
else:
read_end = read_start + remaining
self.data += pt_data[read_start:read_end]
bytes_read = read_end - read_start
remaining -= bytes_read
read_start = (read_start + bytes_read) % VPAGE_SIZE
if remaining > 0:
assert(read_start == 0)
except ValueError:
print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(page_addr), pp_start)
page_addr += VPAGE_SIZE
def as_str(self, indent):
desc = ''
desc += indent + 'Start: 0x{:x}\n'.format(self.start)
desc += indent + 'Len: {}\n'.format(self.len)
desc += indent + 'Resvd: 0x{:x}\n'.format(self.reserved)
desc += indent + 'Age: {}\n'.format(self.age)
desc += indent + 'Flags: '
if self.valid:
desc += 'VALID'
if self.unresolved:
desc += indent + 'UNRESOLVED'
desc += '\n'
desc += indent + 'Data (hex): {}\n'.format(self.data.hex())
desc += indent + 'Data (txt): {}\n'.format(self.data.decode('utf8', errors='ignore'))
return (desc)
class BasisDicts:
DICT_VSTRIDE = 0xFE_0000
KV_STRIDE = 127
MAX_NAME_LEN = 115
def __init__(self, index, v2p, disk, key, name):
global PAGE_SIZE
dict_header_vaddr = self.DICT_VSTRIDE * (index + 1)
pp = v2p[dict_header_vaddr]
self.keys = {}
try:
print('dict decrypt @ pp {:x}, nonce: {}'.format(pp, disk[pp:pp+12].hex()))
cipher = AES_GCM_SIV(key, disk[pp:pp+12])
@ -210,12 +279,40 @@ class BasisDicts:
except ValueError:
print("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr]))
if self.num_keys > 0:
keys_found = 0
keys_tried = 1
while (keys_found < self.num_keys) and keys_tried < 131071:
keyindex_start_vaddr = self.DICT_VSTRIDE * (index + 1) + (keys_tried * self.KV_STRIDE)
pp_start = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE]
pp_data = disk[pp_start:pp_start + PAGE_SIZE]
try:
cipher = AES_GCM_SIV(key, pp_data[:12])
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE
print(key_index)
maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name)
if maybe_key.valid:
self.keys[maybe_key.name] = maybe_key
keys_found += 1
except ValueError:
print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr), pp_start)
keys_tried += 1
def as_str(self):
desc = ''
desc += ' Name: {}\n'.format(self.name)
desc += ' Age: {}\n'.format(self.age)
desc += ' Flags: {}\n'.format(self.flags)
desc += ' Key count: {}\n'.format(self.num_keys)
desc += ' Keys:\n'
for (name, key) in self.keys.items():
desc += ' {}:\n'.format(name)
desc += key.as_str(' ')
return desc