saymrwulf/xous-core
1
0
Fork 0
mirror of https://github.com/saymrwulf/xous-core.git synced 2026-11-14 08:03:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

dictionary adding now seems to work

Browse source
This commit is contained in:
bunnie 2021-11-18 02:01:30 +08:00
parent 507d99ee41
commit c62b2a25fb
5 changed files with 120 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
services/pddb/src/backend/basis.rs
View file

@ -160,6 +160,10 @@ impl DerefMut for BasisRoot {
}
/// A list of open Basis that we can use to search and operate upon. Sort of the "root" data structure of the PDDB.
///
/// Note to self: it's tempting to integrate the "hw" parameter (the pointer to the PddbOs structure). However, this
/// results in interior mutability problems. I guess we could wrap it in a Rc or RefCell or something like that; but
/// the inconvenience of passing the hw structure around doesn't seem too bad so far...
pub(crate) struct BasisCache {
/// the cache entries themselves
cache: Vec::<BasisCacheEntry>,
@ -224,8 +228,8 @@ impl BasisCache {
index: dict_index,
keys: HashMap::<String, KeyCacheEntry>::new(),
clean: false,
age: 0,
flags: 0,
age: 111,
flags: 222,
key_count: 0,
};
log::info!("adding dictionary {}", name);
@ -235,6 +239,8 @@ impl BasisCache {
basis.dict_sync(hw, name)?;
// sync the root basis structure as well, while we're at it...
basis.basis_sync(hw);
// finally, sync the page tables.
basis.pt_sync(hw);
Ok(())
} else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
@ -387,6 +393,17 @@ impl BasisCacheEntry {
}
}
/// Looks for dirty entries in the page table, and flushes them to disk.
pub(crate) fn pt_sync(&mut self, hw: &mut PddbOs) {
for (&virt, phys) in self.v2p_map.iter_mut() {
if !phys.clean() {
log::info!("syncing dirty pte va: {:x?} pa: {:x?}", virt, phys);
hw.pt_patch_mapping(virt, phys.page_number());
phys.set_clean(true);
}
}
}
/// This will sync the named Dictionary header + dirty *key descriptors*. It does not
/// sync the key data itself. Note this does not mark the surrounding basis structure as clean
/// when it exits, even if there are no more dirty entries within.
@ -410,6 +427,8 @@ impl BasisCacheEntry {
num_keys: dict.key_count,
name: dict_name,
};
log::info!("syncing dict: {:?}", dict_disk);
// log::info!("raw: {:x?}", dict_disk.deref());
// observation: all keys to be flushed to disk will be in the KeyCacheEntry. Some may be clean,
// but definitely all the dirty ones are in there (if they aren't, where else would they be??)
@ -447,6 +466,7 @@ impl BasisCacheEntry {
// the assumption that the KeyCacheEntry doesn't ever get to a very large N.
let next_vpage = VirtAddr::new(cur_vpage.get() + VPAGE_SIZE as u64).unwrap();
for (key_name, key) in dict.keys.iter_mut() {
log::info!("merging in key {}", key_name);
if !key.clean {
if key.descriptor_vaddr(dict_offset) >= cur_vpage &&
key.descriptor_vaddr(dict_offset) < next_vpage {
@ -501,7 +521,7 @@ impl BasisCacheEntry {
aad: &self.aad,
};
let ciphertext = self.cipher.encrypt(&nonce, payload).expect("failed to encrypt DictKeys");
hw.patch_data(&ciphertext, pp.page_number() * PAGE_SIZE as u32);
hw.patch_data(&[nonce.as_slice(), &ciphertext].concat(), pp.page_number() * PAGE_SIZE as u32);
// 4. Check for dirty keys, if there are still some, update vpage_num to target them; otherwise
// exit the loop
@ -562,7 +582,7 @@ impl BasisCacheEntry {
msg: &block,
}
).unwrap();
hw.patch_data(&ciphertext, pp.page_number() * PAGE_SIZE as u32);
hw.patch_data(&[nonce.as_slice(), &ciphertext].concat(), pp.page_number() * PAGE_SIZE as u32);
}
}
@ -585,6 +605,7 @@ pub(crate) struct DictCacheEntry {
pub(crate) flags: u32,
}
#[derive(Debug)]
/// On-disk representation of the dictionary header.
#[repr(C, align(8))]
pub(crate) struct Dictionary {

3
services/pddb/src/backend/fastspace.rs
View file

@ -153,8 +153,9 @@ impl DerefMut for SpaceUpdate {
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_fast_space_size() {
assert!(core::mem::size_of::<FastSpaceInFlash>() & (PAGE_SIZE - 1) == 0, "FastSpaceInFlash is not exactly a multiple of one page in size");

43
services/pddb/src/backend/hw.rs
View file

@ -77,6 +77,8 @@ pub(crate) struct PddbOs {
fscb_phys_base: PageAlignedPa,
data_phys_base: PageAlignedPa,
system_basis_key: Option<[u8; AES_KEYSIZE]>,
/// derived cipher for encrypting PTEs -- cache it, so we can save the time cost of constructing the cipher key schedule
cipher_ecb: Option<Aes256>,
/// fast space cache
fspace_cache: HashSet<PhysPage>,
/// memoize the location of the fscb log pages
@ -135,6 +137,7 @@ impl PddbOs {
fscb_phys_base,
data_phys_base: PageAlignedPa::from(fscb_phys_base.as_u32() + FSCB_PAGES as u32 * PAGE_SIZE as u32),
system_basis_key: None,
cipher_ecb: None,
//v2p_map: HashMap::<String, HashMap<VirtAddr, PhysPage>>::new(),
fspace_cache: HashSet::<PhysPage>::new(),
fspace_log_addrs: Vec::<PageAlignedPa>::new(),
@ -158,6 +161,7 @@ impl PddbOs {
fscb_phys_base,
data_phys_base: PageAlignedPa::from(fscb_phys_base.as_u32() + FSCB_PAGES as u32 * PAGE_SIZE as u32),
system_basis_key: None,
cipher_ecb: None,
//v2p_map: HashMap::<String, HashMap<VirtAddr, PhysPage>>::new(),
fspace_cache: HashSet::<PhysPage>::new(),
fspace_log_addrs: Vec::<PageAlignedPa>::new(),
@ -301,6 +305,20 @@ impl PddbOs {
).expect("couldn't burn fscb");
}
/// Public function that "does the right thing" for patching in a page table entry based on a virtual to
/// physical mapping. Note that the `va` is an *address* (in units of bytes), and the `phy_page_num` is
/// a *page number* (so a physical address divided by the page size). It's a slightly awkward units, but
/// it saves a bit of math going back and forth between the native storage formats of the records.
pub(crate) fn pt_patch_mapping(&mut self, va: VirtAddr, phys_page_num: u32) {
self.syskey_ensure();
let cipher = self.cipher_ecb.as_ref().expect("Inconsistent internal state - syskey_ensure() failed");
let mut pte = Pte::new(va, PtFlags::CLEAN, Rc::clone(&self.entropy));
let mut block = Block::from_mut_slice(pte.deref_mut());
//log::info!("pte pt: {:x?}", block);
cipher.encrypt_block(&mut block);
//log::info!("pte ct: {:x?}", block);
self.patch_pagetable(&block, phys_page_num * aes::BLOCK_SIZE as u32);
}
/// Searches the page table for an MBBB slot. This is currently an O(N) search but
/// in practice for Precursor there are only 8 pages, so it's quite fast on average.
/// This would want to be optimized or cached for a much larger filesystem.
@ -344,6 +362,7 @@ impl PddbOs {
if let Some(pte) = Pte::try_from_slice(block.as_slice()) {
let mut pp = PhysPage(0);
pp.set_page_number(((page_index * PAGE_SIZE / aes::BLOCK_SIZE) + index) as PhysAddr);
// the state is clean because this entry is, by definition, synchronized with the disk
pp.set_clean(true);
// handle conflicting journal versions here
if let Some(prev_page) = map.get(&pte.vaddr()) {
@ -399,6 +418,7 @@ impl PddbOs {
}
core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst);
}
self.cipher_ecb = None;
}
fn syskey_ensure(&mut self) {
if self.system_basis_key.is_none() {
@ -412,6 +432,10 @@ impl PddbOs {
}
self.system_basis_key = Some(system_key);
}
if self.cipher_ecb.is_none() {
let cipher = Aes256::new(GenericArray::from_slice(&self.system_basis_key.unwrap()));
self.cipher_ecb = Some(cipher);
}
}
fn mbbb_as_slice(&self) -> &[u8] {
@ -821,6 +845,7 @@ impl PddbOs {
// the entry will be reclaimed on the next full-space scan. This is a less-bad outcome than filling up
// the log with 2x the number of operations to record MaybeUsed and then Used.
ppc.set_space_state(SpaceState::Used);
ppc.set_clean(false); // the allocated page is not clean, because it hasn't been written to disk
ppc.set_journal(pp.journal() + 1); // this is guaranteed not to overflow because of a check in the "if" clause above
// commit the usage to the journal
@ -924,7 +949,7 @@ impl PddbOs {
Some(data)
},
Err(e) => {
log::info!("Error decrypting page: {:?}", e);
log::trace!("Error decrypting page: {:?}", e); // sometimes this is totally "normal", like when we're testing for valid data.
None
}
}
@ -1151,18 +1176,10 @@ impl PddbOs {
}
// step 9. generate & write initial page table entries
if let Some(system_key) = self.system_basis_key {
log::info!("encryption key: {:x?}", system_key);
let cipher = Aes256::new(GenericArray::from_slice(&system_key));
for (&virt, &phys) in basis_v2p_map.iter() {
let mut pte = Pte::new(virt, PtFlags::CLEAN, Rc::clone(&self.entropy));
let mut block = Block::from_mut_slice(pte.deref_mut());
//log::info!("pte pt: {:x?}", block);
cipher.encrypt_block(&mut block);
//log::info!("pte ct: {:x?}", block);
self.patch_pagetable(&block, phys.page_number() * aes::BLOCK_SIZE as u32);
//log::info!("pte ct loc: {:x?}", phys.page_number() * aes::BLOCK_SIZE as u32);
}
for (&virt, phys) in basis_v2p_map.iter_mut() {
self.pt_patch_mapping(virt, phys.page_number());
// mark the entry as clean, as it has been sync'd to disk
phys.set_clean(true);
}
Ok(())

5
services/pddb/src/backend/types.rs
View file

@ -176,8 +176,9 @@ impl Add for PageAlignedPa {
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
/// PAGE_SIZE is required to be a power of two. 0x1000 -> 0x1000 - 1 = 0xFFF, which forms the bitmasks.
fn test_page_size() {
@ -190,7 +191,7 @@ mod tests {
/// obvious but not explicitly stated fact always remains true.
#[test]
fn test_bitfield_bool() {
super::bitfield! {
bitfield! {
pub struct Test(u8);
impl Debug;
pub test, set_test: 1;

76
tools/pddbdbg.py
View file

@ -10,6 +10,7 @@ import hashlib
SYSTEM_BASIS = '.System'
PAGE_SIZE = 4096
VPAGE_SIZE = 4064
MBBB_PAGES = 10
# from https://github.com/wc-duck/pymmh3/blob/master/pymmh3.py
@ -130,6 +131,7 @@ def main():
FSCB_PAGES = 16
KEY_PAGES = 1
global PAGE_SIZE
global VPAGE_SIZE
with open(imagefile, 'rb') as img_f:
raw_img = img_f.read()
@ -155,26 +157,68 @@ def main():
p2v_table = tables[name][1]
basis_data = bytearray()
errors = 0
for vp in sorted(v2p_table):
pp_start = v2p_table[vp]
# print("pp_start: {}, vp: {}".format(pp_start, vp))
pp_data = data[pp_start:pp_start + PAGE_SIZE]
try:
cipher = AES_GCM_SIV(key, pp_data[:12])
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
basis_data.extend(bytearray(pt_data))
print("decrypted vpage @ {} ppage @ {:x}".format(vp, v2p_table[vp]))
except ValueError:
errors += 1
print("couldn't decrypt vpage @ {} ppage @ {:x}".format(vp, v2p_table[vp]))
if errors == 0:
pp_start = v2p_table[VPAGE_SIZE]
# print("pp_start: {}, vp: {}".format(pp_start, vp))
pp_data = data[pp_start:pp_start + PAGE_SIZE]
try:
cipher = AES_GCM_SIV(key, pp_data[:12])
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
basis_data.extend(bytearray(pt_data))
print("decrypted vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE]))
# print([hex(x) for x in basis_data[:256]])
basis = Basis(basis_data)
print(basis.as_str())
basis_dicts = {}
for dict_index in range(basis.num_dicts):
bdict = BasisDicts(dict_index, v2p_table, data, key, name)
basis_dicts[bdict.name] = bdict
print(" Dictionaries: ")
for bdict in basis_dicts.values():
print(bdict.as_str())
except ValueError:
print("couldn't decrypt basis root vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE]))
class BasisDicts:
DICT_VSTRIDE = 0xFE_0000
MAX_NAME_LEN = 115
def __init__(self, index, v2p, disk, key, name):
global PAGE_SIZE
dict_header_vaddr = self.DICT_VSTRIDE * (index + 1)
pp = v2p[dict_header_vaddr]
try:
print('dict decrypt @ pp {:x}, nonce: {}'.format(pp, disk[pp:pp+12].hex()))
cipher = AES_GCM_SIV(key, disk[pp:pp+12])
pt_data = cipher.decrypt(disk[pp+12:pp+PAGE_SIZE], basis_aad(name))
# print('raw pt_data: {}'.format(pt_data[:127].hex()))
i = 0
self.journal = int.from_bytes(pt_data[i:i+4], 'little')
i += 4
self.flags = int.from_bytes(pt_data[i:i+4], 'little')
i += 4
self.age = int.from_bytes(pt_data[i:i+4], 'little')
i += 4
self.num_keys = int.from_bytes(pt_data[i:i+4], 'little')
i += 4
self.name = pt_data[i:i+BasisDicts.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
i += BasisDicts.MAX_NAME_LEN
print("dict header len: {}".format(i-4)) # subtract 4 because of the journal
except ValueError:
print("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr]))
def as_str(self):
desc = ''
desc += ' Name: {}\n'.format(self.name)
desc += ' Age: {}\n'.format(self.age)
desc += ' Flags: {}\n'.format(self.flags)
desc += ' Key count: {}\n'.format(self.num_keys)
return desc
class Basis:
MAX_NAME_LEN = 64
def __init__(self, i_bytes):