mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-15 08:30:02 +00:00
fixes for patch update test
lots of corner cases found and ifxed
This commit is contained in:
parent
472d60ae95
commit
42e926fcab
5 changed files with 156 additions and 21 deletions
|
|
@ -368,40 +368,82 @@ impl BasisCache {
|
|||
// for now, the rule is, if we have a small key, we also have its data in cache.
|
||||
if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") {
|
||||
let mut bytes_read = 0;
|
||||
for (&src, dst) in cache_data.data[offset.unwrap_or(0)..].iter().zip(data.iter_mut()) {
|
||||
if offset.unwrap_or(0) as u64 > kcache.len {
|
||||
return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length"));
|
||||
}
|
||||
for (&src, dst) in cache_data.data[offset.unwrap_or(0)..kcache.len as usize].iter().zip(data.iter_mut()) {
|
||||
*dst = src;
|
||||
bytes_read += 1;
|
||||
}
|
||||
if bytes_read != data.len() {
|
||||
log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, bytes_read, data.len());
|
||||
log::debug!("Key shorter than read buffer {}:{} ({}/{})", dict, key, bytes_read, data.len());
|
||||
}
|
||||
return Ok(bytes_read)
|
||||
} else {
|
||||
panic!("Key allocated to small area but its cache data was not of the small type");
|
||||
}
|
||||
} else {
|
||||
if offset.unwrap_or(0) as u64 > kcache.len {
|
||||
return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length"));
|
||||
}
|
||||
if data.len() == 0 { // mostly because i don't want to have to think about this case in the later logic.
|
||||
return Ok(0)
|
||||
}
|
||||
// large pool fetch
|
||||
let mut cur_offset = offset.unwrap_or(0) as u64;
|
||||
let mut first_iter = true;
|
||||
while cur_offset < data.len() as u64 {
|
||||
let mut blocks_read = 0;
|
||||
loop {
|
||||
let start_vpage_addr = ((kcache.start + cur_offset) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64;
|
||||
// handle the case of terminating the last block read nicely:
|
||||
// we can either end because we hit the end of the key data (so the data readback buffer is too large)
|
||||
let key_endstop = if (kcache.len - cur_offset) < VPAGE_SIZE as u64 {
|
||||
(kcache.len - cur_offset) as usize + size_of::<JournalType>()
|
||||
} else {
|
||||
VPAGE_SIZE + size_of::<JournalType>()
|
||||
};
|
||||
// or we can hit the end because we ran out of datareadback buffer
|
||||
let data_endstop = if (data.len() as u64 - cur_offset) < VPAGE_SIZE as u64 {
|
||||
(data.len() as u64 - cur_offset) as usize + size_of::<JournalType>()
|
||||
} else {
|
||||
VPAGE_SIZE + size_of::<JournalType>()
|
||||
};
|
||||
// of the two possible end criteria, pick the nearest one
|
||||
let endstop = if key_endstop < data_endstop {
|
||||
key_endstop
|
||||
} else {
|
||||
data_endstop
|
||||
};
|
||||
log::trace!("reading offset {}/{}:{}", cur_offset, endstop, kcache.len);
|
||||
let mut read_offset = (cur_offset % VPAGE_SIZE as u64) as usize;
|
||||
if let Some(pp) = basis.v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()) {
|
||||
let pt_data = hw.data_decrypt_page(&basis.cipher, &basis.aad, pp).expect("Decryption auth error");
|
||||
if !first_iter {
|
||||
assert!((cur_offset as usize % VPAGE_SIZE) == 0, "algorithm error in handling offset data");
|
||||
if blocks_read != 0 {
|
||||
assert!(read_offset == 0, "algorithm error in handling offset data");
|
||||
}
|
||||
// note: cur_offset % VPAGE_SIZE handles mis-aligned starts; cur_offset will increment until start + cur_offset is
|
||||
// a multiple of a VPAGE_SIZE, so on later iterations, cur_offset % VPAGE_SIZE should be equal to 0.
|
||||
for (&src, dst) in
|
||||
pt_data[size_of::<JournalType>() + cur_offset as usize % VPAGE_SIZE..].iter().zip(data.iter_mut()) {
|
||||
pt_data[
|
||||
size_of::<JournalType>() // always this fixed offset per block
|
||||
+ read_offset // this should be 0 after the first block
|
||||
..endstop
|
||||
].iter().zip(data[cur_offset as usize..].iter_mut()) {
|
||||
*dst = src;
|
||||
cur_offset += 1;
|
||||
read_offset += 1;
|
||||
}
|
||||
first_iter = false;
|
||||
blocks_read += 1;
|
||||
} else {
|
||||
log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, cur_offset, data.len());
|
||||
return Ok(cur_offset as usize)
|
||||
}
|
||||
// cur_offset won't hit (start + len) if we're hitting an endstop, so the loop won't exit.
|
||||
// if we hit an endstop, we should exit the read loop.
|
||||
if endstop != VPAGE_SIZE + size_of::<JournalType>() {
|
||||
break;
|
||||
}
|
||||
// break if we've hit or exceeded our endstop
|
||||
if read_offset >= endstop {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return Ok(cur_offset as usize)
|
||||
}
|
||||
|
|
@ -581,6 +623,31 @@ impl BasisCache {
|
|||
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn key_attributes(&mut self, hw: &mut PddbOs, dict: &str, key: &str, basis_name: Option<&str>) -> Result<KeyAttributes> {
|
||||
if let Some(basis_index) = self.select_basis(basis_name) {
|
||||
let basis = &mut self.cache[basis_index];
|
||||
if !basis.ensure_dict_in_cache(hw, dict) {
|
||||
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
|
||||
} else {
|
||||
let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!");
|
||||
if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) {
|
||||
let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!");
|
||||
Ok(KeyAttributes {
|
||||
len: kcache.len as usize,
|
||||
reserved: kcache.reserved as usize,
|
||||
age: kcache.age as usize,
|
||||
dict: dict.to_string(),
|
||||
basis: (&basis.name).to_string(),
|
||||
})
|
||||
} else {
|
||||
return Err(Error::new(ErrorKind::NotFound, "key not found"));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This is the RAM cached copy of a basis as maintained in the PDDB.
|
||||
|
|
@ -892,7 +959,12 @@ impl BasisCacheEntry {
|
|||
|
||||
// this is the virtual page within the dictionary region that we're currently serializing
|
||||
let mut vpage_num = 0;
|
||||
let mut loopcheck= 0;
|
||||
loop {
|
||||
loopcheck += 1;
|
||||
if loopcheck == 256 {
|
||||
log::warn!("potential infinite loop detected sync dict {}", name);
|
||||
}
|
||||
// 1. resolve the virtual address to a target page
|
||||
let cur_vpage = VirtAddr::new(dict_offset.get() + (vpage_num as u64 * VPAGE_SIZE as u64)).unwrap();
|
||||
let pp = self.v2p_map.entry(cur_vpage).or_insert_with(||
|
||||
|
|
@ -919,9 +991,9 @@ impl BasisCacheEntry {
|
|||
let next_vpage = VirtAddr::new(cur_vpage.get() + VPAGE_SIZE as u64).unwrap();
|
||||
for (key_name, key) in dict.keys.iter_mut() {
|
||||
if !key.clean {
|
||||
log::info!("merging in key {}", key_name);
|
||||
if key.descriptor_vaddr(dict_offset) >= cur_vpage &&
|
||||
key.descriptor_vaddr(dict_offset) < next_vpage {
|
||||
log::info!("merging in key {}", key_name);
|
||||
// key is within the current page, add it to the target list
|
||||
let mut dk_entry = DictKeyEntry::default();
|
||||
let mut kn = [0u8; KEY_NAME_LEN];
|
||||
|
|
@ -941,6 +1013,8 @@ impl BasisCacheEntry {
|
|||
}
|
||||
dk_vpage.elements[key.descriptor_index.get() as usize % DK_PER_VPAGE] = Some(dk_entry);
|
||||
key.clean = true;
|
||||
} else {
|
||||
log::debug!("proposed key fell outside of our vpage: {} vpage{:x}/vaddr{:x}", key_name, cur_vpage.get(), key.descriptor_vaddr(dict_offset));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -979,6 +1053,7 @@ impl BasisCacheEntry {
|
|||
break;
|
||||
}
|
||||
}
|
||||
log::info!("done syncing dict");
|
||||
dict.clean = true;
|
||||
}
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -336,13 +336,27 @@ impl DictCacheEntry {
|
|||
} else {
|
||||
kcache.age = kcache.age.saturating_add(1);
|
||||
kcache.clean = false;
|
||||
if data.len() == 4 {
|
||||
use std::convert::TryInto;
|
||||
log::info!("patching checksum: {:x} at offset {}", u32::from_le_bytes(data.try_into().unwrap()), offset);
|
||||
}
|
||||
// 1. handle unaligned start offsets
|
||||
let mut written: usize = 0;
|
||||
if ((kcache.start + offset as u64) % VPAGE_SIZE as u64) != 0 {
|
||||
if ((kcache.start + offset as u64 + written as u64) % VPAGE_SIZE as u64) != 0 {
|
||||
let start_vpage_addr = ((kcache.start + offset as u64) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64;
|
||||
let pp = v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()).expect("large key data allocation missing");
|
||||
let mut pt_data = hw.data_decrypt_page(&cipher, &self.aad, pp).expect("Decryption auth error");
|
||||
for (&src, dst) in data[written..].iter().zip(pt_data[size_of::<JournalType>() + offset..].iter_mut()) {
|
||||
let mut pt_data = match hw.data_decrypt_page(&cipher, &self.aad, pp) {
|
||||
Some(data) => data,
|
||||
None => {
|
||||
// perhaps we could just "zero out" previously data, but for now, let's make this undefined behavior
|
||||
log::warn!("Attempt to patch data outside of the previously initialized length. This is an error.");
|
||||
return Err(Error::new(ErrorKind::UnexpectedEof, "patching outside of previously written data"));
|
||||
}
|
||||
};
|
||||
if offset > 0 {
|
||||
log::info!("patching offset {}, total length {}, data length {}", offset % VPAGE_SIZE, kcache.len, data.len());
|
||||
}
|
||||
for (&src, dst) in data[written..].iter().zip(pt_data[size_of::<JournalType>() + (offset % VPAGE_SIZE)..].iter_mut()) {
|
||||
*dst = src;
|
||||
written += 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -175,3 +175,17 @@ impl PartialEq for KeySmallPoolOrd {
|
|||
self.avail == other.avail
|
||||
}
|
||||
}
|
||||
|
||||
/// A structure for passing around key metadata
|
||||
pub(crate) struct KeyAttributes {
|
||||
/// actual length of data in the key
|
||||
pub(crate) len: usize,
|
||||
/// pre-reserved storage space for the key (growable to this bound "at no cost")
|
||||
pub(crate) reserved: usize,
|
||||
/// access count
|
||||
pub(crate) age: usize,
|
||||
/// owning dictionary
|
||||
pub(crate) dict: String,
|
||||
/// the specific basis from which this key's metadata was found
|
||||
pub(crate) basis: String,
|
||||
}
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
use rand::Rng;
|
||||
use crate::*;
|
||||
|
||||
const UPPER_BOUND: usize = 9000;
|
||||
|
||||
fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bound: usize) -> (String, Vec::<u8>) {
|
||||
let mut rng = rand::thread_rng();
|
||||
// we want roughly half our keys to be in the small bin, and half in the large bin
|
||||
|
|
@ -40,7 +42,7 @@ pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCach
|
|||
|
||||
let num_dicts = maybe_num_dicts.unwrap_or(4);
|
||||
let num_keys = maybe_num_keys.unwrap_or(34);
|
||||
let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, 9000));
|
||||
let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, UPPER_BOUND - 4)); // 4 bytes for the CI checksum
|
||||
|
||||
// make all the directories first
|
||||
for dictnum in 1..=num_dicts {
|
||||
|
|
@ -101,12 +103,28 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache,
|
|||
for dict in dict_list.iter() {
|
||||
if let Ok(key_list) = basis_cache.key_list(hw, dict) {
|
||||
for key in key_list.iter() {
|
||||
let mut patchbuf = [0u8; UPPER_BOUND];
|
||||
// this actually does something a bit more complicated on a multi-basis system than you'd think:
|
||||
// it will get the union of all key names, and then patch the *latest open basis* only with new data.
|
||||
// note: if the key didn't already exist in the latest open basis, it's added, with just that patch data in it.
|
||||
// to override this behavior, you'd want to specify a _specific_ basis, but for testing purposes, we only have one
|
||||
// so we're doing this way that would lead to surprising results if it were copied as template code elsewhere.
|
||||
basis_cache.key_update(hw, dict, key, patch_data.as_bytes(), Some(patch_offset), None, None, false).unwrap();
|
||||
|
||||
// now fix the CI checksum. structured as two separate patches. not because it's efficient,
|
||||
// but because it exercises the code harder.
|
||||
let readlen = basis_cache.key_read(hw, dict, key, &mut patchbuf, Some(0), None).unwrap();
|
||||
// now extend and compute the checksum
|
||||
let mut checkdata = Vec::<u8>::new();
|
||||
for &b in &patchbuf[..readlen-4] {
|
||||
checkdata.push(b);
|
||||
}
|
||||
log::trace!("checkdata len: {}", checkdata.len());
|
||||
while checkdata.len() % 4 != 0 {
|
||||
checkdata.push(0);
|
||||
}
|
||||
let checksum = murmur3_32(&checkdata, 0);
|
||||
basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen-4), None, None, false).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -163,6 +163,7 @@ def main():
|
|||
print("Basis {}".format(name))
|
||||
v2p_table = tables[name][0]
|
||||
p2v_table = tables[name][1]
|
||||
# v2p_table[0xfe0fe0] = 0x1200* 0x100
|
||||
|
||||
basis_data = bytearray()
|
||||
pp_start = v2p_table[VPAGE_SIZE]
|
||||
|
|
@ -207,6 +208,7 @@ def main():
|
|||
except ValueError:
|
||||
print("couldn't decrypt basis root vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE]))
|
||||
|
||||
PRINTED_FULL = False
|
||||
class KeyDescriptor:
|
||||
MAX_NAME_LEN = 95
|
||||
def __init__(self, record, v2p, disk, key, name):
|
||||
|
|
@ -271,6 +273,7 @@ class KeyDescriptor:
|
|||
|
||||
def as_str(self, indent=''):
|
||||
PRINT_LEN = 64
|
||||
global PRINTED_FULL
|
||||
desc = ''
|
||||
desc += indent + 'Start: 0x{:x}\n'.format(self.start)
|
||||
desc += indent + 'Len: {}/0x{:x}\n'.format(self.len, self.reserved)
|
||||
|
|
@ -283,12 +286,17 @@ class KeyDescriptor:
|
|||
desc += ' | Age: {}'.format(self.age)
|
||||
desc += ' | CI OK: {}'.format(self.ci_ok)
|
||||
desc += '\n'
|
||||
if len(self.data) < 64:
|
||||
if len(self.data) < PRINT_LEN:
|
||||
print_len = len(self.data)
|
||||
extra = ''
|
||||
else:
|
||||
print_len = 64
|
||||
extra = '...'
|
||||
if (PRINTED_FULL == False) and (self.ci_ok == False):
|
||||
print_len = len(self.data)
|
||||
extra = ''
|
||||
PRINTED_FULL = True
|
||||
else:
|
||||
print_len = PRINT_LEN
|
||||
extra = '...'
|
||||
desc += indent + 'Data (hex): {}{}\n'.format(self.data[:print_len].hex(), extra)
|
||||
desc += indent + 'Data (txt): {}{}'.format(make_printable(self.data[:print_len].decode('utf-8', errors='ignore')), extra) + '\n'
|
||||
return (desc)
|
||||
|
|
@ -347,7 +355,7 @@ class BasisDicts:
|
|||
keys_found += 1
|
||||
|
||||
except ValueError:
|
||||
print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr), pp_start)
|
||||
print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr, pp_start))
|
||||
keys_tried += 1
|
||||
except KeyError:
|
||||
# the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried"
|
||||
|
|
@ -504,9 +512,15 @@ class Pte:
|
|||
def find_mbbb(mbbb):
|
||||
global MBBB_PAGES
|
||||
pages = [mbbb[i:i+PAGE_SIZE] for i in range(0, MBBB_PAGES * PAGE_SIZE, PAGE_SIZE)]
|
||||
candidates = []
|
||||
for page in pages:
|
||||
if bytearray(page[:16]) != bytearray([0xff] * 16):
|
||||
return page
|
||||
candidates.append(page)
|
||||
|
||||
if len(candidates) > 1:
|
||||
print("More than one MBBB found, this is an error!")
|
||||
else:
|
||||
return candidates[0]
|
||||
return None
|
||||
|
||||
def decode_pagetable(img, entries, keys, mbbb):
|
||||
|
|
@ -577,7 +591,7 @@ class PhysPage:
|
|||
return (self.pp >> 24) & 0xF
|
||||
|
||||
def as_str(self):
|
||||
return 'a_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal())
|
||||
return 'pp_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal())
|
||||
|
||||
class Fscb:
|
||||
def __init__(self, i_bytes):
|
||||
|
|
|
|||
Loading…
Reference in a new issue