fixes for patch update test

lots of corner cases found and ifxed
This commit is contained in:
bunnie 2021-11-23 23:33:06 +08:00
parent 472d60ae95
commit 42e926fcab
5 changed files with 156 additions and 21 deletions

View file

@ -368,40 +368,82 @@ impl BasisCache {
// for now, the rule is, if we have a small key, we also have its data in cache.
if let KeyCacheData::Small(cache_data) = kcache.data.as_mut().expect("small pool should all have their data 'hot' if the index entry is also in cache") {
let mut bytes_read = 0;
for (&src, dst) in cache_data.data[offset.unwrap_or(0)..].iter().zip(data.iter_mut()) {
if offset.unwrap_or(0) as u64 > kcache.len {
return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length"));
}
for (&src, dst) in cache_data.data[offset.unwrap_or(0)..kcache.len as usize].iter().zip(data.iter_mut()) {
*dst = src;
bytes_read += 1;
}
if bytes_read != data.len() {
log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, bytes_read, data.len());
log::debug!("Key shorter than read buffer {}:{} ({}/{})", dict, key, bytes_read, data.len());
}
return Ok(bytes_read)
} else {
panic!("Key allocated to small area but its cache data was not of the small type");
}
} else {
if offset.unwrap_or(0) as u64 > kcache.len {
return Err(Error::new(ErrorKind::UnexpectedEof, "offest requested is beyond the key length"));
}
if data.len() == 0 { // mostly because i don't want to have to think about this case in the later logic.
return Ok(0)
}
// large pool fetch
let mut cur_offset = offset.unwrap_or(0) as u64;
let mut first_iter = true;
while cur_offset < data.len() as u64 {
let mut blocks_read = 0;
loop {
let start_vpage_addr = ((kcache.start + cur_offset) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64;
// handle the case of terminating the last block read nicely:
// we can either end because we hit the end of the key data (so the data readback buffer is too large)
let key_endstop = if (kcache.len - cur_offset) < VPAGE_SIZE as u64 {
(kcache.len - cur_offset) as usize + size_of::<JournalType>()
} else {
VPAGE_SIZE + size_of::<JournalType>()
};
// or we can hit the end because we ran out of datareadback buffer
let data_endstop = if (data.len() as u64 - cur_offset) < VPAGE_SIZE as u64 {
(data.len() as u64 - cur_offset) as usize + size_of::<JournalType>()
} else {
VPAGE_SIZE + size_of::<JournalType>()
};
// of the two possible end criteria, pick the nearest one
let endstop = if key_endstop < data_endstop {
key_endstop
} else {
data_endstop
};
log::trace!("reading offset {}/{}:{}", cur_offset, endstop, kcache.len);
let mut read_offset = (cur_offset % VPAGE_SIZE as u64) as usize;
if let Some(pp) = basis.v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()) {
let pt_data = hw.data_decrypt_page(&basis.cipher, &basis.aad, pp).expect("Decryption auth error");
if !first_iter {
assert!((cur_offset as usize % VPAGE_SIZE) == 0, "algorithm error in handling offset data");
if blocks_read != 0 {
assert!(read_offset == 0, "algorithm error in handling offset data");
}
// note: cur_offset % VPAGE_SIZE handles mis-aligned starts; cur_offset will increment until start + cur_offset is
// a multiple of a VPAGE_SIZE, so on later iterations, cur_offset % VPAGE_SIZE should be equal to 0.
for (&src, dst) in
pt_data[size_of::<JournalType>() + cur_offset as usize % VPAGE_SIZE..].iter().zip(data.iter_mut()) {
pt_data[
size_of::<JournalType>() // always this fixed offset per block
+ read_offset // this should be 0 after the first block
..endstop
].iter().zip(data[cur_offset as usize..].iter_mut()) {
*dst = src;
cur_offset += 1;
read_offset += 1;
}
first_iter = false;
blocks_read += 1;
} else {
log::warn!("Not enough bytes available to read for key {}:{} ({}/{})", dict, key, cur_offset, data.len());
return Ok(cur_offset as usize)
}
// cur_offset won't hit (start + len) if we're hitting an endstop, so the loop won't exit.
// if we hit an endstop, we should exit the read loop.
if endstop != VPAGE_SIZE + size_of::<JournalType>() {
break;
}
// break if we've hit or exceeded our endstop
if read_offset >= endstop {
break;
}
}
return Ok(cur_offset as usize)
}
@ -581,6 +623,31 @@ impl BasisCache {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
}
}
pub(crate) fn key_attributes(&mut self, hw: &mut PddbOs, dict: &str, key: &str, basis_name: Option<&str>) -> Result<KeyAttributes> {
if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.ensure_dict_in_cache(hw, dict) {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
} else {
let dict_entry = basis.dicts.get_mut(dict).expect("Entry was assured, but not there!");
if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) {
let kcache = dict_entry.keys.get_mut(key).expect("Entry was assured, but then not there!");
Ok(KeyAttributes {
len: kcache.len as usize,
reserved: kcache.reserved as usize,
age: kcache.age as usize,
dict: dict.to_string(),
basis: (&basis.name).to_string(),
})
} else {
return Err(Error::new(ErrorKind::NotFound, "key not found"));
}
}
} else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
}
}
}
/// This is the RAM cached copy of a basis as maintained in the PDDB.
@ -892,7 +959,12 @@ impl BasisCacheEntry {
// this is the virtual page within the dictionary region that we're currently serializing
let mut vpage_num = 0;
let mut loopcheck= 0;
loop {
loopcheck += 1;
if loopcheck == 256 {
log::warn!("potential infinite loop detected sync dict {}", name);
}
// 1. resolve the virtual address to a target page
let cur_vpage = VirtAddr::new(dict_offset.get() + (vpage_num as u64 * VPAGE_SIZE as u64)).unwrap();
let pp = self.v2p_map.entry(cur_vpage).or_insert_with(||
@ -919,9 +991,9 @@ impl BasisCacheEntry {
let next_vpage = VirtAddr::new(cur_vpage.get() + VPAGE_SIZE as u64).unwrap();
for (key_name, key) in dict.keys.iter_mut() {
if !key.clean {
log::info!("merging in key {}", key_name);
if key.descriptor_vaddr(dict_offset) >= cur_vpage &&
key.descriptor_vaddr(dict_offset) < next_vpage {
log::info!("merging in key {}", key_name);
// key is within the current page, add it to the target list
let mut dk_entry = DictKeyEntry::default();
let mut kn = [0u8; KEY_NAME_LEN];
@ -941,6 +1013,8 @@ impl BasisCacheEntry {
}
dk_vpage.elements[key.descriptor_index.get() as usize % DK_PER_VPAGE] = Some(dk_entry);
key.clean = true;
} else {
log::debug!("proposed key fell outside of our vpage: {} vpage{:x}/vaddr{:x}", key_name, cur_vpage.get(), key.descriptor_vaddr(dict_offset));
}
}
}
@ -979,6 +1053,7 @@ impl BasisCacheEntry {
break;
}
}
log::info!("done syncing dict");
dict.clean = true;
}
Ok(())

View file

@ -336,13 +336,27 @@ impl DictCacheEntry {
} else {
kcache.age = kcache.age.saturating_add(1);
kcache.clean = false;
if data.len() == 4 {
use std::convert::TryInto;
log::info!("patching checksum: {:x} at offset {}", u32::from_le_bytes(data.try_into().unwrap()), offset);
}
// 1. handle unaligned start offsets
let mut written: usize = 0;
if ((kcache.start + offset as u64) % VPAGE_SIZE as u64) != 0 {
if ((kcache.start + offset as u64 + written as u64) % VPAGE_SIZE as u64) != 0 {
let start_vpage_addr = ((kcache.start + offset as u64) / VPAGE_SIZE as u64) * VPAGE_SIZE as u64;
let pp = v2p_map.get(&VirtAddr::new(start_vpage_addr).unwrap()).expect("large key data allocation missing");
let mut pt_data = hw.data_decrypt_page(&cipher, &self.aad, pp).expect("Decryption auth error");
for (&src, dst) in data[written..].iter().zip(pt_data[size_of::<JournalType>() + offset..].iter_mut()) {
let mut pt_data = match hw.data_decrypt_page(&cipher, &self.aad, pp) {
Some(data) => data,
None => {
// perhaps we could just "zero out" previously data, but for now, let's make this undefined behavior
log::warn!("Attempt to patch data outside of the previously initialized length. This is an error.");
return Err(Error::new(ErrorKind::UnexpectedEof, "patching outside of previously written data"));
}
};
if offset > 0 {
log::info!("patching offset {}, total length {}, data length {}", offset % VPAGE_SIZE, kcache.len, data.len());
}
for (&src, dst) in data[written..].iter().zip(pt_data[size_of::<JournalType>() + (offset % VPAGE_SIZE)..].iter_mut()) {
*dst = src;
written += 1;
}

View file

@ -175,3 +175,17 @@ impl PartialEq for KeySmallPoolOrd {
self.avail == other.avail
}
}
/// A structure for passing around key metadata
pub(crate) struct KeyAttributes {
/// actual length of data in the key
pub(crate) len: usize,
/// pre-reserved storage space for the key (growable to this bound "at no cost")
pub(crate) reserved: usize,
/// access count
pub(crate) age: usize,
/// owning dictionary
pub(crate) dict: String,
/// the specific basis from which this key's metadata was found
pub(crate) basis: String,
}

View file

@ -1,6 +1,8 @@
use rand::Rng;
use crate::*;
const UPPER_BOUND: usize = 9000;
fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bound: usize) -> (String, Vec::<u8>) {
let mut rng = rand::thread_rng();
// we want roughly half our keys to be in the small bin, and half in the large bin
@ -40,7 +42,7 @@ pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCach
let num_dicts = maybe_num_dicts.unwrap_or(4);
let num_keys = maybe_num_keys.unwrap_or(34);
let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, 9000));
let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, UPPER_BOUND - 4)); // 4 bytes for the CI checksum
// make all the directories first
for dictnum in 1..=num_dicts {
@ -101,12 +103,28 @@ pub(crate) fn patch_test(hw: &mut PddbOs, basis_cache: &mut BasisCache,
for dict in dict_list.iter() {
if let Ok(key_list) = basis_cache.key_list(hw, dict) {
for key in key_list.iter() {
let mut patchbuf = [0u8; UPPER_BOUND];
// this actually does something a bit more complicated on a multi-basis system than you'd think:
// it will get the union of all key names, and then patch the *latest open basis* only with new data.
// note: if the key didn't already exist in the latest open basis, it's added, with just that patch data in it.
// to override this behavior, you'd want to specify a _specific_ basis, but for testing purposes, we only have one
// so we're doing this way that would lead to surprising results if it were copied as template code elsewhere.
basis_cache.key_update(hw, dict, key, patch_data.as_bytes(), Some(patch_offset), None, None, false).unwrap();
// now fix the CI checksum. structured as two separate patches. not because it's efficient,
// but because it exercises the code harder.
let readlen = basis_cache.key_read(hw, dict, key, &mut patchbuf, Some(0), None).unwrap();
// now extend and compute the checksum
let mut checkdata = Vec::<u8>::new();
for &b in &patchbuf[..readlen-4] {
checkdata.push(b);
}
log::trace!("checkdata len: {}", checkdata.len());
while checkdata.len() % 4 != 0 {
checkdata.push(0);
}
let checksum = murmur3_32(&checkdata, 0);
basis_cache.key_update(hw, dict, key, &checksum.to_le_bytes(), Some(readlen-4), None, None, false).unwrap();
}
}

View file

@ -163,6 +163,7 @@ def main():
print("Basis {}".format(name))
v2p_table = tables[name][0]
p2v_table = tables[name][1]
# v2p_table[0xfe0fe0] = 0x1200* 0x100
basis_data = bytearray()
pp_start = v2p_table[VPAGE_SIZE]
@ -207,6 +208,7 @@ def main():
except ValueError:
print("couldn't decrypt basis root vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE]))
PRINTED_FULL = False
class KeyDescriptor:
MAX_NAME_LEN = 95
def __init__(self, record, v2p, disk, key, name):
@ -271,6 +273,7 @@ class KeyDescriptor:
def as_str(self, indent=''):
PRINT_LEN = 64
global PRINTED_FULL
desc = ''
desc += indent + 'Start: 0x{:x}\n'.format(self.start)
desc += indent + 'Len: {}/0x{:x}\n'.format(self.len, self.reserved)
@ -283,12 +286,17 @@ class KeyDescriptor:
desc += ' | Age: {}'.format(self.age)
desc += ' | CI OK: {}'.format(self.ci_ok)
desc += '\n'
if len(self.data) < 64:
if len(self.data) < PRINT_LEN:
print_len = len(self.data)
extra = ''
else:
print_len = 64
extra = '...'
if (PRINTED_FULL == False) and (self.ci_ok == False):
print_len = len(self.data)
extra = ''
PRINTED_FULL = True
else:
print_len = PRINT_LEN
extra = '...'
desc += indent + 'Data (hex): {}{}\n'.format(self.data[:print_len].hex(), extra)
desc += indent + 'Data (txt): {}{}'.format(make_printable(self.data[:print_len].decode('utf-8', errors='ignore')), extra) + '\n'
return (desc)
@ -347,7 +355,7 @@ class BasisDicts:
keys_found += 1
except ValueError:
print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr), pp_start)
print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr, pp_start))
keys_tried += 1
except KeyError:
# the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried"
@ -504,9 +512,15 @@ class Pte:
def find_mbbb(mbbb):
global MBBB_PAGES
pages = [mbbb[i:i+PAGE_SIZE] for i in range(0, MBBB_PAGES * PAGE_SIZE, PAGE_SIZE)]
candidates = []
for page in pages:
if bytearray(page[:16]) != bytearray([0xff] * 16):
return page
candidates.append(page)
if len(candidates) > 1:
print("More than one MBBB found, this is an error!")
else:
return candidates[0]
return None
def decode_pagetable(img, entries, keys, mbbb):
@ -577,7 +591,7 @@ class PhysPage:
return (self.pp >> 24) & 0xF
def as_str(self):
return 'a_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal())
return 'pp_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal())
class Fscb:
def __init__(self, i_bytes):