debugging alignment issues on serialization of compound structures

This commit is contained in:
bunnie 2021-11-15 04:03:55 +08:00
parent 696e99b173
commit ba9b136f87
4 changed files with 145 additions and 39 deletions

View file

@ -43,6 +43,8 @@ impl<'a> BasisEncryptor<'a> {
aad.extend_from_slice(&PDDB_VERSION.to_le_bytes());
aad.extend_from_slice(&dna.to_le_bytes());
log::info!("aad: {:?}", aad);
BasisEncryptor {
root,
dicts,

View file

@ -103,6 +103,7 @@ pub (crate) struct SpaceUpdate {
#[cfg(not(feature = "u64_pa"))]
impl SpaceUpdate {
pub fn try_into_phys_page(slice: &[u8]) -> Option<PhysPage> {
// note that the seed uses big-endian re-encoding of a portion of the nonce!
let computed_sum = murmur3_32(&slice[..12], u32::from_be_bytes(slice[4..8].try_into().unwrap()));
if u32::from_le_bytes(slice[12..].try_into().unwrap()) == computed_sum {
let pp = u32::from_le_bytes(slice[8..12].try_into().unwrap());

View file

@ -174,7 +174,7 @@ impl PddbOs {
/// patches data at an offset starting from the data physical base address, which corresponds
/// exactly to the first entry in the page table
fn patch_data(&self, data: &[u8], offset: u32) {
log::info!("patch offset: {:x} len: {:x}", offset, data.len());
log::trace!("patch offset: {:x} len: {:x}", offset, data.len());
assert!(data.len() + offset as usize <= PDDB_A_LEN - self.data_phys_base.as_usize(), "attempt to store past disk boundary");
self.spinor.patch(
self.pddb_mr.as_slice(),
@ -444,16 +444,19 @@ impl PddbOs {
// 1. scan through the entire space, and look for the FastSpace record. It can be identified by the
// first 16 (aes::BLOCK_SIZE) bytes not being all 1's.
let not_aes: [u8; aes::BLOCK_SIZE] = [0xff; aes::BLOCK_SIZE];
let blank: [u8; aes::BLOCK_SIZE] = [0xff; aes::BLOCK_SIZE];
let mut fscb_pages = 0;
let mut blank_pages = Vec::new();
for page_start in (0..fscb_slice.len()).step_by(PAGE_SIZE) {
if (fscb_slice[page_start..page_start + aes::BLOCK_SIZE] == not_aes)
&& (fscb_slice[page_start + aes::BLOCK_SIZE..page_start + aes::BLOCK_SIZE * 2] == not_aes) {
if (fscb_slice[page_start..page_start + aes::BLOCK_SIZE] == blank)
&& (fscb_slice[page_start + aes::BLOCK_SIZE..page_start + aes::BLOCK_SIZE * 2] == blank) {
//if fscb_slice[page_start..page_start + aes::BLOCK_SIZE].iter().zip(blank.iter()).all(|(&a,&b)| a==b)
//&& fscb_slice[page_start + aes::BLOCK_SIZE..page_start + aes::BLOCK_SIZE * 2].iter().zip(blank.iter()).all(|(&a,&b)| a==b) {
// page has met the criteria for being blank, skip to the next page
blank_pages.push(page_start);
continue
} else if fscb_slice[page_start..page_start+aes::BLOCK_SIZE] == not_aes {
} else if fscb_slice[page_start..page_start+aes::BLOCK_SIZE] == blank {
//} else if fscb_slice[page_start..page_start+aes::BLOCK_SIZE].iter().zip(blank.iter()).all(|(&a,&b)| a==b) {
// this page contains update records; stash it for scanning after we've read in the master record
self.fspace_log_addrs.push(PageAlignedPa::from(page_start));
continue
@ -502,6 +505,7 @@ impl PddbOs {
// 2. visit the update_page_addrs and modify the fspace_cache accordingly.
let cipher = Aes256::new(GenericArray::from_slice(&system_key));
let mut block = Block::default();
log::info!("space_log_addrs len: {}", self.fspace_log_addrs.len());
for page in &self.fspace_log_addrs {
for (index, ct_block) in
fscb_slice[page.as_usize() + aes::BLOCK_SIZE .. page.as_usize() + PAGE_SIZE]
@ -523,6 +527,7 @@ impl PddbOs {
}
cipher.decrypt_block(&mut block);
if let Some(pp) = SpaceUpdate::try_into_phys_page(block.as_slice()) {
log::info!("maybe replacing fspace block: {:x?}", pp);
// note: pp.valid() isn't the cryptographic check, the cryptographic check of record validity is in try_into_phys_page()
if pp.valid() { // PS: it should always be valid!
if let Some(prev_pp) = self.fspace_cache.get(&pp) {
@ -638,8 +643,10 @@ impl PddbOs {
let cipher = Aes256::new(GenericArray::from_slice(&system_key));
let mut update = SpaceUpdate::new(self.entropy.borrow_mut().get_u64(), ppc);
let mut block = Block::from_mut_slice(update.deref_mut());
log::info!("block: {:x?}", block);
cipher.encrypt_block(&mut block);
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
log::info!("patch: {:x?}", block);
self.patch_fscb(&block, log_addr);
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
@ -818,8 +825,8 @@ impl PddbOs {
version: api::PDDB_VERSION,
name,
age: 0,
num_dictionaries: 0,
prealloc_open_end: PageAlignedVa::from(INITIAL_BASIS_ALLOC * VPAGE_SIZE),
num_dictionaries: 0,
dict_ptr: None,
};
log::info!("prealloc_open_end: {:x?}", basis_root.prealloc_open_end);
@ -891,11 +898,11 @@ impl PddbOs {
for (&virt, &phys) in basis_v2p_map.into_iter() {
let mut pte = Pte::new(virt, PtFlags::CLEAN, Rc::clone(&self.entropy));
let mut block = Block::from_mut_slice(pte.deref_mut());
log::info!("pte pt: {:x?}", block);
//log::info!("pte pt: {:x?}", block);
cipher.encrypt_block(&mut block);
log::info!("pte ct: {:x?}", block);
//log::info!("pte ct: {:x?}", block);
self.patch_pagetable(&block, phys.page_number() * aes::BLOCK_SIZE as u32);
log::info!("pte ct loc: {:x?}", phys.page_number() * aes::BLOCK_SIZE as u32);
//log::info!("pte ct loc: {:x?}", phys.page_number() * aes::BLOCK_SIZE as u32);
}
}
}

View file

@ -142,13 +142,80 @@ def main():
mbbb = raw_img[img_index : img_index + PAGE_SIZE * MBBB_PAGES]
img_index += PAGE_SIZE * MBBB_PAGES
fscb = decode_fscb(raw_img[img_index: img_index + PAGE_SIZE * FSCB_PAGES], keys)
img_index += PAGE_SIZE * FSCB_PAGES
data = raw_img[img_index:]
for name, key in keys.items():
if name in tables:
print("Basis {}".format(name))
v2p_table = tables[name][0]
p2v_table = tables[name][1]
basis_data = bytearray()
errors = 0
for vp in sorted(v2p_table):
pp_start = v2p_table[vp]
# print("pp_start: {}, vp: {}".format(pp_start, vp))
pp_data = data[pp_start:pp_start + PAGE_SIZE]
try:
cipher = AES_GCM_SIV(key, pp_data[:12])
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
basis_data.extend(bytearray(pt_data))
except KeyError:
errors += 1
print("couldn't decrypt vpage @ {} ppage @ {:x}".format(vp, v2p_table[vp]))
if errors == 0:
print([hex(x) for x in basis_data[:256]])
basis = Basis(basis_data)
print(basis.as_str())
class Basis:
MAX_NAME_LEN = 64
def __init__(self, i_bytes):
self.bytes = i_bytes
i = 0
self.journal = int.from_bytes(i_bytes[i:i+4], 'little')
i += 4
self.magic = int.from_bytes(i_bytes[i:i+4], 'little')
i += 4
self.version = int.from_bytes(i_bytes[i:i+2], 'little')
i += 2
self.name = i_bytes[i:i+Basis.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
i += Basis.MAX_NAME_LEN
print("alloc on disk: {}".format(i_bytes[i:i+8].hex()))
self.prealloc_open_end = int.from_bytes(i_bytes[i:i+8], 'little')
i += 8
self.num_dicts = int.from_bytes(i_bytes[i:i+4], 'little')
i += 4
self.dict_ptr = int.from_bytes(i_bytes[i:i+8], 'little')
i += 8
def as_str(self):
desc = ''
desc += ' Journal rev: {}\n'.format(self.journal)
desc += ' Magic: {:x}\n'.format(self.magic)
desc += ' Version: {}\n'.format(self.version)
desc += ' Name: {}\n'.format(self.name)
desc += ' Alloc: {:x}\n'.format(self.prealloc_open_end)
desc += ' NumDicts: {:x}\n'.format(self.num_dicts)
desc += ' DictPtr: {:x}\n'.format(self.dict_ptr)
return desc
def basis_aad(name, version=0, dna=0):
name_bytes = bytearray(name, 'utf-8')
name_bytes += bytearray(([0] * (Basis.MAX_NAME_LEN - len(name))))
name_bytes += version.to_bytes(2, 'little')
name_bytes += dna.to_bytes(8, 'little')
return name_bytes
class Pte:
PTE_LEN = 16
def __init__(self, bytes):
assert(len(bytes) == self.PTE_LEN)
self.bytes = bytes
def __init__(self, i_bytes):
assert(len(i_bytes) == self.PTE_LEN)
self.bytes = i_bytes
def len_bytes(self):
return(self.bytes.len())
@ -171,12 +238,12 @@ class Pte:
def is_valid(self):
return (self.checksum() == mm3_hash(self.bytes[:12], self.nonce()))
def print(self, ppage=None):
def as_str(self, ppage=None):
#print('checksum: {:08x}, hash: {:08x}'.format(self.checksum(), mm3_hash(self.bytes[:12], self.nonce())))
if ppage != None:
print('{:08x}: a_{:012x} | f_{} | n_{:08x} | c_{:08x}'.format(ppage, self.addr(), self.flags(), self.nonce(), self.checksum()))
return '{:08x}: a_{:012x} | f_{} | n_{:08x} | c_{:08x}'.format(ppage, self.addr(), self.flags(), self.nonce(), self.checksum())
else:
print('a_{:012x} | f_{} | n_{:08x} | c_{:08x}'.format(self.addr(), self.flags(), self.nonce(), self.checksum()))
return 'a_{:012x} | f_{} | n_{:08x} | c_{:08x}'.format(self.addr(), self.flags(), self.nonce(), self.checksum())
def decode_pagetable(img, entries, keys):
key_table = {}
@ -191,8 +258,8 @@ def decode_pagetable(img, entries, keys):
for pte in encrypted_ptes:
maybe_pte = Pte(cipher.decrypt(pte))
if maybe_pte.is_valid():
maybe_pte.print(page)
p_addr = page * (4096 / Pte.PTE_LEN)
print(maybe_pte.as_str(page))
p_addr = page * (4096 // Pte.PTE_LEN)
v2p_table[maybe_pte.addr()] = p_addr
p2v_table[p_addr] = maybe_pte.addr()
@ -201,8 +268,9 @@ def decode_pagetable(img, entries, keys):
return key_table
class PhysPage:
def __init__(self, bytes):
self.pp = int.from_bytes(bytes[:4], 'little')
PP_LEN = 4
def __init__(self, i_bytes):
self.pp = int.from_bytes(i_bytes[:4], 'little')
def page_number(self):
return self.pp & 0xF_FFFF
@ -233,18 +301,18 @@ class PhysPage:
def journal(self):
return (self.pp >> 24) & 0xF
def print(self):
print('a_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal()))
def as_str(self):
return 'a_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal())
class Fscb:
def __init__(self, bytes):
def __init__(self, i_bytes):
FASTSPACE_PAGES = 2
NONCE_LEN = 12
TAG_LEN = 16
PP_LEN = 4
FREE_POOL_LEN = ((4096 * FASTSPACE_PAGES) - (NONCE_LEN + TAG_LEN)) // PP_LEN
assert(len(bytes) == FREE_POOL_LEN * PP_LEN)
raw_pps = [bytes[i:i+PP_LEN] for i in range(0, FREE_POOL_LEN * PP_LEN, PP_LEN)]
assert(len(i_bytes) == FREE_POOL_LEN * PP_LEN)
raw_pps = [i_bytes[i:i+PP_LEN] for i in range(0, FREE_POOL_LEN * PP_LEN, PP_LEN)]
self.free_space = {}
for raw_pp in raw_pps:
pp = PhysPage(raw_pp)
@ -259,15 +327,42 @@ class Fscb:
def print(self):
for pagenum, pp in self.free_space.items():
pp.print()
print(pp.as_str())
def try_replace(self, candidate):
if candidate.is_valid():
pp = candidate.get_pp()
cur_pp = self.free_space[pp.page_number() * 4096]
if pp.valid() and (cur_pp.journal() < pp.journal()):
print("FSCB replace {} /with/ {}".format(cur_pp.as_str(), pp.as_str()))
self.free_space[pp.page_number() * 4096] = pp
# this is the "AAD" used to encrypt the FastSpace
def aad(version=0, dna=0):
return bytearray([46, 70, 97, 115, 116, 83, 112, 97, 99, 101]) + version.to_bytes(2, 'little') + dna.to_bytes(8, 'little')
class SpaceUpdate:
SU_LEN = 16
def __init__(self, i_bytes):
self.nonce = i_bytes[:8]
self.page_number = PhysPage(i_bytes[8:12])
self.checksum = int.from_bytes(i_bytes[12:], 'little')
hash = mm3_hash(i_bytes[:12], int.from_bytes(i_bytes[4:8], 'big'))
if self.checksum == hash:
self.valid = True
else:
self.valid = False
def is_valid(self):
return self.valid
def get_pp(self):
return self.page_number
# img is clipped to just the fscb region of the overall disk image
def decode_fscb(img, keys, FSCB_LEN_PAGES=2):
global SYSTEM_BASIS
fscb = None
if SYSTEM_BASIS in keys:
key = keys[SYSTEM_BASIS]
pages = [img[i:i+4096] for i in range(0, len(img), 4096)]
@ -279,40 +374,41 @@ def decode_fscb(img, keys, FSCB_LEN_PAGES=2):
# page is blank
pass
elif bytearray(page[:16] == bytearray([0xff] * 16)):
space_update.append(page)
space_update.append(page[16:])
else:
if fscb_start == None:
fscb_start = pg
pg += 1
if fscb_start:
print("found FSCB at {:x}".format(fscb_start))
print("Found FSCB at {:x}".format(fscb_start))
fscb_enc = img[fscb_start * 4096 : (fscb_start + FSCB_LEN_PAGES) * 4096]
try:
nonce = fscb_enc[:12]
print("key: {}, nonce: {}".format(key.hex(), nonce.hex()))
cipher = AES_GCM_SIV(key, nonce)
print("aad: {}".format(Fscb.aad().hex()))
print("mac: {}".format(fscb_enc[-16:].hex()))
#print("aad: {}".format(Fscb.aad().hex()))
#print("mac: {}".format(fscb_enc[-16:].hex()))
#print("data: {}".format(fscb_enc[12:-16].hex()))
fscb_dec = cipher.decrypt(fscb_enc[12:], Fscb.aad())
print("decrypted")
fscb = Fscb(fscb_dec)
except KeyError:
print("couldn't decrypt")
fscb = None
if fscb:
fscb.print()
if fscb != None:
print("key: {}".format(key.hex()))
cipher = AES.new(key, AES.MODE_ECB)
for update in space_update:
entries = [update[i:i+SpaceUpdate.SU_LEN] for i in range(0, len(update), SpaceUpdate.SU_LEN)]
for entry in entries:
if entry == bytearray([0xff] * 16):
break
else:
fscb.try_replace(SpaceUpdate(cipher.decrypt(entry)))
for update in space_update:
pass
return fscb
if __name__ == "__main__":
main()
exit(0)
# 00 00 00 00 00 00 00 00 00 00 00
# 00 00 00 00 00 00 00 00 00 00
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0