mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-18 08:52:45 +00:00
fastspace regen debugging -- passes test, but needs optimization
should be able to get away with "just" regenerating the fast space log only if that's what we've run out of; however, doing that optimization we run into out of space errors. maybe we need some tuning of the thresholds for when to reclaim space
This commit is contained in:
parent
e02df18374
commit
93ee050079
4 changed files with 75 additions and 15 deletions
|
|
@ -8,7 +8,7 @@ use aes_gcm_siv::{Nonce, Tag};
|
||||||
|
|
||||||
/// Each free_pool entry takes about 4 bytes, so give-or-take we have about 1000 free_pool
|
/// Each free_pool entry takes about 4 bytes, so give-or-take we have about 1000 free_pool
|
||||||
/// entries per page of storage for the free_pool, or 4k * 1000 ~ 4MiB per page, when PhysAddr is a u32
|
/// entries per page of storage for the free_pool, or 4k * 1000 ~ 4MiB per page, when PhysAddr is a u32
|
||||||
pub(crate) const FASTSPACE_PAGES: usize = 2;
|
pub(crate) const FASTSPACE_PAGES: usize = 1;
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq, Copy, Clone)]
|
#[derive(Debug, PartialEq, Eq, Copy, Clone)]
|
||||||
#[repr(u8)]
|
#[repr(u8)]
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ use std::io::{Result, Error, ErrorKind};
|
||||||
/// Implementation-specific PDDB structures: for Precursor/Xous OS pair
|
/// Implementation-specific PDDB structures: for Precursor/Xous OS pair
|
||||||
|
|
||||||
pub(crate) const MBBB_PAGES: usize = 10;
|
pub(crate) const MBBB_PAGES: usize = 10;
|
||||||
pub(crate) const FSCB_PAGES: usize = 16;
|
pub(crate) const FSCB_PAGES: usize = 4;
|
||||||
|
|
||||||
/// size of a physical page
|
/// size of a physical page
|
||||||
pub const PAGE_SIZE: usize = spinor::SPINOR_ERASE_SIZE as usize;
|
pub const PAGE_SIZE: usize = spinor::SPINOR_ERASE_SIZE as usize;
|
||||||
|
|
@ -90,6 +90,8 @@ pub(crate) struct PddbOs {
|
||||||
fspace_log_addrs: Vec::<PageAlignedPa>,
|
fspace_log_addrs: Vec::<PageAlignedPa>,
|
||||||
/// memoize the current target offset for the next log entry
|
/// memoize the current target offset for the next log entry
|
||||||
fspace_log_next_addr: Option<PhysAddr>,
|
fspace_log_next_addr: Option<PhysAddr>,
|
||||||
|
/// track roughly how big the log has gotten, so we can pre-emptively garbage collect it before we get too full.
|
||||||
|
fspace_log_len: usize,
|
||||||
/// a cached copy of the FPGA's DNA ID, used in the AAA records.
|
/// a cached copy of the FPGA's DNA ID, used in the AAA records.
|
||||||
dna: u64,
|
dna: u64,
|
||||||
/// reference to a TrngPool object that's shared among all the hardware functions
|
/// reference to a TrngPool object that's shared among all the hardware functions
|
||||||
|
|
@ -138,6 +140,7 @@ impl PddbOs {
|
||||||
fspace_cache: HashSet::<PhysPage>::new(),
|
fspace_cache: HashSet::<PhysPage>::new(),
|
||||||
fspace_log_addrs: Vec::<PageAlignedPa>::new(),
|
fspace_log_addrs: Vec::<PageAlignedPa>::new(),
|
||||||
fspace_log_next_addr: None,
|
fspace_log_next_addr: None,
|
||||||
|
fspace_log_len: 0,
|
||||||
dna: llio.soc_dna().unwrap(),
|
dna: llio.soc_dna().unwrap(),
|
||||||
entropy: trngpool,
|
entropy: trngpool,
|
||||||
};
|
};
|
||||||
|
|
@ -159,6 +162,7 @@ impl PddbOs {
|
||||||
fspace_cache: HashSet::<PhysPage>::new(),
|
fspace_cache: HashSet::<PhysPage>::new(),
|
||||||
fspace_log_addrs: Vec::<PageAlignedPa>::new(),
|
fspace_log_addrs: Vec::<PageAlignedPa>::new(),
|
||||||
fspace_log_next_addr: None,
|
fspace_log_next_addr: None,
|
||||||
|
fspace_log_len: 0,
|
||||||
dna: llio.soc_dna().unwrap(),
|
dna: llio.soc_dna().unwrap(),
|
||||||
entropy: trngpool,
|
entropy: trngpool,
|
||||||
}
|
}
|
||||||
|
|
@ -543,6 +547,9 @@ impl PddbOs {
|
||||||
// commit the fscb data
|
// commit the fscb data
|
||||||
self.patch_fscb(&[&nonce_array, ct_to_flash].concat(), dest_page * PAGE_SIZE as u32);
|
self.patch_fscb(&[&nonce_array, ct_to_flash].concat(), dest_page * PAGE_SIZE as u32);
|
||||||
} // end "it would be bad if we lost power now" region
|
} // end "it would be bad if we lost power now" region
|
||||||
|
|
||||||
|
// note: this function should be followed up by a fast_space_read() to regenerate the temporary
|
||||||
|
// bookkeeping variables that are not reset by this function.
|
||||||
} else {
|
} else {
|
||||||
panic!("invalid state!");
|
panic!("invalid state!");
|
||||||
}
|
}
|
||||||
|
|
@ -656,6 +663,7 @@ impl PddbOs {
|
||||||
self.fspace_cache.clear();
|
self.fspace_cache.clear();
|
||||||
self.fspace_log_addrs.clear();
|
self.fspace_log_addrs.clear();
|
||||||
self.fspace_log_next_addr = None;
|
self.fspace_log_next_addr = None;
|
||||||
|
self.fspace_log_len = 0;
|
||||||
|
|
||||||
// let fscb_slice = self.fscb_deref(); // can't use this line because it causse self to be immutably borrowed, so we write out the equivalent below.
|
// let fscb_slice = self.fscb_deref(); // can't use this line because it causse self to be immutably borrowed, so we write out the equivalent below.
|
||||||
let fscb_slice = &self.pddb_mr.as_slice()[self.fscb_phys_base.as_usize()..self.fscb_phys_base.as_usize() + FSCB_PAGES * PAGE_SIZE];
|
let fscb_slice = &self.pddb_mr.as_slice()[self.fscb_phys_base.as_usize()..self.fscb_phys_base.as_usize() + FSCB_PAGES * PAGE_SIZE];
|
||||||
|
|
@ -873,6 +881,7 @@ impl PddbOs {
|
||||||
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
||||||
log::trace!("patch: {:x?}", block);
|
log::trace!("patch: {:x?}", block);
|
||||||
self.patch_fscb(&block, log_addr);
|
self.patch_fscb(&block, log_addr);
|
||||||
|
self.fspace_log_len += 1;
|
||||||
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
||||||
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
||||||
self.fspace_log_next_addr = Some(next_addr as PhysAddr);
|
self.fspace_log_next_addr = Some(next_addr as PhysAddr);
|
||||||
|
|
@ -908,6 +917,7 @@ impl PddbOs {
|
||||||
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
||||||
log::trace!("patch: {:x?}", block);
|
log::trace!("patch: {:x?}", block);
|
||||||
self.patch_fscb(&block, log_addr);
|
self.patch_fscb(&block, log_addr);
|
||||||
|
self.fspace_log_len += 1;
|
||||||
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
||||||
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
||||||
self.fspace_log_next_addr = Some(next_addr as PhysAddr);
|
self.fspace_log_next_addr = Some(next_addr as PhysAddr);
|
||||||
|
|
@ -921,9 +931,13 @@ impl PddbOs {
|
||||||
/// This is a "look before you leap" function that will potentially pause all system operations
|
/// This is a "look before you leap" function that will potentially pause all system operations
|
||||||
/// and do a deep scan for space if the required amount is not available.
|
/// and do a deep scan for space if the required amount is not available.
|
||||||
pub fn ensure_fast_space_alloc(&mut self, pages: usize, cache: &Vec::<BasisCacheEntry>) -> bool {
|
pub fn ensure_fast_space_alloc(&mut self, pages: usize, cache: &Vec::<BasisCacheEntry>) -> bool {
|
||||||
if self.fast_space_len() > pages {
|
// make sure we have fast space pages...
|
||||||
|
if (self.fast_space_len() > pages)
|
||||||
|
// ..and make sure we have space for fast space log entries
|
||||||
|
&& (self.fspace_log_len < (FSCB_PAGES - FASTSPACE_PAGES - 1) * PAGE_SIZE / aes::BLOCK_SIZE) {
|
||||||
true
|
true
|
||||||
} else {
|
} else {
|
||||||
|
// if we're really out of space, do an expensive full-space sweep
|
||||||
if let Some(used_pages) = self.pddb_generate_used_map(cache) {
|
if let Some(used_pages) = self.pddb_generate_used_map(cache) {
|
||||||
let free_pool = self.fast_space_generate(used_pages);
|
let free_pool = self.fast_space_generate(used_pages);
|
||||||
if free_pool.len() == 0 {
|
if free_pool.len() == 0 {
|
||||||
|
|
@ -937,6 +951,7 @@ impl PddbOs {
|
||||||
*dst = src;
|
*dst = src;
|
||||||
}
|
}
|
||||||
// write just commits a new record to disk, but doesn't update our internal data cache
|
// write just commits a new record to disk, but doesn't update our internal data cache
|
||||||
|
// this also clears the fast space log.
|
||||||
self.fast_space_write(&fast_space);
|
self.fast_space_write(&fast_space);
|
||||||
// this will ensure the data cache is fully in sync
|
// this will ensure the data cache is fully in sync
|
||||||
self.fast_space_read();
|
self.fast_space_read();
|
||||||
|
|
@ -951,6 +966,21 @@ impl PddbOs {
|
||||||
} else {
|
} else {
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
/* -- note to self -- "to-do" optimization -- if just out of log pages, we can regenerate those only instead of doing an all-basis sweep
|
||||||
|
let mut fast_space = FastSpace {
|
||||||
|
free_pool: [PhysPage(0); FASTSPACE_FREE_POOL_LEN],
|
||||||
|
};
|
||||||
|
// regenerate from the existing fast space cache
|
||||||
|
for (&src, dst) in self.fspace_cache.iter().zip(fast_space.free_pool.iter_mut()) {
|
||||||
|
*dst = src;
|
||||||
|
}
|
||||||
|
// write just commits a new record to disk, but doesn't update our internal data cache
|
||||||
|
// this also clears the fast space log.
|
||||||
|
self.fast_space_write(&fast_space);
|
||||||
|
// this will re-read back in the data, shuffle the alloc order a bit, and ensure the data cache is fully in sync
|
||||||
|
self.fast_space_read();
|
||||||
|
true
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -423,7 +423,7 @@ fn xmain() -> ! {
|
||||||
log::info!("Creating `basecase1e`");
|
log::info!("Creating `basecase1e`");
|
||||||
let mut basis_cache = BasisCache::new();
|
let mut basis_cache = BasisCache::new();
|
||||||
create_basis_testcase(&mut pddb_os, &mut basis_cache, None,
|
create_basis_testcase(&mut pddb_os, &mut basis_cache, None,
|
||||||
None, None, Some(4));
|
None, None, Some(32));
|
||||||
log::info!("Saving `basecase1e` to local host");
|
log::info!("Saving `basecase1e` to local host");
|
||||||
pddb_os.dbg_dump(Some("basecase1e".to_string()));
|
pddb_os.dbg_dump(Some("basecase1e".to_string()));
|
||||||
|
|
||||||
|
|
@ -441,6 +441,35 @@ fn xmain() -> ! {
|
||||||
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
|
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
|
||||||
pddb_os.dbg_dump(Some("patterne".to_string()));
|
pddb_os.dbg_dump(Some("patterne".to_string()));
|
||||||
|
|
||||||
|
// extended tests.
|
||||||
|
// allocation space curtailed to force resource exhaustion faster.
|
||||||
|
// note to self: FSCB_PAGES revert to 16 (hw.rs), FASTSPACE_PAGES revert to 2 (fastspace.rs)
|
||||||
|
log::info!("Doing patch test 2");
|
||||||
|
patch_test(&mut pddb_os, &mut basis_cache, None, None, true);
|
||||||
|
pddb_os.dbg_dump(Some("patche2".to_string()));
|
||||||
|
|
||||||
|
log::info!("Doing delete pattern test 2");
|
||||||
|
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
|
||||||
|
pddb_os.dbg_dump(Some("patterne2".to_string()));
|
||||||
|
|
||||||
|
log::info!("Doing delete/add consistency with data extension 2");
|
||||||
|
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, Some(3),
|
||||||
|
Some(50), None, None);
|
||||||
|
log::info!("Saving `dachecke2` to local host");
|
||||||
|
pddb_os.dbg_dump(Some("dachecke2".to_string()));
|
||||||
|
|
||||||
|
log::info!("Doing delete/add consistency with data extension 3");
|
||||||
|
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, Some(3),
|
||||||
|
Some(50), None, None);
|
||||||
|
log::info!("Saving `dachecke3` to local host");
|
||||||
|
pddb_os.dbg_dump(Some("dachecke3".to_string()));
|
||||||
|
|
||||||
|
log::info!("Doing delete/add consistency with data extension 4");
|
||||||
|
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, Some(6),
|
||||||
|
Some(50), None, None);
|
||||||
|
log::info!("Saving `dachecke4` to local host");
|
||||||
|
pddb_os.dbg_dump(Some("dachecke4".to_string()));
|
||||||
|
|
||||||
log::info!("Doing remount disk test");
|
log::info!("Doing remount disk test");
|
||||||
let mut basis_cache = BasisCache::new();
|
let mut basis_cache = BasisCache::new();
|
||||||
if let Some(sys_basis) = pddb_os.pddb_mount() {
|
if let Some(sys_basis) = pddb_os.pddb_mount() {
|
||||||
|
|
@ -483,12 +512,13 @@ fn xmain() -> ! {
|
||||||
}
|
}
|
||||||
*/
|
*/
|
||||||
/* list of test cases:
|
/* list of test cases:
|
||||||
[done] genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
|
- [done] genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
|
||||||
[done] delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
|
- [done] delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
|
||||||
[done] in-place update consistency: general integrity then patch all keys with a new test pattern
|
- [done] in-place update consistency: general integrity then patch all keys with a new test pattern
|
||||||
[done] extend update consistency: general integrity then patch all keys with a longer test pattern
|
- [done] extend update consistency: general integrity then patch all keys with a longer test pattern
|
||||||
key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data.
|
- [done] key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data.
|
||||||
basis search: create basis A, populate with general integrity. create basis B, add test entries.
|
- fast space exhaustion test: allocate and delete a bunch of stuff. trigger a fast-space regenerate.
|
||||||
|
- basis search: create basis A, populate with general integrity. create basis B, add test entries.
|
||||||
hide basis B, confirm original A; mount basis B, confirm B overlay.
|
hide basis B, confirm original A; mount basis B, confirm B overlay.
|
||||||
*/
|
*/
|
||||||
// register a suspend/resume listener
|
// register a suspend/resume listener
|
||||||
|
|
|
||||||
|
|
@ -135,7 +135,7 @@ def main():
|
||||||
print(keys)
|
print(keys)
|
||||||
|
|
||||||
global MBBB_PAGES
|
global MBBB_PAGES
|
||||||
FSCB_PAGES = 16
|
FSCB_PAGES = 4
|
||||||
KEY_PAGES = 1
|
KEY_PAGES = 1
|
||||||
global PAGE_SIZE
|
global PAGE_SIZE
|
||||||
global VPAGE_SIZE
|
global VPAGE_SIZE
|
||||||
|
|
@ -154,7 +154,7 @@ def main():
|
||||||
img_index += PAGE_SIZE * KEY_PAGES
|
img_index += PAGE_SIZE * KEY_PAGES
|
||||||
mbbb = raw_img[img_index : img_index + PAGE_SIZE * MBBB_PAGES]
|
mbbb = raw_img[img_index : img_index + PAGE_SIZE * MBBB_PAGES]
|
||||||
img_index += PAGE_SIZE * MBBB_PAGES
|
img_index += PAGE_SIZE * MBBB_PAGES
|
||||||
fscb = decode_fscb(raw_img[img_index: img_index + PAGE_SIZE * FSCB_PAGES], keys)
|
fscb = decode_fscb(raw_img[img_index: img_index + PAGE_SIZE * FSCB_PAGES], keys, FSCB_LEN_PAGES=1) ## note this change for testing
|
||||||
img_index += PAGE_SIZE * FSCB_PAGES
|
img_index += PAGE_SIZE * FSCB_PAGES
|
||||||
data = raw_img[img_index:]
|
data = raw_img[img_index:]
|
||||||
|
|
||||||
|
|
@ -610,8 +610,7 @@ class PhysPage:
|
||||||
return 'pp_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal())
|
return 'pp_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal())
|
||||||
|
|
||||||
class Fscb:
|
class Fscb:
|
||||||
def __init__(self, i_bytes):
|
def __init__(self, i_bytes, FASTSPACE_PAGES=1):
|
||||||
FASTSPACE_PAGES = 2
|
|
||||||
NONCE_LEN = 12
|
NONCE_LEN = 12
|
||||||
TAG_LEN = 16
|
TAG_LEN = 16
|
||||||
PP_LEN = 4
|
PP_LEN = 4
|
||||||
|
|
@ -689,6 +688,7 @@ def decode_fscb(img, keys, FSCB_LEN_PAGES=2):
|
||||||
if fscb_start:
|
if fscb_start:
|
||||||
print("Found FSCB at {:x}".format(fscb_start))
|
print("Found FSCB at {:x}".format(fscb_start))
|
||||||
fscb_enc = img[fscb_start * 4096 : (fscb_start + FSCB_LEN_PAGES) * 4096]
|
fscb_enc = img[fscb_start * 4096 : (fscb_start + FSCB_LEN_PAGES) * 4096]
|
||||||
|
print("data: {}".format(fscb_enc.hex()))
|
||||||
try:
|
try:
|
||||||
nonce = fscb_enc[:12]
|
nonce = fscb_enc[:12]
|
||||||
print("key: {}, nonce: {}".format(key.hex(), nonce.hex()))
|
print("key: {}, nonce: {}".format(key.hex(), nonce.hex()))
|
||||||
|
|
@ -697,7 +697,7 @@ def decode_fscb(img, keys, FSCB_LEN_PAGES=2):
|
||||||
print("mac: {}".format(fscb_enc[-16:].hex()))
|
print("mac: {}".format(fscb_enc[-16:].hex()))
|
||||||
#print("data: {}".format(fscb_enc[12:-16].hex()))
|
#print("data: {}".format(fscb_enc[12:-16].hex()))
|
||||||
fscb_dec = cipher.decrypt(fscb_enc[12:], Fscb.aad())
|
fscb_dec = cipher.decrypt(fscb_enc[12:], Fscb.aad())
|
||||||
fscb = Fscb(fscb_dec)
|
fscb = Fscb(fscb_dec, FASTSPACE_PAGES=FSCB_LEN_PAGES)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
print("couldn't decrypt")
|
print("couldn't decrypt")
|
||||||
fscb = None
|
fscb = None
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue