mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-15 08:30:02 +00:00
fastspace regen debugging -- passes test, but needs optimization
should be able to get away with "just" regenerating the fast space log only if that's what we've run out of; however, doing that optimization we run into out of space errors. maybe we need some tuning of the thresholds for when to reclaim space
This commit is contained in:
parent
e02df18374
commit
93ee050079
4 changed files with 75 additions and 15 deletions
|
|
@ -8,7 +8,7 @@ use aes_gcm_siv::{Nonce, Tag};
|
|||
|
||||
/// Each free_pool entry takes about 4 bytes, so give-or-take we have about 1000 free_pool
|
||||
/// entries per page of storage for the free_pool, or 4k * 1000 ~ 4MiB per page, when PhysAddr is a u32
|
||||
pub(crate) const FASTSPACE_PAGES: usize = 2;
|
||||
pub(crate) const FASTSPACE_PAGES: usize = 1;
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Copy, Clone)]
|
||||
#[repr(u8)]
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ use std::io::{Result, Error, ErrorKind};
|
|||
/// Implementation-specific PDDB structures: for Precursor/Xous OS pair
|
||||
|
||||
pub(crate) const MBBB_PAGES: usize = 10;
|
||||
pub(crate) const FSCB_PAGES: usize = 16;
|
||||
pub(crate) const FSCB_PAGES: usize = 4;
|
||||
|
||||
/// size of a physical page
|
||||
pub const PAGE_SIZE: usize = spinor::SPINOR_ERASE_SIZE as usize;
|
||||
|
|
@ -90,6 +90,8 @@ pub(crate) struct PddbOs {
|
|||
fspace_log_addrs: Vec::<PageAlignedPa>,
|
||||
/// memoize the current target offset for the next log entry
|
||||
fspace_log_next_addr: Option<PhysAddr>,
|
||||
/// track roughly how big the log has gotten, so we can pre-emptively garbage collect it before we get too full.
|
||||
fspace_log_len: usize,
|
||||
/// a cached copy of the FPGA's DNA ID, used in the AAA records.
|
||||
dna: u64,
|
||||
/// reference to a TrngPool object that's shared among all the hardware functions
|
||||
|
|
@ -138,6 +140,7 @@ impl PddbOs {
|
|||
fspace_cache: HashSet::<PhysPage>::new(),
|
||||
fspace_log_addrs: Vec::<PageAlignedPa>::new(),
|
||||
fspace_log_next_addr: None,
|
||||
fspace_log_len: 0,
|
||||
dna: llio.soc_dna().unwrap(),
|
||||
entropy: trngpool,
|
||||
};
|
||||
|
|
@ -159,6 +162,7 @@ impl PddbOs {
|
|||
fspace_cache: HashSet::<PhysPage>::new(),
|
||||
fspace_log_addrs: Vec::<PageAlignedPa>::new(),
|
||||
fspace_log_next_addr: None,
|
||||
fspace_log_len: 0,
|
||||
dna: llio.soc_dna().unwrap(),
|
||||
entropy: trngpool,
|
||||
}
|
||||
|
|
@ -543,6 +547,9 @@ impl PddbOs {
|
|||
// commit the fscb data
|
||||
self.patch_fscb(&[&nonce_array, ct_to_flash].concat(), dest_page * PAGE_SIZE as u32);
|
||||
} // end "it would be bad if we lost power now" region
|
||||
|
||||
// note: this function should be followed up by a fast_space_read() to regenerate the temporary
|
||||
// bookkeeping variables that are not reset by this function.
|
||||
} else {
|
||||
panic!("invalid state!");
|
||||
}
|
||||
|
|
@ -656,6 +663,7 @@ impl PddbOs {
|
|||
self.fspace_cache.clear();
|
||||
self.fspace_log_addrs.clear();
|
||||
self.fspace_log_next_addr = None;
|
||||
self.fspace_log_len = 0;
|
||||
|
||||
// let fscb_slice = self.fscb_deref(); // can't use this line because it causse self to be immutably borrowed, so we write out the equivalent below.
|
||||
let fscb_slice = &self.pddb_mr.as_slice()[self.fscb_phys_base.as_usize()..self.fscb_phys_base.as_usize() + FSCB_PAGES * PAGE_SIZE];
|
||||
|
|
@ -873,6 +881,7 @@ impl PddbOs {
|
|||
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
||||
log::trace!("patch: {:x?}", block);
|
||||
self.patch_fscb(&block, log_addr);
|
||||
self.fspace_log_len += 1;
|
||||
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
||||
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
||||
self.fspace_log_next_addr = Some(next_addr as PhysAddr);
|
||||
|
|
@ -908,6 +917,7 @@ impl PddbOs {
|
|||
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
||||
log::trace!("patch: {:x?}", block);
|
||||
self.patch_fscb(&block, log_addr);
|
||||
self.fspace_log_len += 1;
|
||||
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
||||
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
||||
self.fspace_log_next_addr = Some(next_addr as PhysAddr);
|
||||
|
|
@ -921,9 +931,13 @@ impl PddbOs {
|
|||
/// This is a "look before you leap" function that will potentially pause all system operations
|
||||
/// and do a deep scan for space if the required amount is not available.
|
||||
pub fn ensure_fast_space_alloc(&mut self, pages: usize, cache: &Vec::<BasisCacheEntry>) -> bool {
|
||||
if self.fast_space_len() > pages {
|
||||
// make sure we have fast space pages...
|
||||
if (self.fast_space_len() > pages)
|
||||
// ..and make sure we have space for fast space log entries
|
||||
&& (self.fspace_log_len < (FSCB_PAGES - FASTSPACE_PAGES - 1) * PAGE_SIZE / aes::BLOCK_SIZE) {
|
||||
true
|
||||
} else {
|
||||
// if we're really out of space, do an expensive full-space sweep
|
||||
if let Some(used_pages) = self.pddb_generate_used_map(cache) {
|
||||
let free_pool = self.fast_space_generate(used_pages);
|
||||
if free_pool.len() == 0 {
|
||||
|
|
@ -937,6 +951,7 @@ impl PddbOs {
|
|||
*dst = src;
|
||||
}
|
||||
// write just commits a new record to disk, but doesn't update our internal data cache
|
||||
// this also clears the fast space log.
|
||||
self.fast_space_write(&fast_space);
|
||||
// this will ensure the data cache is fully in sync
|
||||
self.fast_space_read();
|
||||
|
|
@ -951,6 +966,21 @@ impl PddbOs {
|
|||
} else {
|
||||
false
|
||||
}
|
||||
/* -- note to self -- "to-do" optimization -- if just out of log pages, we can regenerate those only instead of doing an all-basis sweep
|
||||
let mut fast_space = FastSpace {
|
||||
free_pool: [PhysPage(0); FASTSPACE_FREE_POOL_LEN],
|
||||
};
|
||||
// regenerate from the existing fast space cache
|
||||
for (&src, dst) in self.fspace_cache.iter().zip(fast_space.free_pool.iter_mut()) {
|
||||
*dst = src;
|
||||
}
|
||||
// write just commits a new record to disk, but doesn't update our internal data cache
|
||||
// this also clears the fast space log.
|
||||
self.fast_space_write(&fast_space);
|
||||
// this will re-read back in the data, shuffle the alloc order a bit, and ensure the data cache is fully in sync
|
||||
self.fast_space_read();
|
||||
true
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -423,7 +423,7 @@ fn xmain() -> ! {
|
|||
log::info!("Creating `basecase1e`");
|
||||
let mut basis_cache = BasisCache::new();
|
||||
create_basis_testcase(&mut pddb_os, &mut basis_cache, None,
|
||||
None, None, Some(4));
|
||||
None, None, Some(32));
|
||||
log::info!("Saving `basecase1e` to local host");
|
||||
pddb_os.dbg_dump(Some("basecase1e".to_string()));
|
||||
|
||||
|
|
@ -441,6 +441,35 @@ fn xmain() -> ! {
|
|||
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
|
||||
pddb_os.dbg_dump(Some("patterne".to_string()));
|
||||
|
||||
// extended tests.
|
||||
// allocation space curtailed to force resource exhaustion faster.
|
||||
// note to self: FSCB_PAGES revert to 16 (hw.rs), FASTSPACE_PAGES revert to 2 (fastspace.rs)
|
||||
log::info!("Doing patch test 2");
|
||||
patch_test(&mut pddb_os, &mut basis_cache, None, None, true);
|
||||
pddb_os.dbg_dump(Some("patche2".to_string()));
|
||||
|
||||
log::info!("Doing delete pattern test 2");
|
||||
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
|
||||
pddb_os.dbg_dump(Some("patterne2".to_string()));
|
||||
|
||||
log::info!("Doing delete/add consistency with data extension 2");
|
||||
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, Some(3),
|
||||
Some(50), None, None);
|
||||
log::info!("Saving `dachecke2` to local host");
|
||||
pddb_os.dbg_dump(Some("dachecke2".to_string()));
|
||||
|
||||
log::info!("Doing delete/add consistency with data extension 3");
|
||||
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, Some(3),
|
||||
Some(50), None, None);
|
||||
log::info!("Saving `dachecke3` to local host");
|
||||
pddb_os.dbg_dump(Some("dachecke3".to_string()));
|
||||
|
||||
log::info!("Doing delete/add consistency with data extension 4");
|
||||
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, Some(6),
|
||||
Some(50), None, None);
|
||||
log::info!("Saving `dachecke4` to local host");
|
||||
pddb_os.dbg_dump(Some("dachecke4".to_string()));
|
||||
|
||||
log::info!("Doing remount disk test");
|
||||
let mut basis_cache = BasisCache::new();
|
||||
if let Some(sys_basis) = pddb_os.pddb_mount() {
|
||||
|
|
@ -483,12 +512,13 @@ fn xmain() -> ! {
|
|||
}
|
||||
*/
|
||||
/* list of test cases:
|
||||
[done] genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
|
||||
[done] delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
|
||||
[done] in-place update consistency: general integrity then patch all keys with a new test pattern
|
||||
[done] extend update consistency: general integrity then patch all keys with a longer test pattern
|
||||
key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data.
|
||||
basis search: create basis A, populate with general integrity. create basis B, add test entries.
|
||||
- [done] genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
|
||||
- [done] delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
|
||||
- [done] in-place update consistency: general integrity then patch all keys with a new test pattern
|
||||
- [done] extend update consistency: general integrity then patch all keys with a longer test pattern
|
||||
- [done] key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data.
|
||||
- fast space exhaustion test: allocate and delete a bunch of stuff. trigger a fast-space regenerate.
|
||||
- basis search: create basis A, populate with general integrity. create basis B, add test entries.
|
||||
hide basis B, confirm original A; mount basis B, confirm B overlay.
|
||||
*/
|
||||
// register a suspend/resume listener
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ def main():
|
|||
print(keys)
|
||||
|
||||
global MBBB_PAGES
|
||||
FSCB_PAGES = 16
|
||||
FSCB_PAGES = 4
|
||||
KEY_PAGES = 1
|
||||
global PAGE_SIZE
|
||||
global VPAGE_SIZE
|
||||
|
|
@ -154,7 +154,7 @@ def main():
|
|||
img_index += PAGE_SIZE * KEY_PAGES
|
||||
mbbb = raw_img[img_index : img_index + PAGE_SIZE * MBBB_PAGES]
|
||||
img_index += PAGE_SIZE * MBBB_PAGES
|
||||
fscb = decode_fscb(raw_img[img_index: img_index + PAGE_SIZE * FSCB_PAGES], keys)
|
||||
fscb = decode_fscb(raw_img[img_index: img_index + PAGE_SIZE * FSCB_PAGES], keys, FSCB_LEN_PAGES=1) ## note this change for testing
|
||||
img_index += PAGE_SIZE * FSCB_PAGES
|
||||
data = raw_img[img_index:]
|
||||
|
||||
|
|
@ -610,8 +610,7 @@ class PhysPage:
|
|||
return 'pp_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal())
|
||||
|
||||
class Fscb:
|
||||
def __init__(self, i_bytes):
|
||||
FASTSPACE_PAGES = 2
|
||||
def __init__(self, i_bytes, FASTSPACE_PAGES=1):
|
||||
NONCE_LEN = 12
|
||||
TAG_LEN = 16
|
||||
PP_LEN = 4
|
||||
|
|
@ -689,6 +688,7 @@ def decode_fscb(img, keys, FSCB_LEN_PAGES=2):
|
|||
if fscb_start:
|
||||
print("Found FSCB at {:x}".format(fscb_start))
|
||||
fscb_enc = img[fscb_start * 4096 : (fscb_start + FSCB_LEN_PAGES) * 4096]
|
||||
print("data: {}".format(fscb_enc.hex()))
|
||||
try:
|
||||
nonce = fscb_enc[:12]
|
||||
print("key: {}, nonce: {}".format(key.hex(), nonce.hex()))
|
||||
|
|
@ -697,7 +697,7 @@ def decode_fscb(img, keys, FSCB_LEN_PAGES=2):
|
|||
print("mac: {}".format(fscb_enc[-16:].hex()))
|
||||
#print("data: {}".format(fscb_enc[12:-16].hex()))
|
||||
fscb_dec = cipher.decrypt(fscb_enc[12:], Fscb.aad())
|
||||
fscb = Fscb(fscb_dec)
|
||||
fscb = Fscb(fscb_dec, FASTSPACE_PAGES=FSCB_LEN_PAGES)
|
||||
except KeyError:
|
||||
print("couldn't decrypt")
|
||||
fscb = None
|
||||
|
|
|
|||
Loading…
Reference in a new issue