add a periodic scrub to the PDDB

this periodically polls the PDDB and has it check the memory allocation
levels, and if too high, invokes a scrub attempt.

This bit is probably the most poorly tested of all the code, but
also some of the simpler logic. The more important part this code
brings is analytics on actual heap and cache usage, which will help
inform us on how urgent/bad things are getting in terms of PDDB
cache usage.
This commit is contained in:
bunnie 2022-06-17 02:02:11 +08:00
parent dc0a4e5400
commit 4c18662430
2 changed files with 64 additions and 0 deletions

View file

@ -114,6 +114,9 @@ pub(crate) enum Opcode {
WriteKey,
WriteKeyFlush,
// GC methods
PeriodicScrub,
/// drops any connection state associated with a given key
KeyDrop,

View file

@ -508,6 +508,23 @@ fn main() -> ! {
).expect("couldn't send mount request");
}
});
// a thread to trigger period scrubbing of the PDDB
let scrub_run = Arc::new(AtomicBool::new(false));
let _ = thread::spawn({
let my_cid = my_cid.clone();
let scrub_run = scrub_run.clone();
move || {
let tt = ticktimer_server::Ticktimer::new().unwrap();
loop {
tt.sleep_ms(12_513).unwrap(); // arbitrary interval, but trying to avoid "round" numbers of seconds to interleave periodic tasks
if scrub_run.load(Ordering::SeqCst) {
send_message(my_cid,
Message::new_scalar(Opcode::PeriodicScrub.to_usize().unwrap(), 0, 0, 0, 0)
).expect("couldn't send mount request");
}
}
}
});
// main server loop
let mut key_list = Vec::<String>::new(); // storage for key lists
let mut key_token: Option<[u32; 4]> = None;
@ -521,6 +538,13 @@ fn main() -> ! {
// track processes that want a notification of a mount event
let mut mount_notifications = Vec::<xous::MessageSender>::new();
// track heap usage
let mut initial_heap: usize = 0;
let mut latest_heap: usize = 0;
let mut latest_cache: usize = 0;
const HEAP_GC_THRESH: usize = 400 * 1024; // the heap breaks at 512kiB, so try cleaning up at 400k
const HEAP_GC_TARGET: usize = 64 * 1024; // how much to try cleaning out in any one go.
// register a suspend/resume listener
let mut susres = susres::Susres::new(Some(susres::SuspendOrder::Early), &xns,
Opcode::SuspendResume as u32, my_cid).expect("couldn't create suspend/resume object");
@ -584,9 +608,33 @@ fn main() -> ! {
// user aborted procedure
_ => xous::return_scalar(msg.sender, 0).expect("couldn't return scalar"),
}
initial_heap = heap_usage();
latest_heap = initial_heap;
latest_cache = basis_cache.cache_size();
log::info!("PDDB post-mount caching stats: {} heap, {} cache", latest_heap, latest_cache);
scrub_run.store(true, Ordering::SeqCst);
}
}
}),
Some(Opcode::PeriodicScrub) => {
let current_heap = heap_usage();
let current_cache = basis_cache.cache_size();
if current_heap != latest_heap || current_cache != latest_cache {
log::info!("PDDB caching stats: {} heap, {} cache", latest_heap, current_cache);
}
latest_heap = current_heap;
latest_cache = current_cache;
if (latest_heap > initial_heap
&& (latest_heap - initial_heap) > HEAP_GC_THRESH)
// this line is mostly so this triggers occassionally in hosted mode where heap usage is faked;
// in practice heap threshold will always hit before cache threshold
|| current_cache > HEAP_GC_THRESH {
log::info!("PDDB trim threshold reached: {} heap, {} cache", latest_heap, basis_cache.cache_size());
let pruned = basis_cache.cache_prune(&mut pddb_os, HEAP_GC_TARGET);
latest_heap = heap_usage();
log::info!("{} pruned, now: {} heap, {} cache", pruned, latest_heap, basis_cache.cache_size())
}
}
Some(Opcode::ListBasis) => {
let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
let mut list_ipc = buffer.to_original::<PddbBasisList, _>().unwrap();
@ -1461,3 +1509,16 @@ fn notify_of_disconnect(pddb_os: &mut PddbOs, token_dict: &HashMap::<ApiToken, T
}
}
}
pub(crate) fn heap_usage() -> usize {
match xous::rsyscall(xous::SysCall::IncreaseHeap(0, xous::MemoryFlags::R)).expect("couldn't get heap size") {
xous::Result::MemoryRange(m) => {
let usage = m.len();
usage
}
_ => {
log::error!("Couldn't measure heap usage");
0
},
}
}