From 3146e7c379b2895a189d011c95b658e4c69aba88 Mon Sep 17 00:00:00 2001 From: bunnie Date: Sat, 25 May 2024 14:34:43 +0800 Subject: [PATCH] cleanup documentation in the swapper also re-organize code so there is less duplication. --- services/xous-swapper/src/main.rs | 399 ++++++++++++------------------ 1 file changed, 156 insertions(+), 243 deletions(-) diff --git a/services/xous-swapper/src/main.rs b/services/xous-swapper/src/main.rs index eb18eb3d6..35f245a35 100644 --- a/services/xous-swapper/src/main.rs +++ b/services/xous-swapper/src/main.rs @@ -1,58 +1,25 @@ +// TODO: redo docs to match streamlined architecture + //! ==Architecture== //! //! The Xous philosophy is to leave the kernel lightweight and free of dependencies. The swap implementation //! adheres to this by trying to move as much of the difficult algorithmic processing and performance tuning //! outside of the kernel. //! -//! Incidentally, the one thing swap does introduce to the kernel that is algorithm-y -//! is a renormalization routine for counting page accesses. We track page access frequency with a 32-bit -//! "epoch" counter, which is simply incremented whenever a page table interaction happens. We don't use -//! a 64-bit counter because greatly increases the memory used to track things due to the single 64-bit -//! record forcing the next item to also have 64-bit alignment, thus effectively wasting several bytes -//! per page. Anyways, when the epoch is about to roll-over, a mostly in-place sweep with no allocations -//! beyond a few dozen bytes in stack is done to the memory usage tracker to "compact" the epoch numbers down. -//! There is a #[test] in the kernel crate for this routine. +//! The one thing swap does introduce to the kernel that is algorithm-y is a renormalization routine for +//! counting page accesses. We track page access frequency with a 32-bit "epoch" counter, which is simply +//! incremented whenever a page table interaction happens. We don't use a 64-bit counter because greatly +//! increases the memory used to track things due to the single 64-bit record forcing the next item to also +//! have 64-bit alignment, thus effectively wasting several bytes per page. Anyways, when the epoch is about +//! to roll-over, a mostly in-place sweep with no allocations beyond a few dozen bytes in stack is done to the +//! memory usage tracker to "compact" the epoch numbers down. There is a #[test] in the kernel crate for this +//! routine. //! //! In order to perform all the other processing outside of the kernel, the swapper introduces a special new //! "blocking userspace handler". It's "IRQ-like", in that it borrows the same mechanism used for blocking -//! IRQ handlers, but with different entry and exit magic numbers so we can differentiate the two. Anyways, -//! the blocking userspace handler happens with interrupts disabled, giving it an atomic view of all of -//! memory for the duration of the handler. -//! -//! Time spent in this handler of course reduces the responsiveness of the system, so in order to minimize -//! that, we prefer to use the handler to do the minimal processing it has to in the atomic context, and then -//! fire a message off to a normal, preemptable userspace process to do all the fancy handling. As a result, -//! we have two types of OOM handlers. -//! -//! ==OOM Types== -//! -//! There are two types of OOMs handled by the swapper: -//! - "OOM Doom": impending OOM. This is detected by this userspace process, which allows for more -//! sophisticated processing and algorithms to predict an impending OOM than we can fit in the kernel. The -//! OOM Doom handler is also interruptable, which allows other progresses to make progress while memory is -//! being cleared up, hopefully allowing for a more graceful failure curve. The only processing done in -//! the blocking handler is a copy of the kernel memory view, and then a message is issued to the -//! preemptable userspace manager to deal with the rest. -//! - "Hard OOM": Literally no more pages are available. This is triggered when the page allocator can't map -//! a single more page. The hard-OOM handler turns off all interrupts, and all the OOM processing happens -//! within the blocking userspace handler. This effectively turns the hard-OOM handler "inside out" -//! compared to the OOM Doom handler. In order to support this, a set of calls that implement the kernel -//! space pre-amble and post-amble (i.e. `StealPage` and `ReleaseMemory`) are introduced. -//! -//! ==Some Edge Cases== -//! -//! OOM Doom maybe interrupted by the Hard OOM handler (and in fact, this expected because a very greedy -//! process will likely trip a Hard OOM before the OOM Doom is avoided). In order to handle this, separate -//! RPT copy allocations are made for both handlers. This doubles the amount of memory we have to reserve -//! for OOM processing, so if it turns out the OOM Doom is never fast enough to avoid a Hard OOM, we should -//! consider removing it. We also detect when Hard-OOM happens during OOM Doom, and abort OOM Doom processing -//! because our view of the kernel memory maps are stale and invalid. -//! -//! There is an extra edge case in the exit path of the Hard-OOM, namely, if the hard-OOM failed on a page -//! that was swapped. Here, after clearing memory, we have an empty page, but no data; we have to again -//! re-enter the blocking userspace handler to fill that page. The twist here is we have to remember to -//! re-instate interrupts on this edge case otherwise we lose preemption. This is all handled in the kernel -//! proper. +//! IRQ handlers, but with different entry and exit magic numbers so we can differentiate the two. The +//! blocking userspace handler happens with interrupts disabled, giving it an atomic view of all of memory for +//! the duration of the handler. //! //! == Measuring Memory Usage == //! @@ -61,9 +28,9 @@ //! //! OOM pressure is handled with a syscall to the kernel to query the current `MEMORY_ALLOCATIONS` //! table and return the available RAM. This is queried periodically with a timer, and if we -//! fall below a certain threshold, the swapper will try to suggest pages to the kernel to Evict. +//! fall below a certain threshold, the swapper will force a pre-emptive OOM. //! -//! When `swap` is active, `MEMORY_ALLOCATIONS` is upgraded to be a table of +//! When the `swap` feature is selected, `MEMORY_ALLOCATIONS` is upgraded from a `u8` to a table of //! `timestamp | VPN | PID | FLAGS`, where the timestamp is a u32 that is monotonically //! incremented with every modification to the page, and the VPN | PID | FLAGS portion //! is condensed to fit into a u32. The FLAGS can specify if the address is `wired`, which @@ -75,50 +42,11 @@ //! system. Instead, we have a callback to handle when the "epoch" rolls over. //! //! The `MEMORY_ALLOCATIONS` table is page-aligned, so that it can be mapped into PID 2 inside -//! an interrupt context. When the free memory level hits the low water mark, PID 2 invokes -//! a request to the kernel to call the swapper interrupt context with `MEMORY_ALLOCATIONS` -//! mapped into its memory space. At this point, PID 2 will copy the current `MEMORY_ALLOCATIONS` -//! table into a pre-allocated BinaryHeap in the shared state structure, indexed by the timestamp. -//! On completion of the interrupt context, a message is sent back into the swapper userspace -//! using `try_send_message` to trigger the Evict operation. At this point, the Evict operation -//! can work through a sorted vector of allocations to pick the pages it wants to remove. -//! -//! This trades off making the Evict computation a bit more expensive for making all other -//! allocation bookkepping a bit cheaper, because we no longer need to send AllocAdvice messages -//! to the userspace swapper. However, keep in mind that a 2MiB main memory means we have -//! a `MEMORY_ALLOCATIONS` table with only 512 entries, so processing a binary heap of this -//! should have a cost less than decrypting a single page of memory. By ensuring that every -//! Evict pushes out at least 10 or so pages, we can amortize the cost of computing the heap. -//! We also have the advantage that because we have an absolute sorted list of all allocations, -//! we can keep trying evictions from oldest entry to newest entry until we have the correct -//! number of *successes* -- meaning the kernel can override any requested Evictions without -//! impacting the amount of memory that ultimately gets freed up. -//! -//! One thing to note is that the Eviction process should be kicked off when there is sufficient -//! free memory available to complete the BinaryHeap generation + sorted Vec output. One option -//! is to pre-allocate those quantities, potentially with Heapless. -//! -//! A final feature that should be introduced is the kernel needs to have an API call into the -//! swapper, so that it can invoke a chain of events that clears out memory in case it sees -//! the OOM condition. This is done by having the kernel enter the swapper via the interrupt -//! context, and the swapper then issuing the a message that the userspace then handles. - -// TODO: -// - [done] refactor loader to have a swap allocation tracker (MSB of `count` table) -// - move the alloc earlier in the boot process; update page map at the end -// - have phase 1 mark the table -// - [done] implement the WriteToSwap routine - look for the next free page, increment offset counter, write -// memory to swap -// - [done] refactor memory allocation tracking -// - [done] add kernel OOM callback -// - [done] add test command to force eviction of some variable number of pages -// - [done] handle epoch rollovers -// - [done] hook the kernel OOM callback -// - [done] Implement Evictor routine with a thing that checks free memory level, and then more intelligently -// swaps stuff out to create free space for the kernel -// - [done] create test program that over provisions heap versus available memory and prove that swap really -// works. -// - tune the OOM handlers +//! an interrupt context. To initiate OOM handling, PID 2 is invoked by the kernel with a call the swapper +//! interrupt context with `MEMORY_ALLOCATIONS` mapped into its memory space. At this point, PID 2 will copy +//! the current `MEMORY_ALLOCATIONS` table into a pre-allocated BinaryHeap in the shared state structure, +//! indexed by the timestamp. At this point, the blocking userspace handler can work through a sorted vector +//! of allocations to pick the pages it wants to remove. mod debug; mod platform; @@ -135,8 +63,17 @@ use num_traits::*; use platform::{SwapHal, PAGE_SIZE}; use xous::{MemoryFlags, MemoryRange, Message, Result, CID, PID, SID}; -/// Target of pages to free in case of a Hard OOM +/// Target of pages to free in case of a Hard OOM. Note that the PAGE_TARGET numbers +/// are imprecise, in that there is a chance that one target is active during another +/// invocation of a routine. This is because the hard OOM handler is entirely asynchronous +/// and could be invoked at any time, including while we are trying to handle a soft OOM. const HARD_OOM_PAGE_TARGET: usize = 48; +/// Target of pages to free in case of OOM Doom +const OOM_DOOM_PAGE_TARGET: usize = 64; +/// Polling interval for OOM Doom. Slightly off from an even second so we don't have constant +/// competition with other processes that probably use even-second multiples for polling. +const OOM_DOOM_POLL_INTERVAL_MS: u64 = 1057; + /// Virtual address prefixes to de-prioritize in the OomDoom sweep /// 0 - text region /// 4 - message region @@ -181,8 +118,6 @@ impl SwapAbi { #[derive(Debug, num_derive::FromPrimitive, num_derive::ToPrimitive)] #[repr(usize)] pub enum KernelOp { - /// Take the page lent to us, encrypt it and write it to swap - WriteToSwap = 0, /// Find the requested page, decrypt it, and return it ReadFromSwap = 1, /// Hard OOM invocation - stop everything and free memory! @@ -253,7 +188,8 @@ pub struct SwapperSharedState { /// like create L1 page table entries for the swapper to track evicted pages. pub hard_oom_reserved_page: Option, pub report_full_rpt: bool, - /// number of pages to free in the OOM routine + /// number of pages to free in the OOM routine. Note that this value is imprecise: it can + /// be mutated by the userspace soft-OOM handler at any time. pub pages_to_free: usize, } impl SwapperSharedState { @@ -362,6 +298,108 @@ fn map_swap(ss: &mut SwapperSharedState, swap_phys: usize, virt: usize, owner: u l0_pt[vpn0] = ((ppn1 << 20) | (ppn0 << 10) | loader::FLG_VALID) as u32; } +/// Convenience wrapper for GetFreePages syscall +fn get_free_pages() -> usize { + match xous::rsyscall(xous::SysCall::SwapOp(SwapAbi::GetFreePages as usize, 0, 0, 0, 0, 0, 0)) { + Ok(Result::Scalar5(free_pages, _total_memory, _, _, _)) => free_pages, + _ => panic!("GetFreeMem syscall failed"), + } +} + +/// Core of write_to_swap. +fn write_to_swap_inner( + ss: &mut SwapperSharedState, + candidate: SwapAlloc, + errs: &mut usize, + pages_to_free: &mut usize, +) -> core::result::Result<(), xous::Error> { + // step 1: steal the page from the other process. Its data gets mapped into the + // swapper as `local_ptr`. This will also unmap the page from memory. + let vaddr_in_swap = match xous::rsyscall(xous::SysCall::SwapOp( + SwapAbi::StealPage as usize, + candidate.raw_pid() as usize, + candidate.vaddr(), + 0, + 0, + 0, + 0, + )) { + Ok(Result::Scalar5(page_ptr, _, _, _, _)) => page_ptr, + Ok(_) => panic!("Malformed return value"), + Err(_e) => { + *errs += 1; + return Err(xous::Error::ShareViolation); // try another page + } + }; + + // step 2: write the page to swap + #[cfg(feature = "debug-print")] + writeln!( + DebugUart {}, + "WTS PID{}, vaddr_pid {:x}, vaddr_swap {:x}", + candidate.raw_pid(), + candidate.vaddr(), + vaddr_in_swap + ) + .ok(); + // this is safe because the page is aligned and initialized as it comes from the kernel + // remember that this page is overwritten with encrypted data + let buf: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(vaddr_in_swap as *mut u8, PAGE_SIZE) }; + + // search the swap page tables for the next free page + let mut next_free_page: Option = None; + for slot in 0..ss.sct.counts.len() { + let candidate = (ss.free_swap_search_origin + slot) % ss.sct.counts.len(); + if (ss.sct.counts[candidate] & loader::FLG_SWAP_USED) == 0 { + next_free_page = Some(candidate); + break; + } + } + if let Some(free_page_number) = next_free_page { + ss.free_swap_search_origin = free_page_number + 1; // start search at next page beyond the one about to be used + // increment the swap counter by one, rolling over if full. Note that we only have 31 + // bits; the MSB is the "swap used" status bit + let mut count = ss.sct.counts[free_page_number] & !loader::FLG_SWAP_USED; + count = (count + 1) & !loader::FLG_SWAP_USED; + ss.sct.counts[free_page_number] = count | loader::FLG_SWAP_USED; + + // add a PT mapping for the swap entry + map_swap(ss, free_page_number * PAGE_SIZE, candidate.vaddr(), candidate.raw_pid()); + + ss.hal.encrypt_swap_to( + buf, + count, + free_page_number * PAGE_SIZE, + candidate.vaddr(), + candidate.raw_pid(), + ); + } else { + writeln!(DebugUart {}, "OOM detected, dumping all swap allocs:").ok(); + for (i, &entry) in ss.sct.counts.iter().enumerate() { + writeln!(DebugUart {}, " {:04}:{:x}", i, entry).ok(); + } + // OOS path + panic!("Ran out of swap space, hard OOM!"); + } + + // step 3: release the page (currently mapped into the swapper's memory space). Need + // to demonstrate to the memory system that we know what we are + // doing by also presenting the original PID that owned the page. + xous::rsyscall(xous::SysCall::SwapOp( + SwapAbi::ReleaseMemory as usize, + vaddr_in_swap, + candidate.raw_pid() as usize, + 0, + 0, + 0, + 0, + )) + .expect("Unexpected error: couldn't release a page that was mapped into the swapper's space"); + *pages_to_free -= 1; + + Ok(()) +} + /// blocking swap call handler /// 8 argument values are always pushed on the stack; the meaning is bound differently based upon the specific /// opcode. Not all arguments are used in all cases, unused argument values have no valid meaning (but in @@ -448,22 +486,6 @@ fn swap_handler( #[cfg(feature = "debug-print-verbose")] writeln!(DebugUart {}, "got Opcode: {:?}", op).ok(); match op { - Some(KernelOp::WriteToSwap) => { - let pid = a2 as u8; - let vaddr_in_pid = a3; - let vaddr_in_swap = a4; - write_to_swap_inner(ss, pid, vaddr_in_pid, vaddr_in_swap); - #[cfg(feature = "debug-verbose")] - { - writeln!(DebugUart {}, "Swap count & usage table:").ok(); - for (i, &entry) in ss.sct.counts.iter().enumerate() { - if entry != 0 { - writeln!(DebugUart {}, " {:04}:{:x}", i, entry).ok(); - } - } - } - // writeln!(DebugUart {}, "WTS exit").ok(); - } Some(KernelOp::ReadFromSwap) => { let pid = a2 as u8; let vaddr_in_pid = a3; @@ -555,17 +577,14 @@ fn swap_handler( // stored thread `sepc`. The syscalls used here are all "simple calls" that don't require re-entry // into the swapper context to handle. let target_pages = pages_to_free; - let mut errs = 0; - let mut wired = 0; + let mut errs: usize = 0; + let mut wired: usize = 0; // A holding variable for items that we're de-prioritizing from removal. We only try these // if we've exhausted all the other options. let mut deprioritized = VecDeque::new(); - loop { - if pages_to_free == 0 { - break; - } + while pages_to_free > 0 { if let Some(candidate) = alloc_heap.pop() { if candidate.is_wired() || !candidate.is_valid() @@ -579,85 +598,11 @@ fn swap_handler( deprioritized.push_back(candidate); continue; } - - // step 1: steal the page from the other process. Its data gets mapped into the - // swapper as `local_ptr`. This will also unmap the page from memory. - let local_ptr = match xous::rsyscall(xous::SysCall::SwapOp( - SwapAbi::StealPage as usize, - candidate.raw_pid() as usize, - candidate.vaddr(), - 0, - 0, - 0, - 0, - )) { - Ok(Result::Scalar5(page_ptr, _, _, _, _)) => page_ptr, - Ok(_) => panic!("Malformed return value"), - Err(_e) => { - errs += 1; - continue; // try another page - } - }; - - // step 2: write the page to swap. - write_to_swap_inner(ss, candidate.raw_pid(), candidate.vaddr(), local_ptr); - - // step 3: release the page (currently mapped into the swapper's memory space). Need - // to demonstrate to the memory system that we know what we are - // doing by also presenting the original PID that owned the page. - xous::rsyscall(xous::SysCall::SwapOp( - SwapAbi::ReleaseMemory as usize, - local_ptr, - candidate.raw_pid() as usize, - 0, - 0, - 0, - 0, - )).expect("Unexpected error: couldn't release a page that was mapped into the swapper's space"); - pages_to_free -= 1; + // error is ignored because the correct behavior on error is to try another page + write_to_swap_inner(ss, candidate, &mut errs, &mut pages_to_free).ok(); } } else if let Some(candidate) = deprioritized.pop_front() { - // copy of code above. TODO: figure out some way to reduce this duplication, - // without impacting error tracking, etc. - - // step 1: steal the page from the other process. Its data gets mapped into the - // swapper as `local_ptr`. This will also unmap the page from memory. - let local_ptr = match xous::rsyscall(xous::SysCall::SwapOp( - SwapAbi::StealPage as usize, - candidate.raw_pid() as usize, - candidate.vaddr(), - 0, - 0, - 0, - 0, - )) { - Ok(Result::Scalar5(page_ptr, _, _, _, _)) => page_ptr, - Ok(_) => panic!("Malformed return value"), - Err(_e) => { - errs += 1; - continue; // try another page - } - }; - - // step 2: write the page to swap. - write_to_swap_inner(ss, candidate.raw_pid(), candidate.vaddr(), local_ptr); - - // step 3: release the page (currently mapped into the swapper's memory space). Need - // to demonstrate to the memory system that we know what we are - // doing by also presenting the original PID that owned the page. - xous::rsyscall(xous::SysCall::SwapOp( - SwapAbi::ReleaseMemory as usize, - local_ptr, - candidate.raw_pid() as usize, - 0, - 0, - 0, - 0, - )) - .expect( - "Unexpected error: couldn't release a page that was mapped into the swapper's space", - ); - pages_to_free -= 1; + write_to_swap_inner(ss, candidate, &mut errs, &mut pages_to_free).ok(); } else { writeln!( DebugUart {}, @@ -747,6 +692,21 @@ fn main() { } }); + // This thread pings the free memory level and will try to clear memory to avoid OOM + thread::spawn({ + move || { + loop { + sleep(Duration::from_millis(OOM_DOOM_POLL_INTERVAL_MS)); + if get_free_pages() < OOM_DOOM_PAGE_TARGET { + sss.inner.as_mut().unwrap().pages_to_free = OOM_DOOM_PAGE_TARGET; + xous::rsyscall(xous::SysCall::SwapOp(SwapAbi::ClearMemoryNow as usize, 0, 0, 0, 0, 0, 0)) + .expect("ClearMemoryNow syscall failed"); + sss.inner.as_mut().unwrap().pages_to_free = HARD_OOM_PAGE_TARGET; + } + } + } + }); + let mut msg_opt = None; loop { xous::reply_and_receive_next(sid, &mut msg_opt).unwrap(); @@ -756,7 +716,7 @@ fn main() { match op { #[cfg(feature = "swap-userspace-testing")] Some(Opcode::Test0) => { - log::info!("Free mem before clearing: {}kiB", get_free_pages() * PAGE_SIZE / 1024); + log::info!("Free mem: {}kiB", get_free_pages() * PAGE_SIZE / 1024); } _ => { log::info!("Unknown opcode {:?}", op); @@ -764,50 +724,3 @@ fn main() { } } } - -fn get_free_pages() -> usize { - match xous::rsyscall(xous::SysCall::SwapOp(SwapAbi::GetFreePages as usize, 0, 0, 0, 0, 0, 0)) { - Ok(Result::Scalar5(free_pages, _total_memory, _, _, _)) => free_pages, - _ => panic!("GetFreeMem syscall failed"), - } -} - -/// Core of write_to_swap: this is also used by HardOom. -fn write_to_swap_inner(ss: &mut SwapperSharedState, pid: u8, vaddr_in_pid: usize, vaddr_in_swap: usize) { - #[cfg(feature = "debug-print")] - writeln!(DebugUart {}, "WTS PID{}, vaddr_pid {:x}, vaddr_swap {:x}", pid, vaddr_in_pid, vaddr_in_swap) - .ok(); - // this is safe because the page is aligned and initialized as it comes from the kernel - // remember that this page is overwritten with encrypted data - let buf: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(vaddr_in_swap as *mut u8, PAGE_SIZE) }; - - // search the swap page tables for the next free page - let mut next_free_page: Option = None; - for slot in 0..ss.sct.counts.len() { - let candidate = (ss.free_swap_search_origin + slot) % ss.sct.counts.len(); - if (ss.sct.counts[candidate] & loader::FLG_SWAP_USED) == 0 { - next_free_page = Some(candidate); - break; - } - } - if let Some(free_page_number) = next_free_page { - ss.free_swap_search_origin = free_page_number + 1; // start search at next page beyond the one about to be used - // increment the swap counter by one, rolling over if full. Note that we only have 31 - // bits; the MSB is the "swap used" status bit - let mut count = ss.sct.counts[free_page_number] & !loader::FLG_SWAP_USED; - count = (count + 1) & !loader::FLG_SWAP_USED; - ss.sct.counts[free_page_number] = count | loader::FLG_SWAP_USED; - - // add a PT mapping for the swap entry - map_swap(ss, free_page_number * PAGE_SIZE, vaddr_in_pid, pid); - - ss.hal.encrypt_swap_to(buf, count, free_page_number * PAGE_SIZE, vaddr_in_pid, pid); - } else { - writeln!(DebugUart {}, "OOM detected, dumping all swap allocs:").ok(); - for (i, &entry) in ss.sct.counts.iter().enumerate() { - writeln!(DebugUart {}, " {:04}:{:x}", i, entry).ok(); - } - // OOS path - panic!("Ran out of swap space, hard OOM!"); - } -}