From 2ea6c83bf07cea01efd6e5f25467657d5121e83f Mon Sep 17 00:00:00 2001 From: bunnie Date: Tue, 21 May 2024 01:42:19 +0800 Subject: [PATCH] implement the userspace side of hard-oom and oom-doom handlers see the comments added in the main.rs file for a discussion of all the things. --- services/xous-swapper/Cargo.toml | 1 + services/xous-swapper/src/main.rs | 432 ++++++++++++++++++++++++------ 2 files changed, 354 insertions(+), 79 deletions(-) diff --git a/services/xous-swapper/Cargo.toml b/services/xous-swapper/Cargo.toml index 94f85a4c5..b3a52d118 100644 --- a/services/xous-swapper/Cargo.toml +++ b/services/xous-swapper/Cargo.toml @@ -34,6 +34,7 @@ cramium-fpga = ["utralib/cramium-fpga"] precursor = ["utralib/precursor"] hosted = ["utralib/hosted"] renode = ["utralib/renode"] +swap-userspace-testing = [] # for now, debug-print only works on Precursor targets debug-print = [] diff --git a/services/xous-swapper/src/main.rs b/services/xous-swapper/src/main.rs index 86ef3d527..d50750cfd 100644 --- a/services/xous-swapper/src/main.rs +++ b/services/xous-swapper/src/main.rs @@ -1,3 +1,61 @@ +//! ==Architecture== +//! +//! The Xous philosophy is to leave the kernel lightweight and free of dependencies. The swap implementation +//! adheres to this by trying to move as much of the difficult algorithmic processing and performance tuning +//! outside of the kernel. +//! +//! Incidentally, the one thing swap does introduce to the kernel that is algorithm-y +//! is a renormalization routine for counting page accesses. We track page access frequency with a 32-bit +//! "epoch" counter, which is simply incremented whenever a page table interaction happens. We don't use +//! a 64-bit counter because greatly increases the memory used to track things due to the single 64-bit +//! record forcing the next item to also have 64-bit alignment, thus effectively wasting several bytes +//! per page. Anyways, when the epoch is about to roll-over, a mostly in-place sweep with no allocations +//! beyond a few dozen bytes in stack is done to the memory usage tracker to "compact" the epoch numbers down. +//! There is a #[test] in the kernel crate for this routine. +//! +//! In order to perform all the other processing outside of the kernel, the swapper introduces a special new +//! "blocking userspace handler". It's "IRQ-like", in that it borrows the same mechanism used for blocking +//! IRQ handlers, but with different entry and exit magic numbers so we can differentiate the two. Anyways, +//! the blocking userspace handler happens with interrupts disabled, giving it an atomic view of all of +//! memory for the duration of the handler. +//! +//! Time spent in this handler of course reduces the responsiveness of the system, so in order to minimize +//! that, we prefer to use the handler to do the minimal processing it has to in the atomic context, and then +//! fire a message off to a normal, preemptable userspace process to do all the fancy handling. As a result, +//! we have two types of OOM handlers. +//! +//! ==OOM Types== +//! +//! There are two types of OOMs handled by the swapper: +//! - "OOM Doom": impending OOM. This is detected by this userspace process, which allows for more +//! sophisticated processing and algorithms to predict an impending OOM than we can fit in the kernel. The +//! OOM Doom handler is also interruptable, which allows other progresses to make progress while memory is +//! being cleared up, hopefully allowing for a more graceful failure curve. The only processing done in +//! the blocking handler is a copy of the kernel memory view, and then a message is issued to the +//! preemptable userspace manager to deal with the rest. +//! - "Hard OOM": Literally no more pages are available. This is triggered when the page allocator can't map +//! a single more page. The hard-OOM handler turns off all interrupts, and all the OOM processing happens +//! within the blocking userspace handler. This effectively turns the hard-OOM handler "inside out" +//! compared to the OOM Doom handler. In order to support this, a set of calls that implement the kernel +//! space pre-amble and post-amble (i.e. `StealPage` and `ReleaseMemory`) are introduced. +//! +//! ==Some Edge Cases== +//! +//! OOM Doom maybe interrupted by the Hard OOM handler (and in fact, this expected because a very greedy +//! process will likely trip a Hard OOM before the OOM Doom is avoided). In order to handle this, separate +//! RPT copy allocations are made for both handlers. This doubles the amount of memory we have to reserve +//! for OOM processing, so if it turns out the OOM Doom is never fast enough to avoid a Hard OOM, we should +//! consider removing it. We also detect when Hard-OOM happens during OOM Doom, and abort OOM Doom processing +//! because our view of the kernel memory maps are stale and invalid. +//! +//! There is an extra edge case in the exit path of the Hard-OOM, namely, if the hard-OOM failed on a page +//! that was swapped. Here, after clearing memory, we have an empty page, but no data; we have to again +//! re-enter the blocking userspace handler to fill that page. The twist here is we have to remember to +//! re-instate interrupts on this edge case otherwise we lose preemption. This is all handled in the kernel +//! proper. +//! +//! == Measuring Memory Usage == +//! //! The swapper needs to come up with an answer for which page to swap out, and it //! also needs to know when to do it (OOM pressure). //! @@ -54,11 +112,13 @@ // - [done] refactor memory allocation tracking // - [done] add kernel OOM callback // - [done] add test command to force eviction of some variable number of pages -// - handle epoch rollovers -// - hook the kernel OOM callback somewhere...reasonable. -// - [partially done] Implement Evictor routine with a thing that checks free memory level, and then more -// intelligently swaps stuff out to create free space for the kernel -// - create test program that over provisions heap versus available memory and prove that swap really works. +// - [done] handle epoch rollovers +// - [done] hook the kernel OOM callback +// - [done] Implement Evictor routine with a thing that checks free memory level, and then more intelligently +// swaps stuff out to create free space for the kernel +// - [done] create test program that over provisions heap versus available memory and prove that swap really +// works. +// - tune the OOM handlers mod debug; mod platform; @@ -74,7 +134,7 @@ use debug::*; use loader::swap::{SwapAlloc, SwapSpec, SWAP_CFG_VADDR, SWAP_COUNT_VADDR, SWAP_PT_VADDR, SWAP_RPT_VADDR}; use num_traits::*; use platform::{SwapHal, PAGE_SIZE}; -use xous::{MemoryFlags, Message, Result, CID, PID, SID}; +use xous::{MemoryFlags, MemoryRange, Message, Result, CID, PID, SID}; /// userspace swapper -> kernel ABI /// This ABI is copy-paste synchronized with what's in the kernel. It's left out of @@ -89,6 +149,8 @@ pub enum SwapAbi { GetFreePages = 2, FetchAllocs = 3, SetOomThresh = 4, + StealPage = 5, + ReleaseMemory = 6, } /// SYNC WITH `kernel/src/swap.rs` impl SwapAbi { @@ -99,6 +161,8 @@ impl SwapAbi { 2 => GetFreePages, 3 => FetchAllocs, 4 => SetOomThresh, + 5 => StealPage, + 6 => ReleaseMemory, _ => Invalid, } } @@ -117,8 +181,8 @@ pub enum KernelOp { ReadFromSwap = 1, /// Kernel message advising us that a page of RAM was allocated ExecFetchAllocs = 2, - /// OOM warning from the kernel - OomDoom = 3, + /// Hard OOM invocation - stop everything and free memory! + HardOom = 3, } /// public userspace & swapper handler -> swapper userspace ABI @@ -130,6 +194,7 @@ pub enum Opcode { /// Trigger the OOM routine HandleOomDoom, /// Test messages + #[cfg(feature = "swap-userspace-testing")] Test0, } @@ -150,6 +215,20 @@ pub struct SwapCountTracker { pub counts: &'static mut [u32], } +/// Track if the hard OOM handler ram during the Oom Doom userspace routine +static MAYBE_HARD_OOM_DURING_OOM_DOOM: AtomicBool = AtomicBool::new(false); + +/// Number of pages to reserve for hard OOM handling. In case of a hard OOM, there are 0 pages +/// available, which makes it impossible for the hard OOM handler to do things like allocate an L1 +/// page table entry to track memory being swapped out. This places a hold on some memory that's +/// de-allocated on entry to the hard OOM handler, and re-allocated on exit. +/// +/// Known things the swapper has to allocate memory for in hard-OOM: +/// - L1 page table entries for tracking swap +/// - A second page will be requested in case of panic in hard OOM for TLS, but these shouldn't happen so we +/// don't reserve it. +const HARD_OOM_RESERVED_PAGES: usize = 1; + /// This structure contains shared state accessible between the userspace code and the blocking swap call /// handler. pub struct SwapperSharedState { @@ -171,28 +250,41 @@ pub struct SwapperSharedState { /// starting from the free swap search origin. The unit of this variable is in pages, so it /// can be used to directly index the `sct` `SwapCountTracker`. pub free_swap_search_origin: usize, + /// alloc heap for the OomDoom handler pub alloc_heap: Option>, + /// alloc heap for the Hard OOM handler. Must be separate because it's possible for both + /// to run at the same time. + pub hard_oom_alloc_heap: Option>, + /// Reserve some memory to be freed by the hard OOM manager. These pages are needed to do things + /// like create L1 page table entries for the swapper to track evicted pages. + pub hard_oom_reserved_page: Option, + pub report_full_rpt: bool, } impl SwapperSharedState { pub fn pt_walk(&self, pid: u8, va: usize, mark_free: bool) -> Option { let l1_pt = &self.pts.roots[pid as usize - 1]; - let l1_entry = (l1_pt.entries[(va & 0xFFC0_0000) >> 22] >> 10) << 12; + // mask out bottom 10 bits of flags, shift left by 2 to create the address of L0 table + let l1_entry = l1_pt.entries[va >> 22]; + let l0_address = (l1_entry & 0xFFFF_FC00) << 2; - if l1_entry != 0 { + // 0xF means this also checks that RWX is 0, as well as FLG_VALID is true + if (l1_entry & 0xF) == loader::FLG_VALID as u32 { // this is safe because all possible values can be represented as `u32`, the pointer // is valid, aligned, and the bounds are known. - let l0_pt = unsafe { core::slice::from_raw_parts_mut(l1_entry as *mut u32, 1024) }; + let l0_pt = unsafe { core::slice::from_raw_parts_mut(l0_address as *mut u32, 1024) }; let l0_entry = l0_pt[(va & 0x003F_F000) >> 12]; - if l0_entry & 1 != 0 { + if (l0_entry & loader::FLG_VALID as u32) != 0 { if mark_free { l0_pt[(va & 0x003F_F000) >> 12] = 0; } - // bit 1 is the "valid" bit - Some(((l0_entry as usize >> 10) << 12) | va & 0xFFF) + Some(((l0_entry as usize & 0xFFFF_FC00) << 2) | va & 0xFFF) } else { + // writeln!(DebugUart {}, "pt_walk L0 entry invalid: {:x}", l0_entry).ok(); None } } else { + assert!((l1_entry & 0xE) == 0, "RWX was not zero on L1 PTE, unsupported mode of operation!"); + // writeln!(DebugUart {}, "pt_walk L1 entry invalid: {:x}", l1_entry).ok(); None } } @@ -247,14 +339,14 @@ fn map_swap(ss: &mut SwapperSharedState, swap_phys: usize, virt: usize, owner: u .ok(); // Mark this entry as a leaf node (WRX as 0), and indicate // it is a valid page by setting "V". - l1_pt[vpn1] = (((na >> 12) << 10) | loader::FLG_VALID) as u32; + l1_pt[vpn1] = (((na & 0xFFFF_F000) >> 2) | loader::FLG_VALID) as u32; } let l0_pt_idx = unsafe { &mut (*(((l1_pt[vpn1] << 2) & !((1 << 12) - 1)) as *mut PtPage)) }; let l0_pt = &mut l0_pt_idx.entries; // Ensure the entry hasn't already been mapped to a different address. - if (l0_pt[vpn0] as usize) & 1 != 0 + if ((l0_pt[vpn0] as usize) & loader::FLG_VALID) != 0 && (l0_pt[vpn0] as usize & 0xffff_fc00) != ((ppn1 << 20) | (ppn0 << 10)) { // Panics don't print from the swapper in this context - must use the DebugUart to have the message @@ -270,8 +362,7 @@ fn map_swap(ss: &mut SwapperSharedState, swap_phys: usize, virt: usize, owner: u .ok(); panic!("Swap page already allocated"); } - let previous_flags = l0_pt[vpn0] as usize & 0x3f; - l0_pt[vpn0] = ((ppn1 << 20) | (ppn0 << 10) | previous_flags | loader::FLG_VALID) as u32; + l0_pt[vpn0] = ((ppn1 << 20) | (ppn0 << 10) | loader::FLG_VALID) as u32; } /// blocking swap call handler @@ -302,6 +393,18 @@ fn swap_handler( // have to do mapping requests because it's already done for us! let swap_spec = unsafe { &*(SWAP_CFG_VADDR as *mut SwapSpec) }; + // reserve memory for hard OOM + let mut reserved = xous::map_memory( + None, + None, + PAGE_SIZE * HARD_OOM_RESERVED_PAGES, + MemoryFlags::R | MemoryFlags::W | MemoryFlags::RESERVE, + ) + .expect("could't reserve space for hard OOM handler"); + // *touch* the memory -- otherwise it might not actually be demand-paged + let reserved_slice: &mut [u32] = unsafe { reserved.as_slice_mut() }; // this is safe because `u32` is fully representable + reserved_slice.fill(0); + // swapper is not allowed to use `log` for debugging under most circumstances, because // the swapper can't send messages when handling a swap call. Instead, we use a local // debug UART to handle this. This needs to be enabled with the "debug-print" feature @@ -337,6 +440,9 @@ fn swap_handler( sram_size: swap_spec.sram_size as usize, free_swap_search_origin: 0, alloc_heap: None, + hard_oom_alloc_heap: None, + report_full_rpt: true, + hard_oom_reserved_page: Some(reserved), }); } let ss = sss.inner.as_mut().expect("Shared state should be initialized"); @@ -348,48 +454,7 @@ fn swap_handler( let pid = a2 as u8; let vaddr_in_pid = a3; let vaddr_in_swap = a4; - writeln!( - DebugUart {}, - "WTS PID{}, vaddr_pid {:x}, vaddr_swap {:x}", - pid, - vaddr_in_pid, - vaddr_in_swap - ) - .ok(); - // this is safe because the page is aligned and initialized as it comes from the kernel - // remember that this page is overwritten with encrypted data - let buf: &mut [u8] = - unsafe { core::slice::from_raw_parts_mut(vaddr_in_swap as *mut u8, PAGE_SIZE) }; - - // search the swap page tables for the next free page - let mut next_free_page: Option = None; - for slot in 0..ss.sct.counts.len() { - let candidate = (ss.free_swap_search_origin + slot) % ss.sct.counts.len(); - if (ss.sct.counts[candidate] & loader::FLG_SWAP_USED) == 0 { - next_free_page = Some(candidate); - break; - } - } - if let Some(free_page_number) = next_free_page { - ss.free_swap_search_origin = free_page_number + 1; // start search at next page beyond the one about to be used - // increment the swap counter by one, rolling over if full. Note that we only have 31 - // bits; the MSB is the "swap used" status bit - let mut count = ss.sct.counts[free_page_number] & !loader::FLG_SWAP_USED; - count = (count + 1) & !loader::FLG_SWAP_USED; - ss.sct.counts[free_page_number] = count | loader::FLG_SWAP_USED; - - // add a PT mapping for the swap entry - map_swap(ss, free_page_number * PAGE_SIZE, vaddr_in_pid, pid); - - ss.hal.encrypt_swap_to(buf, count, free_page_number * PAGE_SIZE, vaddr_in_pid, pid); - } else { - writeln!(DebugUart {}, "OOM detected, dumping all swap allocs:").ok(); - for (i, &entry) in ss.sct.counts.iter().enumerate() { - writeln!(DebugUart {}, " {:04}:{:x}", i, entry).ok(); - } - // OOS path - panic!("Ran out of swap space, hard OOM!"); - } + write_to_swap_inner(ss, pid, vaddr_in_pid, vaddr_in_swap); #[cfg(feature = "debug-verbose")] { writeln!(DebugUart {}, "Swap count & usage table:").ok(); @@ -414,24 +479,39 @@ fn swap_handler( ) .ok(); // walk the PT to find the swap data, and remove it from the swap PT - let paddr_in_swap = ss - .pt_walk(pid as u8, vaddr_in_pid, true) - .expect("Couldn't resolve swapped data. Was the page actually swapped?"); + let paddr_in_swap = match ss.pt_walk(pid as u8, vaddr_in_pid, true) { + Some(paddr) => paddr, + None => { + writeln!(DebugUart {}, "Couldn't resolve swapped data. Was the page actually swapped?") + .ok(); + panic!("Couldn't resolve swapped data. Was the page actually swapped?") + } + }; + writeln!(DebugUart {}, " resolved paddr_in_swap {:x}", paddr_in_swap).ok(); // clear the used bit in swap ss.sct.counts[paddr_in_swap / PAGE_SIZE] &= !loader::FLG_SWAP_USED; // safety: this is only safe because the pointer we're passed from the kernel is guaranteed to be // a valid u8-page in memory let buf = unsafe { core::slice::from_raw_parts_mut(vaddr_in_swap as *mut u8, PAGE_SIZE) }; - ss.hal - .decrypt_swap_from( - buf, - ss.sct.counts[paddr_in_swap / PAGE_SIZE], - paddr_in_swap, - vaddr_in_pid, - pid, - ) - .expect("Decryption error: swap image corrupted, the tag does not match the data!"); + match ss.hal.decrypt_swap_from( + buf, + ss.sct.counts[paddr_in_swap / PAGE_SIZE], + paddr_in_swap, + vaddr_in_pid, + pid, + ) { + Ok(_) => {} + Err(e) => { + writeln!( + DebugUart {}, + "Decryption error: swap image corrupted, the tag does not match the data! {:?}", + e + ) + .ok(); + panic!("Decryption error: swap image corrupted, the tag does not match the data!"); + } + } // at this point, the `buf` has our desired data, we're done, modulo updating the count. } Some(KernelOp::ExecFetchAllocs) => { @@ -448,7 +528,10 @@ fn swap_handler( .ok(); } // filter out invalid, wired, or kernel/swapper candidates. - if !entry.is_wired() && entry.is_valid() && entry.raw_pid() != 1 && entry.raw_pid() != 2 { + // report everything if requested (this is used to wire our heap memory prior to an OOM) + if (!entry.is_wired() && entry.is_valid() && entry.raw_pid() != 1 && entry.raw_pid() != 2) + || ss.report_full_rpt + { // writeln!(DebugUart {}, "Pushing {:x?}", entry).ok(); alloc_heap.push(entry); } @@ -462,6 +545,125 @@ fn swap_handler( .ok(); } } + Some(KernelOp::HardOom) => { + // parse the arguments + // be sure to allocate some extra space for the handler itself to run the next time! + let mut pages_to_free = a2 + HARD_OOM_RESERVED_PAGES; + + // set the flag that hard oom ran during an Oom Doom handler pass + MAYBE_HARD_OOM_DURING_OOM_DOOM.store(true, Ordering::SeqCst); + + // free memory for the hard-OOM handler to run + if let Some(reserved_mem) = ss.hard_oom_reserved_page.take() { + writeln!(DebugUart {}, "Entering HARD OOM - freeing scratch memory: {:x?}", reserved_mem) + .ok(); + xous::unmap_memory(reserved_mem).expect("couldn't free memory for hard OOM handler"); + } else { + panic!("No space was reserved for the hard OOM manager to run!"); + } + // recover the RPT from kernel + let mut alloc_heap = + ss.hard_oom_alloc_heap.take().expect("Hard OOM, but no pre-allocated storage for handler!"); + alloc_heap.clear(); + assert!(alloc_heap.len() == 0); + let rpt = unsafe { + core::slice::from_raw_parts(SWAP_RPT_VADDR as *const SwapAlloc, ss.sram_size / PAGE_SIZE) + }; + for (_i, &entry) in rpt.iter().enumerate() { + // filter out invalid, wired, or kernel/swapper candidates. + if !entry.is_wired() && entry.is_valid() && entry.raw_pid() != 1 && entry.raw_pid() != 2 { + // writeln!(DebugUart {}, "Pushing {:x?}", entry).ok(); + alloc_heap.push(entry); + } + } + // Inside the interrupt context, evict pages. No progress on any other process is made until this + // loop is done. The loop is "inside-out" compared to the EvictPage call -- we can't make calls to + // the kernel that would cause us to re-enter the swap context, because that would overwrite the + // stored thread `sepc`. The syscalls used here are all "simple calls" that don't require re-entry + // into the swapper context to handle. + let target_pages = pages_to_free; + let mut errs = 0; + let mut wired = 0; + loop { + if pages_to_free == 0 { + break; + } + if let Some(candidate) = alloc_heap.pop() { + if candidate.is_wired() + || !candidate.is_valid() + || candidate.raw_pid() == 1 + || candidate.raw_pid() == 2 + { + wired += 1; + } else { + // step 1: steal the page from the other process. Its data gets mapped into the + // swapper as `local_ptr`. This will also unmap the page from memory. + let local_ptr = match xous::rsyscall(xous::SysCall::SwapOp( + SwapAbi::StealPage as usize, + candidate.raw_pid() as usize, + candidate.vaddr(), + 0, + 0, + 0, + 0, + )) { + Ok(Result::Scalar5(page_ptr, _, _, _, _)) => page_ptr, + Ok(_) => panic!("Malformed return value"), + Err(_e) => { + errs += 1; + continue; // try another page + } + }; + + // step 2: write the page to swap. + write_to_swap_inner(ss, candidate.raw_pid(), candidate.vaddr(), local_ptr); + + // step 3: release the page (currently mapped into the swapper's memory space). Need + // to demonstrate to the memory system that we know what we are + // doing by also presenting the original PID that owned the page. + xous::rsyscall(xous::SysCall::SwapOp( + SwapAbi::ReleaseMemory as usize, + local_ptr, + candidate.raw_pid() as usize, + 0, + 0, + 0, + 0, + )).expect("Unexpected error: couldn't release a page that was mapped into the swapper's space"); + pages_to_free -= 1; + } + } else { + writeln!( + DebugUart {}, + "Ran out of swappable candidates before we could free the requested number of pages!" + ) + .ok(); + break; + } + } + // put the alloc heap back into the shared state + ss.hard_oom_alloc_heap = Some(alloc_heap); + writeln!( + DebugUart {}, + "Exiting HARD OOM swap free loop: freed {} pages; {} requests rejected, {} wired", + target_pages - pages_to_free - HARD_OOM_RESERVED_PAGES, + errs, + wired + ) + .ok(); + // Restore some reserved memory for the next hard OOM invocation. + let mut reserved = xous::map_memory( + None, + None, + PAGE_SIZE * HARD_OOM_RESERVED_PAGES, + MemoryFlags::R | MemoryFlags::W | MemoryFlags::RESERVE, + ) + .expect("could't reserve space for hard OOM handler"); + // *touch* the memory -- otherwise it might not actually be demand-paged + let reserved_slice: &mut [u32] = unsafe { reserved.as_slice_mut() }; // this is safe because `u32` is fully representable + reserved_slice.fill(0); + ss.hard_oom_reserved_page = Some(reserved); + } _ => { writeln!(DebugUart {}, "Unimplemented or unknown opcode: {}", opcode).ok(); } @@ -503,6 +705,7 @@ fn main() { let total_ram = sss.inner.as_ref().unwrap().sram_size; // Binary heap for storing the view of the memory allocations. sss.inner.as_mut().unwrap().alloc_heap = Some(BinaryHeap::with_capacity(total_ram / PAGE_SIZE)); + sss.inner.as_mut().unwrap().hard_oom_alloc_heap = Some(BinaryHeap::with_capacity(total_ram / PAGE_SIZE)); /// Poll interval for OOMer const OOM_POLL_INTERVAL_MS: u64 = 1000; @@ -528,6 +731,14 @@ fn main() { // a long time let oom_doom_running = Arc::new(AtomicBool::new(false)); + // Do a single invocation at boot with 0 pages to free, to ensure that the page maps are set up, + // and sufficient heap has been allocated for the swapper to run in case of a hard OOM. Failure to + // do this can lead to missing L1 PT entries for the RPT mapping back into user space if the first + // hard-OOM happens before the OOM-doom routine can run. All of swapper's memory is `wired`, so, + // once we've done a dry-run, this memory stays ours forever. + sss.inner.as_mut().unwrap().report_full_rpt = true; + try_invoke_oom_doom(&oom_doom_running, conn, 0); + // This thread is the active OOM monitor thread::spawn({ let conn = conn.clone(); @@ -545,6 +756,7 @@ fn main() { }); // This thread is for testing + #[cfg(feature = "swap-userspace-testing")] thread::spawn({ let conn = conn.clone(); move || { @@ -582,7 +794,8 @@ fn main() { // avoid log calls inside this loop, as we want to process all of these pages without // context switches loop { - if pages_to_free == 0 { + // Abort the loop if a hard OOM happened, because our RPT view is now stale + if pages_to_free == 0 || MAYBE_HARD_OOM_DURING_OOM_DOOM.load(Ordering::SeqCst) { break; } /* @@ -596,11 +809,13 @@ fn main() { || candidate.raw_pid() == 1 || candidate.raw_pid() == 2 { - // TODO: figure out why this count is not 0. I really think we should have - // filtered these inside the interrupt context - // when the heap was formed.... + // this should be 0, as it's pre-filtered by the kernel. wired += 1; } else { + // If hard OOM happened somewhere in this loop, this should return a harmless + // error if the candidate was already evicted; or it'll evict the candidate, + // and either way we'll exit the loop when we wrap around to the top, as our + // RPT views are now inconsistent. match xous::rsyscall(xous::SysCall::SwapOp( SwapAbi::Evict as usize, candidate.raw_pid() as usize, @@ -627,10 +842,29 @@ fn main() { errs, wired ); + assert!( + wired == 0 || ss.report_full_rpt, + "Wired pages were handed to us, but they should have been filtered by the userspace handler!" + ); + if ss.report_full_rpt { + ss.report_full_rpt = false; // reset this flag after the invocation has returned + // copy the alloc heap contents to the oom doom alloc heap so as to force its storage + // to alloc + for alloc in alloc_heap.iter() { + ss.hard_oom_alloc_heap.as_mut().unwrap().push(*alloc); + } + } } // restore the allocation for the binary heap ss.alloc_heap = Some(BinaryHeap::with_capacity(total_ram / PAGE_SIZE)); + // reset the hard OOM during OOM Doom lock + if MAYBE_HARD_OOM_DURING_OOM_DOOM.swap(false, Ordering::SeqCst) { + log::warn!( + "Hard OOM detected during OOM Doom running; run was aborted because RPT view is now inconsistent" + ); + } + // clear the running flag on exit oom_doom_running.store(false, Ordering::SeqCst); log::info!("Free mem after clearing: {}kiB", get_free_pages() * PAGE_SIZE / 1024); @@ -649,6 +883,7 @@ fn main() { panic!("Wrong message type for HandleOomDoom"); } } + #[cfg(feature = "swap-userspace-testing")] Some(Opcode::Test0) => { log::info!("Free mem before clearing: {}kiB", get_free_pages() * PAGE_SIZE / 1024); // Try to free some number of pages @@ -663,7 +898,46 @@ fn main() { fn get_free_pages() -> usize { match xous::rsyscall(xous::SysCall::SwapOp(SwapAbi::GetFreePages as usize, 0, 0, 0, 0, 0, 0)) { - Ok(Result::Scalar5(mem, _, _, _, _)) => mem, - _ => panic!("GetFreeMem sycall failed"), + Ok(Result::Scalar5(free_pages, _total_memory, _, _, _)) => free_pages, + _ => panic!("GetFreeMem syscall failed"), + } +} + +/// Core of write_to_swap: this is also used by HardOom. +fn write_to_swap_inner(ss: &mut SwapperSharedState, pid: u8, vaddr_in_pid: usize, vaddr_in_swap: usize) { + writeln!(DebugUart {}, "WTS PID{}, vaddr_pid {:x}, vaddr_swap {:x}", pid, vaddr_in_pid, vaddr_in_swap) + .ok(); + // this is safe because the page is aligned and initialized as it comes from the kernel + // remember that this page is overwritten with encrypted data + let buf: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(vaddr_in_swap as *mut u8, PAGE_SIZE) }; + + // search the swap page tables for the next free page + let mut next_free_page: Option = None; + for slot in 0..ss.sct.counts.len() { + let candidate = (ss.free_swap_search_origin + slot) % ss.sct.counts.len(); + if (ss.sct.counts[candidate] & loader::FLG_SWAP_USED) == 0 { + next_free_page = Some(candidate); + break; + } + } + if let Some(free_page_number) = next_free_page { + ss.free_swap_search_origin = free_page_number + 1; // start search at next page beyond the one about to be used + // increment the swap counter by one, rolling over if full. Note that we only have 31 + // bits; the MSB is the "swap used" status bit + let mut count = ss.sct.counts[free_page_number] & !loader::FLG_SWAP_USED; + count = (count + 1) & !loader::FLG_SWAP_USED; + ss.sct.counts[free_page_number] = count | loader::FLG_SWAP_USED; + + // add a PT mapping for the swap entry + map_swap(ss, free_page_number * PAGE_SIZE, vaddr_in_pid, pid); + + ss.hal.encrypt_swap_to(buf, count, free_page_number * PAGE_SIZE, vaddr_in_pid, pid); + } else { + writeln!(DebugUart {}, "OOM detected, dumping all swap allocs:").ok(); + for (i, &entry) in ss.sct.counts.iter().enumerate() { + writeln!(DebugUart {}, " {:04}:{:x}", i, entry).ok(); + } + // OOS path + panic!("Ran out of swap space, hard OOM!"); } }