From f1d5a015cc69be25da2f3ed090eecb4fadfbcc62 Mon Sep 17 00:00:00 2001 From: bunnie Date: Sat, 25 May 2024 02:10:55 +0800 Subject: [PATCH] remove oom-doom path handler in practice the hard-OOM only seems to happen, and oom-doom takes up precious memory that could be used for preventing OOMs. Just leave one path in, but make it a bit smarter. The smarts are still in userland, so the kernel is minimal dependencies. --- kernel/src/swap.rs | 179 +++-------------- kernel/src/syscall.rs | 17 +- services/xous-swapper/Cargo.toml | 2 +- services/xous-swapper/src/main.rs | 319 ++++++++---------------------- 4 files changed, 109 insertions(+), 408 deletions(-) diff --git a/kernel/src/swap.rs b/kernel/src/swap.rs index d4a47e061..fe95b91c2 100644 --- a/kernel/src/swap.rs +++ b/kernel/src/swap.rs @@ -16,6 +16,10 @@ use crate::arch::mem::PAGE_SIZE; use crate::mem::MemoryManager; use crate::services::SystemServices; +/// This might change depending on the target, link options, and version of LLVM. +/// The good news is it's static with every version, and constant after the first +/// invocation. So, if it works once, it should keep on working for that particular +/// combination of target/link options/compiler version. const BACKUP_STACK_SIZE_WORDS: usize = 1024 / core::mem::size_of::(); /// userspace swapper -> kernel ABI (see kernel/src/syscall.rs) @@ -27,9 +31,9 @@ const BACKUP_STACK_SIZE_WORDS: usize = 1024 / core::mem::size_of::(); #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum SwapAbi { Invalid = 0, - Evict = 1, + ClearMemoryNow = 1, GetFreePages = 2, - FetchAllocs = 3, + // FetchAllocs = 3, HardOom = 4, StealPage = 5, ReleaseMemory = 6, @@ -39,9 +43,9 @@ impl SwapAbi { pub fn from(val: usize) -> SwapAbi { use SwapAbi::*; match val { - 1 => Evict, + 1 => ClearMemoryNow, 2 => GetFreePages, - 3 => FetchAllocs, + // 3 => FetchAllocs, 4 => HardOom, 5 => StealPage, 6 => ReleaseMemory, @@ -55,17 +59,9 @@ impl SwapAbi { /// necessary for proper return from these calls. #[derive(Debug, Copy, Clone)] pub enum BlockingSwapOp { - /// TID, PID of source, vaddr of source, vaddr in swap space (block must already be - /// mapped into swap space). Returns to Swapper, as this can only originate from the Swapper. - WriteToSwap(TID, PID, usize, usize), /// PID of the target block, current paddr of block, original vaddr in the space of target block PID, /// physical address of block. Returns to PID of the target block. ReadFromSwap(PID, TID, usize, usize, usize), - /// Argument is the original thread ID of swapper. Returns to the swapper, as it can only - /// originate from the Swapper. This is used by the swapper when OomDoom is imminent; basically, - /// the userspace version of HardOom. This call allows progress in other processes while the - /// swapper does its thing. - FetchAllocs(TID), /// Immediate OOM. Drop everything and try to recover; from here until exit, everything runs in /// an un-interruptable context, no progress allowed. This currently can only originate from one /// location, if we need multi-location origin then we have to also track the re-entry point in the @@ -470,10 +466,20 @@ impl Swap { // assemble this upstream of the stack save, because this affects stack let call_precompute = xous_kernel::SysCall::SwapOp(SwapAbi::HardOom as usize, 0, 0, 0, 0, 0, 0); - // at this point we are in user pid/tid, but supervisor mode. The current thread backing is - // about to be smashed by the syscall invocation. we want to return to this backing. - let mut current_sp: usize; + // at this point we are in user pid/tid, but supervisor mode. The current exception handler stack is + // about to be smashed by the syscall invocation. Save the stack with a specially crafted routine + // that has the following properties: + // 1. It does not modify the stack + // 2. It copies all of the current stack to a backup location + // 3. It can restore stack after the call is done, also without modifying stack + // + // Thus the routine is coded to use just temporary registers (in this case, the argument registers + // that we know we won't be using in the upcoming re-entrant syscall), and manually checked for + // no stack manipulations by disassembling the code. It may be necessary to disassemble the code + // again with future Rust/llvm updates because the behavior of no-stack mods is not guaranteed. + // Print the stack pointer for checking purposes + let mut current_sp: usize; unsafe { core::arch::asm!( "mv {current_sp}, sp", @@ -481,6 +487,7 @@ impl Swap { ) } println!(" --> SP extent d'{} bytes", EXCEPTION_STACK_TOP - current_sp); + // make a backup copy of the stack let backup_stack_ptr: usize = self.oom_stack_backing.as_ptr() as usize; let working_stack_end: usize = EXCEPTION_STACK_TOP; let working_stack_ptr: usize = @@ -504,13 +511,18 @@ impl Swap { current_sp = out(reg) current_sp, ) } + // this assert will hopefully save us during CI testing if parameters changed and we have to fix them + // the assert itself would modify stack, but, we're on the path to a panic so that's OK assert!( EXCEPTION_STACK_TOP - current_sp <= self.oom_stack_backing.len() * core::mem::size_of::(), "Backing stack not large enough to handle current kernel stack utilization." ); - // invoke this as a nested syscall so it returns here + // invoke this as a nested syscall so it returns here. No stack mods are allowed between the + // backup and this call xous_kernel::rsyscall(call_precompute).ok(); + // restore the stack immediately after the syscall return (no debug info etc. allowed as it may + // rely on stack). let restore_stack_ptr: usize = self.oom_stack_backing.as_ptr() as usize; let working_restore_end: usize = EXCEPTION_STACK_TOP; let working_restore_ptr: usize = @@ -544,7 +556,7 @@ impl Swap { ) }); - // then restore the thread state on return from the syscall + // then restore the smashed thread backing state on return from the syscall crate::arch::process::Process::with_current_mut(|p| { println!("Returned from hard OOM handler, current pid {}/tid {}", p.pid().get(), p.current_tid()); *p.current_thread_mut() = @@ -554,81 +566,6 @@ impl Swap { true } - /// This call diverges into the userspace swapper. - /// Divergent calls must turn of IRQs before memory spaces are changed. - pub fn fetch_allocs(&mut self) -> ! { - crate::arch::irq::disable_all_irqs(); - let tid = crate::arch::process::Process::with_current(|p| p.current_tid()); - - SystemServices::with(|system_services| { - let swapper_pid = PID::new(xous_kernel::SWAPPER_PID).unwrap(); - // swap to the swapper space - let swapper_map = system_services.get_process(swapper_pid).unwrap().mapping; - swapper_map.activate().unwrap(); - - // map the RPT into userspace - // note that this technically makes the RPT shared with kernel and userspace, but, - // the userspace version is read-only, and the next section runs in an interrupt context, - // so I think it's safe for it to be shared for that duration. - MemoryManager::with_mut(|mm| { - for page in 0..self.mem_alloc_tracker_pages { - crate::arch::mem::map_page_inner( - mm, - swapper_pid, - self.mem_alloc_tracker_paddr + page * PAGE_SIZE, - SWAP_RPT_VADDR + page * PAGE_SIZE, - xous_kernel::MemoryFlags::R, - true, - ) - .ok(); - unsafe { crate::arch::mem::flush_mmu() }; - } - }); - }); - - #[cfg(feature = "debug-swap")] - println!("fetch_allocs - userspace activate"); - // this is safe because we entered the swapper memory context above - unsafe { - self.blocking_activate_swapper(BlockingSwapOp::FetchAllocs(tid)); - } - } - - /// This call normally diverges into the userspace swapper. It will return a SysCallResult - /// if the requested pages fail sanity checks. - /// Divergent calls must turn of IRQs before memory spaces are changed. - pub fn evict_page(&mut self, target_pid: usize, vaddr: usize) -> SysCallResult { - crate::arch::irq::disable_all_irqs(); - // This call can fail for legitimate reasons: for example, the address given was already swapped, or - // invalid. - let evicted_ptr = - match crate::arch::mem::evict_page_inner(PID::new(target_pid as u8).expect("Invalid PID"), vaddr) - { - Ok(ptr) => ptr, - Err(e) => { - #[cfg(feature = "debug-swap")] - println!("evict_page rejecting request for pid{}/{:x}: {:?}", target_pid, vaddr, e); - // evict_page_inner guarantees we are in the swapper PID even on an error edge case - crate::arch::irq::enable_all_irqs(); - return Err(e); - } - }; - - // remember the TID of the swapper we're returning to - let tid = crate::arch::process::Process::with_current(|p| p.current_tid()); - #[cfg(feature = "debug-swap")] - println!("evict_page from pid{}/{:x} - swapper activate", target_pid, vaddr); - // this is safe because evict_page() leaves us in the swapper memory context - unsafe { - self.blocking_activate_swapper(BlockingSwapOp::WriteToSwap( - tid, - PID::new(target_pid as u8).expect("Invalid PID"), - vaddr, - evicted_ptr, - )); - } - } - /// The address space on entry to `retrieve_page` is `target_pid`; it must ensure /// that the address space is still `target_pid` on return. /// @@ -693,14 +630,6 @@ impl Swap { unsafe fn blocking_activate_swapper(&mut self, op: BlockingSwapOp) -> ! { // setup the argument block match op { - // Note to self: get opcode number from `KernelOp` structure in services/xous-swapper/main.rs - BlockingSwapOp::WriteToSwap(_tid, pid, vaddr_in_pid, vaddr_in_swap) => { - self.swapper_args[0] = self.swapper_state; - self.swapper_args[1] = 0; // WriteToSwap - self.swapper_args[2] = pid.get() as usize; - self.swapper_args[3] = vaddr_in_pid; - self.swapper_args[4] = vaddr_in_swap; - } BlockingSwapOp::ReadFromSwap(pid, _tid, vaddr_in_pid, vaddr_in_swap, _paddr) => { self.swapper_args[0] = self.swapper_state; self.swapper_args[1] = 1; // ReadFromSwap @@ -708,11 +637,6 @@ impl Swap { self.swapper_args[3] = vaddr_in_pid; self.swapper_args[4] = vaddr_in_swap; } - BlockingSwapOp::FetchAllocs(_tid) => { - self.swapper_args[0] = self.swapper_state; - self.swapper_args[1] = 2; // ExecFetchAllocs - self.swapper_args[2] = self.mem_alloc_tracker_pages; - } BlockingSwapOp::HardOomSyscall(_tid, _pid) => { self.swapper_args[0] = self.swapper_state; self.swapper_args[1] = 3; // HardOom @@ -852,51 +776,6 @@ impl Swap { } }) } - // Called only from PID2. Resume to the original TID within PID2, as a syscall return. - Some(BlockingSwapOp::WriteToSwap(tid, pid, _addr, swapper_virt_page)) => { - // Update the RPT: mark the physical memory as free. The virtual address is - // in the swapper's context at this point, so free it there (it's already been - // remapped as swapped in the target's context). Note that the swapped page now contains - // encrypted data, as the encryption happens in-place. - MemoryManager::with_mut(|mm| { - let paddr = crate::arch::mem::virt_to_phys(swapper_virt_page).unwrap() as usize; - #[cfg(feature = "debug-swap")] - println!("WTS releasing paddr {:x}", paddr); - // this call unmaps the virtual page from the page table - crate::arch::mem::unmap_page_inner(mm, swapper_virt_page).expect("couldn't unmap page"); - // This call releases the physical page from the RPT - the pid has to match that of the - // original owner. This is the "pointy end" of the stick; after this call, the memory is - // now back into the free pool. - mm.release_page_swap(paddr as *mut usize, pid) - .expect("couldn't free page that was swapped out"); - // clear the caches - flush_dcache(); - }); - - // Switch back to the Swapper's userland thread - SystemServices::with_mut(|ss| { - ss.finish_callback_and_resume(PID::new(SWAPPER_PID).unwrap(), tid) - .expect("Unable to resume to swapper"); - }); - // return as a SysCall - Ok(xous_kernel::Result::Scalar5(0, 0, 0, 0, 0)) - } - // Called only from PID2. Resume to the original TID within PID2, as a syscall return. - Some(BlockingSwapOp::FetchAllocs(tid)) => { - // unmap RPT from userspace - MemoryManager::with_mut(|mm| { - for page in 0..self.mem_alloc_tracker_pages { - crate::arch::mem::unmap_page_inner(mm, SWAP_RPT_VADDR + page * PAGE_SIZE).ok(); - } - }); - // Switch back to the Swapper's userland thread - SystemServices::with_mut(|ss| { - ss.finish_callback_and_resume(PID::new(SWAPPER_PID).unwrap(), tid) - .expect("Unable to resume to swapper"); - }); - // return as a SysCall - Ok(xous_kernel::Result::Scalar5(0, 0, 0, 0, 0)) - } Some(BlockingSwapOp::HardOomSyscall(tid, pid)) => { // We enter this from the swapper's memory space assert!( diff --git a/kernel/src/syscall.rs b/kernel/src/syscall.rs index 195a357e9..5d30a6706 100644 --- a/kernel/src/syscall.rs +++ b/kernel/src/syscall.rs @@ -1017,18 +1017,11 @@ pub fn handle_inner(pid: PID, tid: TID, in_irq: bool, call: SysCall) -> SysCallR #[cfg(feature = "debug-swap")] println!("Swap ABI via syscall: {:?}", SwapAbi::from(op)); match SwapAbi::from(op) { - SwapAbi::Evict => { + SwapAbi::ClearMemoryNow => { if pid.get() != xous_kernel::SWAPPER_PID { return Err(xous_kernel::Error::AccessDenied); - } - let pid = a1; - if pid != 0 && pid != 1 && pid != 2 { - Swap::with_mut(|swap| swap.evict_page(pid, a2)) } else { - // don't allow kernel or swapper to be swapped out - #[cfg(feature = "debug-swap")] - println!("evict_page rejecting request for pid{}/{:x}: InvalidPID", pid, a2); - Err(xous_kernel::Error::InvalidPID) + Swap::with_mut(|swap| swap.hard_oom_syscall()) } } SwapAbi::GetFreePages => { @@ -1037,12 +1030,6 @@ pub fn handle_inner(pid: PID, tid: TID, in_irq: bool, call: SysCall) -> SysCallR } Swap::with(|swap| swap.get_free_mem()) } - SwapAbi::FetchAllocs => { - if pid.get() != xous_kernel::SWAPPER_PID { - return Err(xous_kernel::Error::AccessDenied); - } - Swap::with_mut(|swap| swap.fetch_allocs()) - } // SwapAbi::SetOomThresh => Swap::with_mut(|swap| swap.set_oom_thresh(a1, a2)), SwapAbi::StealPage => { if pid.get() != xous_kernel::SWAPPER_PID { diff --git a/services/xous-swapper/Cargo.toml b/services/xous-swapper/Cargo.toml index ea015815b..5bc36e666 100644 --- a/services/xous-swapper/Cargo.toml +++ b/services/xous-swapper/Cargo.toml @@ -38,4 +38,4 @@ swap-userspace-testing = [] debug-print = [] debug-verbose = [] -default = ["debug-print"] +default = ["debug-print", "swap-userspace-testing"] diff --git a/services/xous-swapper/src/main.rs b/services/xous-swapper/src/main.rs index 8544f94ac..eb18eb3d6 100644 --- a/services/xous-swapper/src/main.rs +++ b/services/xous-swapper/src/main.rs @@ -125,7 +125,6 @@ mod platform; use core::fmt::Write; use std::collections::{BinaryHeap, VecDeque}; use std::fmt::Debug; -use std::sync::{atomic::AtomicBool, atomic::Ordering, Arc}; use std::thread; use std::thread::sleep; use std::time::Duration; @@ -136,10 +135,6 @@ use num_traits::*; use platform::{SwapHal, PAGE_SIZE}; use xous::{MemoryFlags, MemoryRange, Message, Result, CID, PID, SID}; -/// Threshold for OomDoom callback -const OOM_THRESH_PAGES: usize = 16; -/// Target of free pages we want to get to after OomDoom call -const FREE_PAGE_TARGET: usize = 32; /// Target of pages to free in case of a Hard OOM const HARD_OOM_PAGE_TARGET: usize = 48; /// Virtual address prefixes to de-prioritize in the OomDoom sweep @@ -156,9 +151,9 @@ const KEEP_VADDR_PREFIXES: [u8; 2] = [0u8, 4u8]; #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum SwapAbi { Invalid = 0, - Evict = 1, + ClearMemoryNow = 1, GetFreePages = 2, - FetchAllocs = 3, + // FetchAllocs = 3, // HardOom = 4, // meant to be initiated within the kernel to itself StealPage = 5, ReleaseMemory = 6, @@ -168,9 +163,9 @@ impl SwapAbi { pub fn from(val: usize) -> SwapAbi { use SwapAbi::*; match val { - 1 => Evict, + 1 => ClearMemoryNow, 2 => GetFreePages, - 3 => FetchAllocs, + // 3 => FetchAllocs, // 4 => HardOom, 5 => StealPage, 6 => ReleaseMemory, @@ -190,8 +185,6 @@ pub enum KernelOp { WriteToSwap = 0, /// Find the requested page, decrypt it, and return it ReadFromSwap = 1, - /// Kernel message advising us that a page of RAM was allocated - ExecFetchAllocs = 2, /// Hard OOM invocation - stop everything and free memory! HardOom = 3, } @@ -200,10 +193,6 @@ pub enum KernelOp { #[derive(Debug, num_derive::FromPrimitive, num_derive::ToPrimitive)] #[repr(usize)] pub enum Opcode { - /// Trigger back to userspace to indicate that alloc fetching is done. - FetchAllocsDone, - /// Trigger the OOM routine - HandleOomDoom, /// Test messages #[cfg(feature = "swap-userspace-testing")] Test0, @@ -226,9 +215,6 @@ pub struct SwapCountTracker { pub counts: &'static mut [u32], } -/// Track if the hard OOM handler ram during the Oom Doom userspace routine -static MAYBE_HARD_OOM_DURING_OOM_DOOM: AtomicBool = AtomicBool::new(false); - /// Number of pages to reserve for hard OOM handling. In case of a hard OOM, there are 0 pages /// available, which makes it impossible for the hard OOM handler to do things like allocate an L1 /// page table entry to track memory being swapped out. This places a hold on some memory that's @@ -262,15 +248,13 @@ pub struct SwapperSharedState { /// starting from the free swap search origin. The unit of this variable is in pages, so it /// can be used to directly index the `sct` `SwapCountTracker`. pub free_swap_search_origin: usize, - /// alloc heap for the OomDoom handler - pub alloc_heap: Option>, - /// alloc heap for the Hard OOM handler. Must be separate because it's possible for both - /// to run at the same time. pub hard_oom_alloc_heap: Option>, /// Reserve some memory to be freed by the hard OOM manager. These pages are needed to do things /// like create L1 page table entries for the swapper to track evicted pages. pub hard_oom_reserved_page: Option, pub report_full_rpt: bool, + /// number of pages to free in the OOM routine + pub pages_to_free: usize, } impl SwapperSharedState { pub fn pt_walk(&self, pid: u8, va: usize, mark_free: bool) -> Option { @@ -303,7 +287,7 @@ impl SwapperSharedState { } struct SharedStateStorage { pub inner: Option, - pub conn: CID, + pub _conn: CID, } impl SharedStateStorage { pub fn init(&mut self, sid: SID) { @@ -452,10 +436,10 @@ fn swap_handler( sram_start: swap_spec.sram_start as usize, sram_size: swap_spec.sram_size as usize, free_swap_search_origin: 0, - alloc_heap: None, hard_oom_alloc_heap: None, report_full_rpt: true, hard_oom_reserved_page: Some(reserved), + pages_to_free: HARD_OOM_PAGE_TARGET + HARD_OOM_RESERVED_PAGES, }); } let ss = sss.inner.as_mut().expect("Shared state should be initialized"); @@ -532,46 +516,12 @@ fn swap_handler( } // at this point, the `buf` has our desired data, we're done, modulo updating the count. } - Some(KernelOp::ExecFetchAllocs) => { - if let Some(alloc_heap) = &mut ss.alloc_heap { - alloc_heap.clear(); - assert!(alloc_heap.len() == 0); - let rpt = unsafe { - core::slice::from_raw_parts(SWAP_RPT_VADDR as *const SwapAlloc, ss.sram_size / PAGE_SIZE) - }; - for (_i, &entry) in rpt.iter().enumerate() { - #[cfg(feature = "debug-verbose")] - if entry.raw_vpn() != 0 { - writeln!(DebugUart {}, "{:x}: {:x} [{}]", _i, entry.raw_vpn(), entry.timestamp()) - .ok(); - } - // filter out invalid, wired, or kernel/swapper candidates. - // report everything if requested (this is used to wire our heap memory prior to an OOM) - if (!entry.is_wired() && entry.is_valid() && entry.raw_pid() != 1 && entry.raw_pid() != 2) - || ss.report_full_rpt - { - // writeln!(DebugUart {}, "Pushing {:x?}", entry).ok(); - alloc_heap.push(entry); - } - } - #[cfg(feature = "debug-verbose")] - writeln!(DebugUart {}, "Created heap with {} entries", alloc_heap.len()).ok(); - xous::try_send_message( - sss.conn, - Message::new_scalar(Opcode::FetchAllocsDone.to_usize().unwrap(), 0, 0, 0, 0), - ) - .ok(); - } - } // HardOom handling will evict any and all pages that it can -- it does no filtering. Some(KernelOp::HardOom) => { // parse the arguments (none, currently) // be sure to allocate some extra space for the handler itself to run the next time! - let mut pages_to_free = HARD_OOM_PAGE_TARGET + HARD_OOM_RESERVED_PAGES; - - // set the flag that hard oom ran during an Oom Doom handler pass - MAYBE_HARD_OOM_DURING_OOM_DOOM.store(true, Ordering::SeqCst); + let mut pages_to_free = ss.pages_to_free; // free memory for the hard-OOM handler to run if let Some(reserved_mem) = ss.hard_oom_reserved_page.take() { @@ -590,8 +540,11 @@ fn swap_handler( core::slice::from_raw_parts(SWAP_RPT_VADDR as *const SwapAlloc, ss.sram_size / PAGE_SIZE) }; for (_i, &entry) in rpt.iter().enumerate() { - // filter out invalid, wired, or kernel/swapper candidates. - if !entry.is_wired() && entry.is_valid() && entry.raw_pid() != 1 && entry.raw_pid() != 2 { + // filter out invalid, wired, or kernel/swapper candidates + if (!entry.is_wired() && entry.is_valid() && entry.raw_pid() != 1 && entry.raw_pid() != 2) + // report_full_rpt is used to force the heap to reserve all the data we might need in a future oom + || ss.report_full_rpt + { // writeln!(DebugUart {}, "Pushing {:x?}", entry).ok(); alloc_heap.push(entry); } @@ -604,6 +557,11 @@ fn swap_handler( let target_pages = pages_to_free; let mut errs = 0; let mut wired = 0; + + // A holding variable for items that we're de-prioritizing from removal. We only try these + // if we've exhausted all the other options. + let mut deprioritized = VecDeque::new(); + loop { if pages_to_free == 0 { break; @@ -616,6 +574,12 @@ fn swap_handler( { wired += 1; } else { + if KEEP_VADDR_PREFIXES.iter().find(|&&x| x == candidate.vaddr_prefix()).is_some() { + log::trace!("De-prioritizing {:x?}", candidate); + deprioritized.push_back(candidate); + continue; + } + // step 1: steal the page from the other process. Its data gets mapped into the // swapper as `local_ptr`. This will also unmap the page from memory. let local_ptr = match xous::rsyscall(xous::SysCall::SwapOp( @@ -652,6 +616,48 @@ fn swap_handler( )).expect("Unexpected error: couldn't release a page that was mapped into the swapper's space"); pages_to_free -= 1; } + } else if let Some(candidate) = deprioritized.pop_front() { + // copy of code above. TODO: figure out some way to reduce this duplication, + // without impacting error tracking, etc. + + // step 1: steal the page from the other process. Its data gets mapped into the + // swapper as `local_ptr`. This will also unmap the page from memory. + let local_ptr = match xous::rsyscall(xous::SysCall::SwapOp( + SwapAbi::StealPage as usize, + candidate.raw_pid() as usize, + candidate.vaddr(), + 0, + 0, + 0, + 0, + )) { + Ok(Result::Scalar5(page_ptr, _, _, _, _)) => page_ptr, + Ok(_) => panic!("Malformed return value"), + Err(_e) => { + errs += 1; + continue; // try another page + } + }; + + // step 2: write the page to swap. + write_to_swap_inner(ss, candidate.raw_pid(), candidate.vaddr(), local_ptr); + + // step 3: release the page (currently mapped into the swapper's memory space). Need + // to demonstrate to the memory system that we know what we are + // doing by also presenting the original PID that owned the page. + xous::rsyscall(xous::SysCall::SwapOp( + SwapAbi::ReleaseMemory as usize, + local_ptr, + candidate.raw_pid() as usize, + 0, + 0, + 0, + 0, + )) + .expect( + "Unexpected error: couldn't release a page that was mapped into the swapper's space", + ); + pages_to_free -= 1; } else { writeln!( DebugUart {}, @@ -683,6 +689,9 @@ fn swap_handler( let reserved_slice: &mut [u32] = unsafe { reserved.as_slice_mut() }; // this is safe because `u32` is fully representable reserved_slice.fill(0); ss.hard_oom_reserved_page = Some(reserved); + if ss.report_full_rpt { + ss.report_full_rpt = false; + } } _ => { writeln!(DebugUart {}, "Unimplemented or unknown opcode: {}", opcode).ok(); @@ -690,24 +699,10 @@ fn swap_handler( } } -/// Invokes OOM Doom, but only if it hasn't already been invoked. -fn try_invoke_oom_doom(oom_doom_running: &Arc, conn: CID, pages_to_free: usize) -> bool { - if !oom_doom_running.swap(true, Ordering::SeqCst) { - xous::try_send_message( - conn, - Message::new_scalar(Opcode::HandleOomDoom.to_usize().unwrap(), pages_to_free, 0, 0, 0), - ) - .ok(); - true - } else { - false - } -} - fn main() { let sid = xous::create_server().unwrap(); let conn = xous::connect(sid).unwrap(); - let mut sss = Box::new(SharedStateStorage { conn, inner: None }); + let mut sss = Box::new(SharedStateStorage { _conn: conn, inner: None }); sss.init(sid); // init the log, but this is mostly unused. @@ -724,40 +719,20 @@ fn main() { let total_ram = sss.inner.as_ref().unwrap().sram_size; // Binary heap for storing the view of the memory allocations. - sss.inner.as_mut().unwrap().alloc_heap = Some(BinaryHeap::with_capacity(total_ram / PAGE_SIZE)); sss.inner.as_mut().unwrap().hard_oom_alloc_heap = Some(BinaryHeap::with_capacity(total_ram / PAGE_SIZE)); - /// Poll interval for OOMer - const OOM_POLL_INTERVAL_MS: u64 = 1000; - // track the current amount of pages to be freed - let mut pages_to_free = FREE_PAGE_TARGET; - // track if the oom_doom process is running - so we don't get multiple spawns if the oom runner is taking - // a long time - let oom_doom_running = Arc::new(AtomicBool::new(false)); - // Do a single invocation at boot with 0 pages to free, to ensure that the page maps are set up, // and sufficient heap has been allocated for the swapper to run in case of a hard OOM. Failure to // do this can lead to missing L1 PT entries for the RPT mapping back into user space if the first // hard-OOM happens before the OOM-doom routine can run. All of swapper's memory is `wired`, so, // once we've done a dry-run, this memory stays ours forever. sss.inner.as_mut().unwrap().report_full_rpt = true; - try_invoke_oom_doom(&oom_doom_running, conn, 0); - - // This thread is the active OOM monitor - thread::spawn({ - let conn = conn.clone(); - let oom_doom_running = oom_doom_running.clone(); - move || { - loop { - let free_mem_pages = get_free_pages(); - if free_mem_pages < OOM_THRESH_PAGES { - let pages_to_free = FREE_PAGE_TARGET - free_mem_pages; - try_invoke_oom_doom(&oom_doom_running, conn, pages_to_free); - } - sleep(Duration::from_millis(OOM_POLL_INTERVAL_MS)); - } - } - }); + sss.inner.as_mut().unwrap().pages_to_free = 2; + xous::rsyscall(xous::SysCall::SwapOp(SwapAbi::ClearMemoryNow as usize, 0, 0, 0, 0, 0, 0)) + .expect("ClearMemoryNow syscall failed"); + // restore the normal parameters + sss.inner.as_mut().unwrap().report_full_rpt = false; + sss.inner.as_mut().unwrap().pages_to_free = HARD_OOM_PAGE_TARGET + HARD_OOM_RESERVED_PAGES; // This thread is for testing #[cfg(feature = "swap-userspace-testing")] @@ -779,149 +754,9 @@ fn main() { let op: Option = FromPrimitive::from_usize(msg.body.id()); log::debug!("Swapper got {:x?}", op); match op { - Some(Opcode::FetchAllocsDone) => { - let ss = sss.inner.as_mut().unwrap(); - if let Some(mut alloc_heap) = ss.alloc_heap.take() { - log::info!("Attempting to free {} pages, {} candidates", pages_to_free, alloc_heap.len()); - let target_pages = pages_to_free; - let mut errs = 0; - let mut wired = 0; - /* // for debugging ordering of suggested pages - use std::collections::VecDeque; - let mut preview_vec = VecDeque::new(); - for _ in 0..16 { - preview_vec.push_back(alloc_heap.pop().unwrap()); - } - for (i, entry) in preview_vec.iter().enumerate() { - println!(" {:x}: {:x} [{}]", i, entry.raw_vpn(), entry.timestamp()); - } */ - - // A holding variable for items that we're de-prioritizing from removal. We only try these - // if we've exhausted all the other options. - let mut deprioritized = VecDeque::new(); - - // avoid log calls inside this loop, as we want to process all of these pages without - // context switches - loop { - // Abort the loop if a hard OOM happened, because our RPT view is now stale - if pages_to_free == 0 || MAYBE_HARD_OOM_DURING_OOM_DOOM.load(Ordering::SeqCst) { - break; - } - /* - if let Some(candidate) = - Some(preview_vec.pop_front().unwrap_or(alloc_heap.pop().unwrap())) - { - */ - if let Some(candidate) = alloc_heap.pop() { - if candidate.is_wired() - || !candidate.is_valid() - || candidate.raw_pid() == 1 - || candidate.raw_pid() == 2 - { - // this should be 0, as it's pre-filtered by the kernel. - wired += 1; - } else { - if KEEP_VADDR_PREFIXES - .iter() - .find(|&&x| x == candidate.vaddr_prefix()) - .is_some() - { - log::trace!("De-prioritizing {:x?}", candidate); - deprioritized.push_back(candidate); - continue; - } - - // If hard OOM happened somewhere in this loop, this should return a harmless - // error if the candidate was already evicted; or it'll evict the candidate, - // and either way we'll exit the loop when we wrap around to the top, as our - // RPT views are now inconsistent. - match xous::rsyscall(xous::SysCall::SwapOp( - SwapAbi::Evict as usize, - candidate.raw_pid() as usize, - candidate.vaddr(), - 0, - 0, - 0, - 0, - )) { - Ok(_) => pages_to_free -= 1, - Err(_e) => errs += 1, - } - } - } else if let Some(candidate) = deprioritized.pop_front() { - log::trace!("Falling back to: {:x?}", candidate); - match xous::rsyscall(xous::SysCall::SwapOp( - SwapAbi::Evict as usize, - candidate.raw_pid() as usize, - candidate.vaddr(), - 0, - 0, - 0, - 0, - )) { - Ok(_) => pages_to_free -= 1, - Err(_e) => errs += 1, - } - pages_to_free -= 1; - } else { - log::warn!( - "Ran out of swappable candidates before we could free the requested number of pages!" - ); - break; - } - } - log::info!( - "Exiting swap free loop: freed {} pages; {} requests rejected, {} wired", - target_pages - pages_to_free, - errs, - wired - ); - assert!( - wired == 0 || ss.report_full_rpt, - "Wired pages were handed to us, but they should have been filtered by the userspace handler!" - ); - if ss.report_full_rpt { - ss.report_full_rpt = false; // reset this flag after the invocation has returned - // copy the alloc heap contents to the oom doom alloc heap so as to force its storage - // to alloc - for alloc in alloc_heap.iter() { - ss.hard_oom_alloc_heap.as_mut().unwrap().push(*alloc); - } - } - } - // restore the allocation for the binary heap - ss.alloc_heap = Some(BinaryHeap::with_capacity(total_ram / PAGE_SIZE)); - - // reset the hard OOM during OOM Doom lock - if MAYBE_HARD_OOM_DURING_OOM_DOOM.swap(false, Ordering::SeqCst) { - log::warn!( - "Hard OOM detected during OOM Doom running; run was aborted because RPT view is now inconsistent" - ); - } - - // clear the running flag on exit - oom_doom_running.store(false, Ordering::SeqCst); - log::info!("Free mem after clearing: {}kiB", get_free_pages() * PAGE_SIZE / 1024); - } - Some(Opcode::HandleOomDoom) => { - if let Some(scalar) = msg.body.scalar_message() { - // This should be a redundant store -- the caller should have set this to prevent multiple - // HandleOomDoom messages from being shoved into the server, blocking - // our ability to make progress. But we re-assert it just in case. - oom_doom_running.store(true, Ordering::SeqCst); - pages_to_free = scalar.arg1; - // trigger alloc fetch so we know what to free up - xous::rsyscall(xous::SysCall::SwapOp(SwapAbi::FetchAllocs as usize, 0, 0, 0, 0, 0, 0)) - .ok(); - } else { - panic!("Wrong message type for HandleOomDoom"); - } - } #[cfg(feature = "swap-userspace-testing")] Some(Opcode::Test0) => { log::info!("Free mem before clearing: {}kiB", get_free_pages() * PAGE_SIZE / 1024); - // Try to free some number of pages - try_invoke_oom_doom(&oom_doom_running, conn, 64); } _ => { log::info!("Unknown opcode {:?}", op);