From bfb25af370e69c729cdb8eed67412f79a6505f9f Mon Sep 17 00:00:00 2001 From: bunnie Date: Sat, 25 May 2024 00:06:38 +0800 Subject: [PATCH] make hard OOM a re-entrant syscall Re-entrant syscall is tricky because we end up smashing the return thread context and stack of the current syscall. We stash the thread context and stack in order to make this happen, but we have to make some assumption about the potential stack size. There's a assert to save us if this is wrong, but generally the kernel is extremely well behaved on stack consumption so we shouldn't be bumping into it. This version of the OOM handler allows us to bury the OOM handler "anywhere", which means we can handle the hard OOM in multiple allocation methods without having to track the entry and exit point of every single possible one. This means we can chop a lot of fragile code out of the kernel. We'll do that after this commit, and some testing, to make sure this is the right path, but it feels "much nicer". We'll probably also cut the oom-doomer, because I think in practice it won't be effective and it consumes a non-trivial amount of RAM, which would otherwise make OOMs more infrequent -- probably an amount that would be greater than the benefit that any soft-OOM handler can get us. --- kernel/src/swap.rs | 248 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 241 insertions(+), 7 deletions(-) diff --git a/kernel/src/swap.rs b/kernel/src/swap.rs index f483d8ad1..5fa34f1e5 100644 --- a/kernel/src/swap.rs +++ b/kernel/src/swap.rs @@ -11,6 +11,7 @@ use xous_kernel::{SysCallResult, PID, SID, TID}; use crate::arch::current_pid; use crate::arch::mem::flush_mmu; use crate::arch::mem::MMUFlags; +use crate::arch::mem::EXCEPTION_STACK_TOP; use crate::arch::mem::PAGE_SIZE; use crate::mem::MemoryManager; use crate::services::SystemServices; @@ -27,7 +28,7 @@ pub enum SwapAbi { Evict = 1, GetFreePages = 2, FetchAllocs = 3, - // SetOomThresh = 4, + HardOom = 4, StealPage = 5, ReleaseMemory = 6, } @@ -39,7 +40,7 @@ impl SwapAbi { 1 => Evict, 2 => GetFreePages, 3 => FetchAllocs, - // 4 => SetOomThresh, + 4 => HardOom, 5 => StealPage, 6 => ReleaseMemory, _ => Invalid, @@ -69,6 +70,7 @@ pub enum BlockingSwapOp { /// kernel memory cycle. Arguments are the TID/PID of the context running that triggered the HardOom, /// as well as the virtual address that triggered the hard OOM. HardOom(TID, PID, usize), + HardOomSyscall(TID, PID), } /// This structure is a copy of what's defined in the loader's swap module. The reason @@ -185,6 +187,8 @@ static mut SWAP: Swap = Swap { mem_alloc_tracker_pages: 0, oom_irq_backing: None, unmap_rpt_after_hard_oom: false, + oom_thread_backing: None, + oom_stack_backing: [0usize; 512], }; pub struct Swap { @@ -218,6 +222,9 @@ pub struct Swap { /// uncommon, because the soft-OOM handler is interruptable and can only keep up with gradual memory /// demand. unmap_rpt_after_hard_oom: bool, + /// backing for the thread state that is smashed by the OOM handler + oom_thread_backing: Option, + oom_stack_backing: [usize; 512], } impl Swap { pub fn with_mut(f: F) -> R @@ -463,6 +470,188 @@ impl Swap { } } + pub fn hard_oom_syscall(&mut self) -> SysCallResult { + // disable all IRQs; no context swapping is allowed + self.oom_irq_backing = Some(sim_read()); + #[cfg(feature = "debug-swap")] + println!("Hard OOM syscall stored SIM: {:x?}", self.oom_irq_backing); + sim_write(0x0); + + let original_pid = crate::arch::process::current_pid(); + let original_tid = crate::arch::process::current_tid(); + + // move into the swapper's memory space & map the RPT into the swapper's space so it can + // make decisions about what to move out. + SystemServices::with(|system_services| { + let swapper_pid = PID::new(xous_kernel::SWAPPER_PID).unwrap(); + // swap to the swapper space + let swapper_map = system_services.get_process(swapper_pid).unwrap().mapping; + swapper_map.activate().unwrap(); + + // map the RPT into userspace + // note that this technically makes the RPT shared with kernel and userspace, but, + // the userspace version is read-only, and the next section runs in an interrupt context, + // so I think it's safe for it to be shared for that duration. + let mut mapped_pages = false; + let mut already_mapped = false; + MemoryManager::with_mut(|mm| { + for page in 0..self.mem_alloc_tracker_pages { + let virt = SWAP_RPT_VADDR + page * PAGE_SIZE; + let entry = unsafe { + // assume that the soft-OOM handler has run at least once already, so that + // the L1 PTEs exist. If we hard-OOM before the soft-OOM handler ever runs, + // the below would panic. We can solve this by having the user space swapper + // force at least one fetch of the RPT on boot (this is probably a good idea + // anyways as it ensures that any heap space it needs to handle this is + // allocated). + crate::arch::mem::pagetable_entry(virt) + .expect("Couldn't access PTE for SWAP_RPT_VADDR; ensure soft-OOM handler runs once before hard-OOM") + .read_volatile() + }; + // only map pages if they aren't already mapped + if entry & MMUFlags::VALID.bits() == 0 { + mapped_pages = true; + crate::arch::mem::map_page_inner( + mm, + swapper_pid, + self.mem_alloc_tracker_paddr + page * PAGE_SIZE, + virt, + xous_kernel::MemoryFlags::R, + true, + ) + .ok(); + unsafe { crate::arch::mem::flush_mmu() }; + } else { + already_mapped = true; + } + } + if mapped_pages && already_mapped { + // I *think* that whether we map or don't map is always going to be all-or-nothing. + // However, in the case that only some pages have to be mapped, it probably means that + // somehow, we hard-OOM'd right in the middle of the page map/unmap routine within + // the soft-OOM handler. Shouldn't be possible -- `fetch_allocs` immediately disables + // interrupts -- but let's sanity check this assumption anyways. + todo!("Need to handle partial RPT maps -- tracking vector required in swapper"); + } else if mapped_pages { + self.unmap_rpt_after_hard_oom = true; + } + }); + }); + + #[cfg(feature = "debug-swap")] + println!("hard_oom - userspace activate"); + // this is safe because we're now in the swapper memory context, thanks to the previous call + unsafe { + self.blocking_activate_swapper(BlockingSwapOp::HardOomSyscall(original_tid, original_pid)); + } + } + + pub fn hard_oom_inline(&mut self) -> bool { + // we are in the pid/tid of the invoking process + // stash the thread state of the current pid/tid because the syscall return trampoline will smash it + self.oom_thread_backing = + Some(crate::arch::process::Process::with_current(|p| p.current_thread().clone())); + + println!("Entering inline hard OOM handler"); + crate::arch::process::Process::with_current(|p| { + println!( + "BEF HARD OOM HANDLER {}.{} sepc: {:x} sstatus: {:x?} satp: {:x?} reg {:08x?}", + p.pid().get(), + p.current_tid(), + riscv::register::sepc::read(), + riscv::register::sstatus::read().spp(), + riscv::register::satp::read(), + p.current_thread().registers, + ) + }); + // assemble this upstream of the stack save, because this affects stack + let call_precompute = xous_kernel::SysCall::SwapOp(SwapAbi::HardOom as usize, 0, 0, 0, 0, 0, 0); + + // at this point we are in user pid/tid, but supervisor mode. The current thread backing is + // about to be smashed by the syscall invocation. we want to return to this backing. + let mut current_sp: usize = 0; + + unsafe { + core::arch::asm!( + "mv {current_sp}, sp", + current_sp = out(reg) current_sp, + ) + } + println!(" --> SP extent d'{} bytes", EXCEPTION_STACK_TOP - current_sp); + let backup_stack_ptr: usize = self.oom_stack_backing.as_ptr() as usize; + let working_stack_end: usize = EXCEPTION_STACK_TOP; + let working_stack_ptr: usize = + EXCEPTION_STACK_TOP - self.oom_stack_backing.len() * core::mem::size_of::(); + unsafe { + core::arch::asm!( + "mv a7, {working_stack_end}", + "mv a6, {working_stack_ptr}", + "mv a5, {backup_stack_ptr}", + "100:", + "lw a4, 0(a6)", + "sw a4, 0(a5)", + "addi a6, a6, 4", + "addi a5, a5, 4", + "bltu a6, a7, 100b", + "mv {current_sp}, sp", + + working_stack_end = in(reg) working_stack_end, + working_stack_ptr = in(reg) working_stack_ptr, + backup_stack_ptr = in(reg) backup_stack_ptr, + current_sp = out(reg) current_sp, + ) + } + assert!( + EXCEPTION_STACK_TOP - current_sp <= self.oom_stack_backing.len() * core::mem::size_of::(), + "Backing stack not large enough to handle current kernel stack utilization." + ); + // invoke this as a nested syscall so it returns here + xous_kernel::rsyscall(call_precompute).ok(); + + let restore_stack_ptr: usize = self.oom_stack_backing.as_ptr() as usize; + let working_restore_end: usize = EXCEPTION_STACK_TOP; + let working_restore_ptr: usize = + EXCEPTION_STACK_TOP - self.oom_stack_backing.len() * core::mem::size_of::(); + unsafe { + core::arch::asm!( + "mv a7, {working_restore_end}", + "mv a6, {working_restore_ptr}", + "mv a5, {restore_stack_ptr}", + "100:", + "lw a4, 0(a5)", + "sw a4, 0(a6)", + "addi a6, a6, 4", + "addi a5, a5, 4", + "bltu a6, a7, 100b", + + working_restore_end = in(reg) working_restore_end, + working_restore_ptr = in(reg) working_restore_ptr, + restore_stack_ptr = in(reg) restore_stack_ptr, + ) + } + + crate::arch::process::Process::with_current(|p| { + println!( + "AFT HARD OOM HANDLER {}.{} sepc: {:x} sstatus: {:x?} satp: {:x?} reg {:08x?}", + p.pid().get(), + p.current_tid(), + riscv::register::sepc::read(), + riscv::register::sstatus::read().spp(), + riscv::register::satp::read(), + p.current_thread().registers, + ) + }); + + // then restore the thread state on return from the syscall + crate::arch::process::Process::with_current_mut(|p| { + println!("Returned from hard OOM handler, current pid {}/tid {}", p.pid().get(), p.current_tid()); + *p.current_thread_mut() = + self.oom_thread_backing.take().expect("No thread backing was set prior to OOM handler") + }); + + true + } + /// This call diverges into the userspace swapper. /// Divergent calls must turn of IRQs before memory spaces are changed. pub fn fetch_allocs(&mut self) -> ! { @@ -626,6 +815,10 @@ impl Swap { self.swapper_args[0] = self.swapper_state; self.swapper_args[1] = 3; // HardOom } + BlockingSwapOp::HardOomSyscall(_tid, _pid) => { + self.swapper_args[0] = self.swapper_state; + self.swapper_args[1] = 3; // HardOom + } } if let Some(op) = self.prev_op.take() { if let Some(dop) = self.nested_op { @@ -672,6 +865,7 @@ impl Swap { /// /// Safety: this call must only be invoked in the swapper's memory context pub unsafe fn exit_blocking_call(&mut self) -> Result { + println!("prev_op: {:x?}", self.prev_op); let result = match self.prev_op.take() { // Called from any process. Resume as if recovering from a page fault; absorb the rest of the // page fault handler code into this routine at the point where it would have @@ -806,11 +1000,6 @@ impl Swap { // return as a SysCall Ok(xous_kernel::Result::Scalar5(0, 0, 0, 0, 0)) } - // The caller point is still TBD. This is complicated because ideally we want the OOM'ing - // process to stop trying to do allocate memory until the swapper has had some time to run, - // *but* the swapper is running as a userspace process which means it could be interrupted. - // We might be able to play shenanigans with disabling IRQs *very carefully* though, so as - // to kill pre-emption and make the swapper the only thing that can possibly run. Some(BlockingSwapOp::HardOom(tid, pid, on_vaddr)) => { // We enter this from the swapper's memory space assert!( @@ -922,6 +1111,51 @@ impl Swap { // the current memory space is the target PID, so we will resume into the target PID Ok(xous_kernel::Result::ResumeProcess) } + Some(BlockingSwapOp::HardOomSyscall(tid, pid)) => { + // We enter this from the swapper's memory space + assert!( + crate::arch::process::current_pid().get() == SWAPPER_PID, + "Hard OOM syscall did not return from swapper's space" + ); + // unmap RPT from swapper's userspace, if it was previously unmapped + if self.unmap_rpt_after_hard_oom { + MemoryManager::with_mut(|mm| { + for page in 0..self.mem_alloc_tracker_pages { + crate::arch::mem::unmap_page_inner(mm, SWAP_RPT_VADDR + page * PAGE_SIZE).ok(); + } + }); + } + + // return to the original pid memory space, now that we have memory + SystemServices::with_mut(|system_services| { + // Cleanup the swapper + system_services.finish_swap(); + + if !crate::arch::irq::is_handling_irq() { + system_services.swap_resume_to_userspace(pid, tid).expect("couldn't swap_resume"); + } + + // Switch to target process + let process = system_services.get_process_mut(pid).unwrap(); + process.mapping.activate().unwrap(); + process.activate().unwrap(); + // Activate the current context + crate::arch::process::Process::current().set_tid(tid).unwrap(); + process.current_thread = tid; + }); + + // return to kernel space -- this call can only be originated from kernel space + unsafe { riscv::register::sstatus::set_spp(riscv::register::sstatus::SPP::Supervisor) }; + + // restore IRQ state (don't borrow the system handler's state tracker, so we don't smash + // it by accident) + let sim = self.oom_irq_backing.take().expect("Someone stole our IRQ backing!"); + #[cfg(feature = "debug-swap")] + println!("OOM syscall restoring IRQ: {:x}", sim); + sim_write(sim); + + Ok(xous_kernel::Result::Scalar5(0, 0, 0, 0, 0)) + } None => panic!("No previous swap op was set"), }; if let Some(op) = self.nested_op.take() {