// SPDX-FileCopyrightText: 2020 Sean Cross // SPDX-License-Identifier: Apache-2.0 use core::fmt; use xous_kernel::{MemoryFlags, MemoryRange, PID, arch::*}; pub use crate::arch::mem::MemoryMapping; #[cfg(baremetal)] use crate::arch::mem::{MMUFlags, flush_mmu, pagetable_entry}; use crate::arch::process::Process; #[cfg(feature = "swap")] use crate::swap::SwapAlloc; #[derive(Debug)] // below suppresses warning from unused Move argument in hosted mode #[allow(dead_code)] enum ClaimReleaseMove { Claim, Release, Move(PID /* from */), } #[allow(dead_code)] #[repr(C)] pub struct MemoryRangeExtra { mem_start: u32, mem_size: u32, mem_tag: u32, _padding: u32, } impl fmt::Display for MemoryRangeExtra { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "{}{}{}{} - ({:08x}) {:08x} - {:08x} {} bytes", ((self.mem_tag) & 0xff) as u8 as char, ((self.mem_tag >> 8) & 0xff) as u8 as char, ((self.mem_tag >> 16) & 0xff) as u8 as char, ((self.mem_tag >> 24) & 0xff) as u8 as char, self.mem_tag, self.mem_start, self.mem_start + self.mem_size, self.mem_size ) } } pub struct MemoryManager { ram_start: usize, ram_size: usize, #[allow(dead_code)] ram_name: u32, #[allow(dead_code)] last_ram_page: usize, } impl Default for MemoryManager { fn default() -> Self { Self::default_hack() } } #[cfg(not(baremetal))] std::thread_local!(static MEMORY_MANAGER: core::cell::RefCell = core::cell::RefCell::new(MemoryManager::default())); #[cfg(baremetal)] static mut MEMORY_MANAGER: MemoryManager = MemoryManager::default_hack(); #[cfg(baremetal)] #[cfg(feature = "swap")] static mut MEMORY_ALLOCATIONS: &mut [SwapAlloc] = &mut []; #[cfg(baremetal)] #[cfg(not(feature = "swap"))] static mut MEMORY_ALLOCATIONS: &mut [Option] = &mut []; #[cfg(baremetal)] static mut EXTRA_ALLOCATIONS: &mut [Option] = &mut []; #[cfg(baremetal)] static mut EXTRA_REGIONS: &[MemoryRangeExtra] = &[]; #[cfg(feature = "swap")] /// Epoch rollover handling -- two options: /// - Fast epoch rollover, but long-lasting performance impact: just reset all counters to 0, and let the /// system re-discover LRU order based on usage patterns again /// - Slow epoch rollover, with no performance impact: go through all pages and "compact" the count down to /// the lowest level, resetting the epoch counter to the next available epoch. LRU patterns are /// maintained, but the search could take a long time. /// /// In this implementation, we go with the slow epoch rollover, under the theory that (a) the /// actual size of the list to normalize is "small" (few hundred to thousands of /// entries) compared to the cost of re-discovering the LRU order by swapping out (e.g. /// encrypting and decrypting) 4096-byte pages. The implementation itself could be more /// efficient if we could, for example, make a full copy of the table, but we don't /// want to blow out the kernel stack or allocate a static structure for temporary /// data, so the entire algorithm is implemented with about a dozen entries of scratch /// space, iterating over the list until every item is renormalized. /// /// The code is in the `mem` crate instead of `swap`, because it has to directly access the locally scoped /// MEM_ALLOCATIONS variable. The loop is likewise marked unsafe, because it accesses this static mut. pub unsafe fn renormalize_allocs() -> u32 { #[cfg(feature = "debug-swap-verbose")] for (i, alloc) in (&mut *(&raw mut MEMORY_ALLOCATIONS)).iter().enumerate() { if alloc.get_raw_vpn() != 0 || alloc.get_timestamp() != 0 { println!(" {:x}: {:08x}[{:x}]", i, alloc.get_raw_vpn(), alloc.get_timestamp()); } } // min_search_limit tracks the window of values we can renormalize in a single pass. // It is guaranteed to increase by exactly RENORM_PASS_SIZE every pass, except for the final pass. let mut min_search_limit = 0; loop { let mut ms = crate::utils::MinSet::new(); for d in (&mut *(&raw mut MEMORY_ALLOCATIONS)).iter() { if d.get_timestamp() >= min_search_limit { ms.insert(d.get_timestamp()); } } // remap elements in d that match the minset for d in (&mut *(&raw mut MEMORY_ALLOCATIONS)).iter_mut() { if let Some(i) = ms.index_of(d.get_timestamp()) { d.set_timestamp(i as u32 + min_search_limit); } } min_search_limit += crate::utils::RENORM_PASS_SIZE as u32; if ms.max() == u32::MAX { break; } } #[cfg(feature = "debug-swap-verbose")] for (i, alloc) in (&mut *(&raw mut MEMORY_ALLOCATIONS)).iter().enumerate() { if alloc.get_raw_vpn() != 0 || alloc.get_timestamp() != 0 { println!(" {:x}: {:08x}[{:x}]", i, alloc.get_raw_vpn(), alloc.get_timestamp()); } } (&mut *(&raw mut MEMORY_ALLOCATIONS)).iter().max().unwrap().get_timestamp() } /// Initialize the memory map. /// This will go through memory and map anything that the kernel is /// using to process 1, then allocate a pagetable for this process /// and place it at the usual offset. The MMU will not be enabled yet, /// as the process entry has not yet been created. impl MemoryManager { const fn default_hack() -> Self { MemoryManager { ram_start: 0, ram_size: 0, ram_name: 0, last_ram_page: 0 } } // /// Calls the provided function with the current inner process state. // pub fn with(f: F) -> R // where // F: FnOnce(&MemoryManager) -> R, // { // #[cfg(baremetal)] // unsafe { // f(&MEMORY_MANAGER) // } // #[cfg(not(baremetal))] // MEMORY_MANAGER.with(|ss| f(&ss.borrow())) // } pub fn with_mut(f: F) -> R where F: FnOnce(&mut MemoryManager) -> R, { #[cfg(baremetal)] unsafe { f(&mut *core::ptr::addr_of_mut!(MEMORY_MANAGER)) } #[cfg(not(baremetal))] MEMORY_MANAGER.with(|ss| f(&mut ss.borrow_mut())) } #[cfg(baremetal)] pub fn with(f: F) -> R where F: FnOnce(&MemoryManager) -> R, { #[cfg(baremetal)] unsafe { f(&*core::ptr::addr_of!(MEMORY_MANAGER)) } #[cfg(not(baremetal))] MEMORY_MANAGER.with(|ss| f(&ss.borrow_mut())) } #[cfg(baremetal)] pub fn init_from_memory( &mut self, rpt_base: usize, xpt_base: usize, args: &crate::args::KernelArguments, ) -> Result<(), xous_kernel::Error> { use core::slice; let mut args_iter = args.iter(); let xarg_def = args_iter.next().expect("mm: no kernel arguments found"); unsafe { assert!( (&mut *(&raw mut EXTRA_REGIONS)).is_empty(), "mm: self.extra.len() was {}, not 0", (&mut *(&raw mut EXTRA_REGIONS)).len() ); } assert!(xarg_def.name == u32::from_le_bytes(*b"XArg"), "mm: first tag wasn't XArg"); assert!(xarg_def.data[1] == 1, "mm: XArg had unexpected version"); self.ram_start = xarg_def.data[2] as usize; self.ram_size = xarg_def.data[3] as usize; self.ram_name = xarg_def.data[4]; let mem_size = self.ram_size / PAGE_SIZE; let mut extra_size = 0; for tag in args_iter { if tag.name == u32::from_le_bytes(*b"MREx") { unsafe { assert!( (&mut *(&raw mut EXTRA_REGIONS)).is_empty(), "mm: MREx tag appears twice! self.extra.len() is {}, not 0", (&mut *(&raw mut EXTRA_REGIONS)).len() ); let ptr = tag.data.as_ptr() as *mut MemoryRangeExtra; *(&mut *(&raw mut EXTRA_REGIONS)) = slice::from_raw_parts_mut( ptr, tag.data.len() * 4 / core::mem::size_of::(), ) }; } } unsafe { for range in (&mut *(&raw mut EXTRA_REGIONS)).iter() { extra_size += range.mem_size as usize / PAGE_SIZE; } } #[cfg(feature = "swap")] unsafe { MEMORY_ALLOCATIONS = slice::from_raw_parts_mut(rpt_base as *mut SwapAlloc, mem_size); crate::swap::Swap::with_mut(|s| { s.init_rpt( rpt_base, ((mem_size * core::mem::size_of::() + PAGE_SIZE - 1) & !(PAGE_SIZE - 1)) / PAGE_SIZE, ) }); } #[cfg(not(feature = "swap"))] unsafe { MEMORY_ALLOCATIONS = slice::from_raw_parts_mut(rpt_base as *mut Option, mem_size) }; unsafe { EXTRA_ALLOCATIONS = slice::from_raw_parts_mut(xpt_base as *mut Option, extra_size) } Ok(()) } /// Print the number of RAM bytes used by the specified process. /// This does not include memory such as peripherals and CSRs. #[cfg(baremetal)] pub fn ram_used_by(&self, pid: PID) -> usize { let mut owned_bytes = 0; #[cfg(baremetal)] #[cfg(not(feature = "swap"))] unsafe { for owner in &(&mut *(&raw mut MEMORY_ALLOCATIONS))[0..self.ram_size / PAGE_SIZE] { if owner == &Some(pid) { owned_bytes += PAGE_SIZE; } } } #[cfg(baremetal)] #[cfg(feature = "swap")] unsafe { for owner in &(&mut *(&raw mut MEMORY_ALLOCATIONS))[0..self.ram_size / PAGE_SIZE] { if owner.is_pid(pid) { owned_bytes += PAGE_SIZE; } } } owned_bytes } #[cfg(all(baremetal, feature = "debug-print"))] #[allow(dead_code)] pub fn print_ownership(&self) { println!("Ownership ({} bytes in all):", unsafe { (&mut *(&raw mut MEMORY_ALLOCATIONS)).len() + (&mut *(&raw mut EXTRA_ALLOCATIONS)).len() }); let mut offset = 0; unsafe { // First, we build a &[u8]... let name_bytes = self.ram_name.to_le_bytes(); // ... and then convert that slice into a string slice let _ram_name = core::str::from_utf8_unchecked(&name_bytes); println!( " Region {} ({:08x}) {:08x} - {:08x} {} bytes:", _ram_name, self.ram_name, self.ram_start, self.ram_start + self.ram_size, self.ram_size ); }; #[cfg(feature = "swap")] for o in 0..self.ram_size / PAGE_SIZE { println!( " {:08x} => {:?}", self.ram_size + o * PAGE_SIZE, (&mut *(&raw mut MEMORY_ALLOCATIONS))[offset + o].get_pid() ); } offset = 0; // Go through additional regions looking for this address, and claim it // if it's not in use. unsafe { for region in (&mut *(&raw mut EXTRA_REGIONS)).iter() { println!(" Region {}:", region); for o in 0..(region.mem_size as usize) / PAGE_SIZE { if let Some(allocation) = (&mut *(&raw mut EXTRA_ALLOCATIONS))[offset + o] { println!( " {:08x} => {}", (region.mem_start as usize) + o * PAGE_SIZE, allocation.get() ) } } offset += region.mem_size as usize / PAGE_SIZE; } } } /// Allocate a single page to the given process. DOES NOT ZERO THE PAGE!!! /// This function CANNOT zero the page, as it hasn't been mapped yet. #[cfg(baremetal)] #[cfg(not(feature = "swap"))] pub fn alloc_page(&mut self, pid: PID) -> Result { // Go through all RAM pages looking for a free page. // println!("Allocating page for PID {}", pid); unsafe { let end_point = self.ram_size / PAGE_SIZE; let starting_point = self.last_ram_page.max(end_point); for (allocation, index) in (&mut *(&raw mut MEMORY_ALLOCATIONS))[starting_point..end_point] .iter_mut() .zip(starting_point..) .chain((&mut *(&raw mut MEMORY_ALLOCATIONS))[..starting_point].iter_mut().zip(0..)) { if allocation.is_none() { *allocation = Some(pid); self.last_ram_page = index + 1; let page = index * PAGE_SIZE + self.ram_start; return Ok(page); } } } Err(xous_kernel::Error::OutOfMemory) } #[cfg(baremetal)] #[cfg(feature = "swap")] // Take a physical address and indicate that it's been queried so it doesn't get suggested as LRU any time // soon. Addresses outside of RAM are just ignored. pub fn touch(&self, paddr: usize) { if paddr >= self.ram_start && paddr < self.ram_start + self.ram_size { unsafe { (&mut *(&raw mut MEMORY_ALLOCATIONS))[(paddr - self.ram_start) / PAGE_SIZE].touch(); } } } #[cfg(baremetal)] #[cfg(feature = "swap")] pub fn memory_size(&self) -> usize { self.ram_size } #[cfg(feature = "debug-swap")] #[allow(dead_code)] pub fn rpt_base(&self) -> usize { unsafe { (&mut *(&raw mut MEMORY_ALLOCATIONS)).as_ptr() as usize } } #[cfg(feature = "debug-swap")] #[allow(dead_code)] /// This function is "improper" in that it returns a bogus value if the memory allocations are /// out of range, but its purpose is only for debugging. This is not suitable for use in any /// other context. pub unsafe fn get_timestamp(&self, paddr: usize) -> u32 { if paddr >= self.ram_start && paddr < self.ram_start + self.ram_size { unsafe { (&mut *(&raw mut MEMORY_ALLOCATIONS))[(paddr - self.ram_start) / PAGE_SIZE].get_timestamp() } } else { 0xDEAD_BEEF } } #[cfg(baremetal)] #[cfg(feature = "swap")] /// Similar to alloc_page(), but this implementation takes the vaddr being allocated so we /// can do more detailed bookkeeping on least recently used pages. pub fn alloc_page_oomable( &mut self, pid: PID, vaddr: Option, ) -> Result { loop { // Go through all RAM pages looking for a free page. // println!("Allocating page for PID {}", pid); unsafe { let end_point = self.ram_size / PAGE_SIZE; let starting_point = self.last_ram_page.max(end_point); for (allocation, index) in (&mut *(&raw mut MEMORY_ALLOCATIONS))[starting_point..end_point] .iter_mut() .zip(starting_point..) .chain((&mut *(&raw mut MEMORY_ALLOCATIONS))[..starting_point].iter_mut().zip(0..)) { if allocation.is_none() { allocation.update(Some(pid), vaddr); self.last_ram_page = index + 1; let page = index * PAGE_SIZE + self.ram_start; return Ok(page); } } } crate::swap::Swap::with_mut(|s| { s.swap_reentrant_syscall(xous_kernel::SysCall::SwapOp( crate::swap::SwapAbi::HardOom as usize, 0, 0, 0, 0, 0, 0, )) }); } } /// Find a virtual address in the current process that is big enough /// to fit `size` bytes. pub fn find_virtual_address( &mut self, virt_ptr: *mut u8, size: usize, kind: xous_kernel::MemoryType, ) -> Result<*mut u8, xous_kernel::Error> { // If we were supplied a perfectly good address, return that. if !virt_ptr.is_null() { return Ok(virt_ptr); } // let process = Process::current(); Process::with_inner_mut(|process_inner| { let (start, end, initial) = match kind { xous_kernel::MemoryType::Stack => return Err(xous_kernel::Error::BadAddress), xous_kernel::MemoryType::Heap => { let new_virt = process_inner.mem_heap_base + process_inner.mem_heap_size + PAGE_SIZE; if new_virt + size > process_inner.mem_heap_base + process_inner.mem_heap_max { return Err(xous_kernel::Error::OutOfMemory); } return Ok(new_virt as *mut u8); } xous_kernel::MemoryType::Default => ( process_inner.mem_default_base, process_inner.mem_default_base + 0x1000_0000, process_inner.mem_default_last, ), xous_kernel::MemoryType::Messages => ( process_inner.mem_message_base, process_inner.mem_message_base + 0x40_0000, // Limit to one superpage process_inner.mem_message_last, ), }; // Look for a sequence of `size` pages that are free. for potential_start in (initial..end - size).step_by(PAGE_SIZE) { let mut all_free = true; for check_page in (potential_start..potential_start + size).step_by(PAGE_SIZE) { if !crate::arch::mem::address_available(check_page) { all_free = false; break; } } if all_free { match kind { xous_kernel::MemoryType::Default => process_inner.mem_default_last = potential_start, xous_kernel::MemoryType::Messages => process_inner.mem_message_last = potential_start, other => panic!("invalid kind: {:?}", other), } return Ok(potential_start as *mut u8); } } for potential_start in (start..initial).step_by(PAGE_SIZE) { let mut all_free = true; for check_page in (potential_start..potential_start + size).step_by(PAGE_SIZE) { if !crate::arch::mem::address_available(check_page) { all_free = false; break; } } if all_free { match kind { xous_kernel::MemoryType::Default => process_inner.mem_default_last = potential_start, xous_kernel::MemoryType::Messages => process_inner.mem_message_last = potential_start, other => panic!("invalid kind: {:?}", other), } return Ok(potential_start as *mut u8); } } Err(xous_kernel::Error::BadAddress) }) } /// Reserve the given range without actually allocating memory. /// That way we can overpromise on stack size and heap size without /// needing to actually have pages to back it. pub fn reserve_range( &mut self, virt_ptr: *mut u8, size: usize, flags: MemoryFlags, ) -> Result { // If no address was specified, pick the next address that fits // in the "default" range let virt = self.find_virtual_address(virt_ptr, size, xous_kernel::MemoryType::Default)? as usize; if virt & 0xfff != 0 { return Err(xous_kernel::Error::BadAlignment); } if size & 0xfff != 0 { return Err(xous_kernel::Error::BadAlignment); } let mut mm = MemoryMapping::current(); for virt in (virt..(virt + size)).step_by(PAGE_SIZE) { // FIXME: Un-reserve addresses if we encounter an error here mm.reserve_address(self, virt, flags)?; } unsafe { xous_kernel::MemoryRange::new(virt_ptr as usize, size) } } /// Attempt to allocate a single page from the default section. /// Note that this will be backed by a real page. #[cfg(baremetal)] pub fn map_zeroed_page(&mut self, pid: PID, is_user: bool) -> Result<*mut usize, xous_kernel::Error> { let virt = self.find_virtual_address(core::ptr::null_mut(), PAGE_SIZE, xous_kernel::MemoryType::Default)? as usize; // Grab the next available page. This claims it for this process. #[cfg(not(feature = "swap"))] let phys = self.alloc_page(pid)?; #[cfg(feature = "swap")] let phys = self.alloc_page_oomable(pid, Some(virt))?; // Actually perform the map. At this stage, every physical page should be owned by us. if let Err(e) = crate::arch::mem::map_page_inner( self, pid, phys as usize, virt as usize, xous_kernel::MemoryFlags::R | xous_kernel::MemoryFlags::W, false, ) { self.release_page(phys as *mut usize, pid).ok(); return Err(e); } let virt = virt as *mut usize; // Zero-out the page let range_start = virt; let range_end = range_start.wrapping_add(PAGE_SIZE / core::mem::size_of::()); unsafe { crate::mem::bzero(range_start, range_end); }; if is_user { crate::arch::mem::hand_page_to_user(virt as _)?; } // klog!( // "Mapped {:08x} -> {:08x} (user? {})", // phys as usize, virt as usize, is_user // ); Ok(virt) } pub fn is_main_memory(&self, phys: *mut u8) -> bool { (phys as usize) >= self.ram_start && (phys as usize) < self.ram_start + self.ram_size } #[cfg(feature = "memmap-flash")] pub fn is_mapped_flash(&self, virt: *mut u8) -> bool { // true if the address starts with the bitmask of the virtual start of the MMAP region (virt as usize & xous_kernel::arch::MMAP_VIRT_BASE) == xous_kernel::arch::MMAP_VIRT_BASE } /// Attempt to map the given physical address into the virtual address space /// of this process. /// /// # Errors /// /// * MemoryInUse - The specified page is already mapped pub fn map_range( &mut self, phys_ptr: *mut u8, virt_ptr: *mut u8, size: usize, pid: PID, flags: MemoryFlags, kind: xous_kernel::MemoryType, ) -> Result { #[cfg(baremetal)] let mut phys = phys_ptr as usize; #[cfg(not(baremetal))] let phys = phys_ptr as usize; let virt = self.find_virtual_address(virt_ptr, size, kind)?; // Determine if a contiguous chunk of RAM needs to be allocated for a device // This case happens when we don't specify a physical address, but also specify the DEV // flag. let device_ram = (flags & MemoryFlags::DEV == MemoryFlags::DEV) && (phys == 0); // If no physical address is specified and the range is in the pure virtual mapping request, // just allocate the region as "swapped". No further checks is done on the validity of the requested // range - if the range is out of bounds, it will be caught as a runtime error in the resolver // that attempts to find the physical page that corresponds to a virtual mapping. #[cfg(baremetal)] if phys == 0 && (flags & MemoryFlags::VIRT == MemoryFlags::VIRT) && ((virt_ptr as usize & MMAP_VIRT_BASE) == MMAP_VIRT_BASE) { let mut mm = MemoryMapping::current(); // round down any virtual address to the next page let start = virt_ptr as usize & !(PAGE_SIZE - 1); // round up to the nearest page boundary let end = (virt_ptr as usize + size + PAGE_SIZE - 1) & !(PAGE_SIZE - 1); for virt in (start..end).step_by(PAGE_SIZE) { // Pages are read-only. Writes take a special call to ensure atomicity of write // updates (and subsequent page unmap). Valid is not set, because it's not // wired into memory, and "P" (swap) is set to indicate this is a swapper managed page. mm.reserve_address(self, virt, MemoryFlags::R | MemoryFlags::P)?; // now mark the page as USER let pte = pagetable_entry(virt)?; unsafe { pte.write_volatile(pte.read_volatile() | MMUFlags::USER.bits()); flush_mmu(); } } // note that the region returned is snapped to the nearest page boundary, even if // the use called us with unaligned addresses. return unsafe { xous_kernel::MemoryRange::new(start as usize, end - start) }; } // If no physical address is specified, give the user the next available pages if phys == 0 && !device_ram { return self.reserve_range(virt, size, flags); } #[cfg(baremetal)] if device_ram { // Device RAM allocation: search for contiguous block of physical RAM so we can share // the pages with e.g. DMA or other hardware resources. let pages_to_claim = size / PAGE_SIZE; // this is correct because only page-sized requests are allowed. #[cfg(feature = "debug-swap")] println!("Device RAM request. Searching for {} contiguous pages.", pages_to_claim); // Safety: is it safe to iterate through MEMORY_ALLOCATIONS like this? I'm not actually sure. // I suppose we could end up in trouble if the kernel is interrupted and the allocation table // changes, but we don't have a locking mechanism for this sort of thing (yet). Until we have // a better way to do this, it will have to do! let mut range_start: Option = None; let mut current_run = 0; unsafe { for (index, entry) in (&mut *(&raw mut MEMORY_ALLOCATIONS)).iter().enumerate() { if entry.is_none() { if let Some(_start) = range_start { current_run += 1; } else { range_start = Some(index); current_run = 1; } } else { range_start = None; current_run = 0; } if current_run >= pages_to_claim { break; } } } if let Some(start) = range_start { if current_run >= pages_to_claim { // success! phys = start * PAGE_SIZE + self.ram_start; #[cfg(feature = "debug-swap")] println!("Block found starting at {:x}", phys); } else { return Err(xous_kernel::Error::OutOfMemory); } } else { // couldn't find a contiguous location large enough; OOM for now. // Punt to userland to clear up memory. return Err(xous_kernel::Error::OutOfMemory); } } // 1. Attempt to claim all physical pages in the range for claim_phys in (phys..(phys + size)).step_by(PAGE_SIZE) { if let Err(err) = self.claim_page(claim_phys as *mut usize, pid) { // If we were unable to claim one or more pages, release everything and return for rel_phys in (phys..claim_phys).step_by(PAGE_SIZE) { self.release_page(rel_phys as *mut usize, pid).ok(); } return Err(err); } else { #[cfg(feature = "swap")] if device_ram { let offset = (claim_phys - self.ram_start) / PAGE_SIZE; #[cfg(feature = "debug-swap")] println!("Address {:x} marked as wired", claim_phys); unsafe { MEMORY_ALLOCATIONS[offset].set_wired(); } } } } #[cfg(feature = "debug-swap")] println!("Mapping pa {:x} | va {:x} region of {} bytes", phys, virt as usize, size); // Actually perform the map. At this stage, every physical page should be owned by us. for offset in (0..size).step_by(PAGE_SIZE) { if let Err(e) = crate::arch::mem::map_page_inner( self, pid, offset + phys as usize, offset + virt as usize, flags, false, ) { for unmap_offset in (0..offset).step_by(PAGE_SIZE) { crate::arch::mem::unmap_page_inner(self, unmap_offset + virt as usize).ok(); self.release_page((unmap_offset + phys) as *mut usize, pid).ok(); } #[cfg(feature = "debug-swap")] println!("Error encountered in map_page_inner at last stage of mapping; releasing all pages"); return Err(e); } } if device_ram { // The assumption is that device_ram pages are only ever allocated by a userspace call. // If the kernel uses this to allocate kernel structures, it will fail. // clear the memory first unsafe { crate::mem::bzero(virt, virt.wrapping_add(size)) }; // now hand it to userspace for offset in (0..size).step_by(PAGE_SIZE) { crate::arch::mem::hand_page_to_user(virt.wrapping_add(offset)) .expect("couldn't hand page to user"); } // sanity check it #[cfg(feature = "debug-swap-verbose")] crate::arch::mem::MemoryMapping::current().print_map(); } unsafe { MemoryRange::new(virt as usize, size) } } /// Attempt to map the given physical address into the virtual address space /// of this process. /// /// # Errors /// /// * MemoryInUse - The specified page is already mapped pub fn unmap_page(&mut self, virt: *mut usize) -> Result { let pid = crate::arch::process::current_pid(); // If the virtual address has an assigned physical address, release that // address from this process. if let Ok(phys) = crate::arch::mem::virt_to_phys(virt as usize) { self.release_page(phys as *mut usize, pid).ok(); } // Free the virtual address. crate::arch::mem::unmap_page_inner(self, virt as usize) } /// Move a page from one process into another, keeping its permissions. #[allow(dead_code)] pub fn move_page( &mut self, src_pid: PID, src_mapping: &MemoryMapping, src_addr: *mut u8, dest_pid: PID, dest_mapping: &MemoryMapping, dest_addr: *mut u8, ) -> Result<(), xous_kernel::Error> { let phys_addr = crate::arch::mem::virt_to_phys(src_addr as usize)?; crate::arch::mem::move_page_inner(self, src_mapping, src_addr, dest_pid, dest_mapping, dest_addr)?; self.claim_release_move(phys_addr as *mut usize, dest_pid, ClaimReleaseMove::Move(src_pid)) } #[allow(dead_code)] /// Move the page in the process mapping listing without manipulating /// the pagetables at all. pub fn move_page_raw(&mut self, phys_addr: *mut usize, dest_pid: PID) -> Result<(), xous_kernel::Error> { self.claim_release_move( phys_addr as *mut usize, dest_pid, ClaimReleaseMove::Move(crate::arch::process::current_pid()), ) } /// Mark the page in the current process as being lent. If the borrow is /// read-only, then additionally remove the "write" bit on it. If the page /// is writable, then remove it from the current process until the borrow is /// returned. #[allow(dead_code)] pub fn lend_page( &mut self, src_mapping: &MemoryMapping, src_addr: *mut u8, dest_pid: PID, dest_mapping: &MemoryMapping, dest_addr: *mut u8, mutable: bool, ) -> Result { // If this page is to be writable, detach it from this process. // Otherwise, mark it as read-only to prevent a process from modifying // the page while it's borrowed. crate::arch::mem::lend_page_inner( self, src_mapping, src_addr as _, dest_pid, dest_mapping, dest_addr as _, mutable, ) } /// Return the range from `src_mapping` back to `dest_mapping` #[allow(dead_code)] pub fn unlend_page( &mut self, src_mapping: &MemoryMapping, src_addr: *mut u8, dest_pid: PID, dest_mapping: &MemoryMapping, dest_addr: *mut u8, ) -> Result { // If this page is to be writable, detach it from this process. // Otherwise, mark it as read-only to prevent a process from modifying // the page while it's borrowed. crate::arch::mem::return_page_inner(self, src_mapping, src_addr, dest_pid, dest_mapping, dest_addr) } #[cfg(baremetal)] pub fn ensure_page_exists(&mut self, address: usize) -> Result<(), xous_kernel::Error> { crate::arch::mem::ensure_page_exists_inner(address).and(Ok(())) } /// Claim the given memory for the given process, or release the memory /// back to the free pool. #[cfg(not(baremetal))] fn claim_release_move( &mut self, _addr: *mut usize, _pid: PID, _action: ClaimReleaseMove, ) -> Result<(), xous_kernel::Error> { Ok(()) } #[cfg(baremetal)] fn claim_release_move( &mut self, addr: *mut usize, pid: PID, action: ClaimReleaseMove, ) -> Result<(), xous_kernel::Error> { /// Modify the memory tracking table to note which process owns /// the specified address. fn action_inner( owner_addr: &mut Option, pid: PID, action: ClaimReleaseMove, ) -> Result<(), xous_kernel::Error> { if let Some(current_pid) = *owner_addr { if current_pid != pid { // klog!( // "In claim_or_release({}, {}, {:?}) -- addr is owned by {} not {}", // owner_addr.map(|v| v.get()).unwrap_or_default(), // pid, // action, // current_pid, // pid // ); if let ClaimReleaseMove::Move(existing_pid) = action { if existing_pid != current_pid { return Err(xous_kernel::Error::MemoryInUse); } } else { return Err(xous_kernel::Error::MemoryInUse); } } } match action { ClaimReleaseMove::Claim | ClaimReleaseMove::Move(_) => { *owner_addr = Some(pid); } ClaimReleaseMove::Release => { *owner_addr = None; } } Ok(()) } #[cfg(feature = "swap")] fn action_inner_tracking( owner_addr: &mut SwapAlloc, pid: PID, action: ClaimReleaseMove, addr: usize, ) -> Result<(), xous_kernel::Error> { if let Some(current_pid) = owner_addr.get_pid() { if current_pid != pid { // klog!( // "In claim_or_release({}, {}, {:?}) -- addr is owned by {} not {}", // owner_addr.map(|v| v.get()).unwrap_or_default(), // pid, // action, // current_pid, // pid // ); if let ClaimReleaseMove::Move(existing_pid) = action { if existing_pid != current_pid { return Err(xous_kernel::Error::MemoryInUse); } } else { println!("ERR: {:?} != {:?}", current_pid, pid); return Err(xous_kernel::Error::MemoryInUse); } } } match action { ClaimReleaseMove::Claim | ClaimReleaseMove::Move(_) => { unsafe { owner_addr.update(Some(pid), Some(addr)) }; } ClaimReleaseMove::Release => { unsafe { owner_addr.update(None, None) }; } } Ok(()) } let addr = addr as usize; // Ensure the address lies on a page boundary if cfg!(baremetal) && addr & 0xfff != 0 { return Err(xous_kernel::Error::BadAlignment); } let mut offset = 0; // Happy path: The address is in main RAM if addr >= self.ram_start && addr < self.ram_start + self.ram_size { offset += (addr - self.ram_start) / PAGE_SIZE; #[cfg(not(feature = "swap"))] return unsafe { action_inner(&mut MEMORY_ALLOCATIONS[offset], pid, action) }; #[cfg(feature = "swap")] return unsafe { action_inner_tracking(&mut MEMORY_ALLOCATIONS[offset], pid, action, addr) }; } offset = 0; // Go through additional regions looking for this address, and claim it // if it's not in use. unsafe { for region in EXTRA_REGIONS { if addr >= (region.mem_start as usize) && addr < (region.mem_start + region.mem_size) as usize { // ------------------------------- // FIXME: workaround to allow to share the same UART peripheral // between the kernel and xous-log processes #[cfg(feature = "atsama5d27")] { let uart_base = crate::platform::atsama5d2::uart::HW_UART_BASE as usize; if pid.get() != 1 && (addr == uart_base || addr == uart_base + 0x1000 || addr == uart_base + 0x2000 || addr == uart_base + 0x3000) { klog!("[!] UART sharing workaround used for {:08x} address", addr); return Ok(()); } } // ------------------------------- offset += (addr - (region.mem_start as usize)) / PAGE_SIZE; return action_inner(&mut EXTRA_ALLOCATIONS[offset], pid, action); } offset += region.mem_size as usize / PAGE_SIZE; } } // println!( // "mem: unable to claim or release physical address {:08x}", // addr // ); Err(xous_kernel::Error::BadAddress) } /// Mark a given address as being owned by the specified process ID fn claim_page(&mut self, addr: *mut usize, pid: PID) -> Result<(), xous_kernel::Error> { self.claim_release_move(addr, pid, ClaimReleaseMove::Claim) } /// Mark a given address as no longer being owned by the specified process ID fn release_page(&mut self, addr: *mut usize, pid: PID) -> Result<(), xous_kernel::Error> { self.claim_release_move(addr, pid, ClaimReleaseMove::Release) } #[cfg(feature = "swap")] /// Same as `release_page`, but with public visibility when the `swap` feature is active pub fn release_page_swap(&mut self, addr: *mut usize, pid: PID) -> Result<(), xous_kernel::Error> { self.claim_release_move(addr, pid, ClaimReleaseMove::Release) } /// Convert an offset in the `MEMORY_ALLOCATIONS` array into a physical address. #[cfg(baremetal)] fn allocation_offset_to_address(&self, offset: usize) -> Option { // If the offset is within the RAM size, simply turn it into // an address. if offset < self.ram_size as usize / PAGE_SIZE { Some(self.ram_start as usize + offset * PAGE_SIZE) } else { // No region was found. None } } /// Convert an offset in the `EXTRA_ALLOCATIONS` array into a physical address. #[cfg(baremetal)] fn allocation_offset_to_address_extra(&self, offset: usize) -> Option { // Loop through all regions looking for the address. // NOTE: This needs to be linear because each memory region has a different length. let mut offset_in_region = offset; unsafe { for region in EXTRA_REGIONS { // If the offset exceeds the current region, skip to the // next region. if offset_in_region >= (region.mem_size as usize / PAGE_SIZE) { offset_in_region -= region.mem_size as usize / PAGE_SIZE; continue; } return Some(region.mem_start as usize + (offset_in_region * PAGE_SIZE)); } } // No region was found. None } /// Free all memory that belongs to a process. This does not unmap the /// memory from the process, it only marks it as free. /// This is very unsafe because the memory can immediately be re-allocated /// to another process, so only call this as part of destroying a process. pub unsafe fn release_all_memory_for_process(&mut self, _pid: PID) { // release the main memory allocs #[cfg(baremetal)] for (idx, owner) in (&mut *(&raw mut MEMORY_ALLOCATIONS)).iter_mut().enumerate() { // If this address has been allocated to this process, consider // freeing it or reparenting it. #[cfg(not(feature = "swap"))] if owner == &mut Some(_pid) { let phys_addr = self.allocation_offset_to_address(idx).unwrap(); if crate::arch::mem::page_is_lent(phys_addr as *mut u8) { // If the page is lent, reparent it to PID 1 so it will // get freed when it is returned. *owner = PID::new(1); } else { // Mark this page as free, which allows it to be re-allocated. *owner = None; } } #[cfg(feature = "swap")] if owner.is_pid(_pid) { let phys_addr = self.allocation_offset_to_address(idx).unwrap(); if crate::arch::mem::page_is_lent(phys_addr as *mut u8) { // If the page is lent, reparent it to PID 1 so it will // get freed when it is returned. owner.reparent(PID::new(1).unwrap()); } else { // Mark this page as free, which allows it to be re-allocated. owner.update(None, None); } } } // release the extra allocs #[cfg(baremetal)] for (idx, owner) in (&mut *(&raw mut EXTRA_ALLOCATIONS)).iter_mut().enumerate() { if owner == &mut Some(_pid) { let phys_addr = self.allocation_offset_to_address_extra(idx).unwrap(); if crate::arch::mem::page_is_lent(phys_addr as *mut u8) { // If the page is lent, reparent it to PID 1 so it will // get freed when it is returned. *owner = PID::new(1); } else { // Mark this page as free, which allows it to be re-allocated. *owner = None; } } } } /// Adjust the flags on the given memory range. This allows for stripping flags from a memory /// range but does not allow adding flags. The memory range must exist, and the flags must be valid. pub fn update_memory_flags( &mut self, range: MemoryRange, flags: MemoryFlags, ) -> Result<(), xous_kernel::Error> { let virt = range.as_mut_ptr() as usize; let size = range.len(); if virt & (PAGE_SIZE - 1) != 0 { return Err(xous_kernel::Error::BadAlignment); } if size & (PAGE_SIZE - 1) != 0 { return Err(xous_kernel::Error::BadAlignment); } // Pre-check the range to ensure the new flags are valid for virt in (virt..(virt + size)).step_by(PAGE_SIZE) { let existing_flags = crate::arch::mem::page_flags(virt).ok_or(xous_kernel::Error::MemoryInUse)?; // If the new flags add to the range, return an error. if !(!existing_flags & flags).is_empty() { return Err(xous_kernel::Error::MemoryInUse); } } // Now that the flags are validated, perform the update. This is fine as long as // we're unicore. for virt in (virt..(virt + size)).step_by(PAGE_SIZE) { let existing_flags = crate::arch::mem::page_flags(virt).ok_or(xous_kernel::Error::MemoryInUse)?; // If the new flags add to the range, return an error. if !(!existing_flags & flags).is_empty() { return Err(xous_kernel::Error::MemoryInUse); } crate::arch::mem::update_page_flags(virt, flags)?; } Ok(()) } #[cfg(all(baremetal, target_arch = "riscv32", not(feature = "bao1x")))] pub fn check_for_duplicates(&self) { use crate::services::SystemServices; SystemServices::with(|system_services| { let current_pid = system_services.current_pid(); // Activate the debugging process and iterate through it, // noting down each active thread. for phys in (self.ram_start..self.ram_start + self.ram_size).step_by(PAGE_SIZE) { let mut owner = None; for pid in 1..crate::services::MAX_PROCESS_COUNT { let pid = PID::new(pid as u8).unwrap(); let Ok(process) = system_services.get_process(pid) else { continue; }; let Ok(_) = process.activate() else { continue; }; match MemoryMapping::current().phys_to_virt(phys) { Err(e) => { println!("!!! ERROR {:?} !!!", e); continue; } Ok(None) => continue, Ok(Some(virt)) => { let allocation_offset = (phys - self.ram_start) / PAGE_SIZE; let existing_owner = unsafe { &MEMORY_ALLOCATIONS[allocation_offset] }; #[cfg(feature = "swap")] let eo = &existing_owner.get_pid(); #[cfg(not(feature = "swap"))] let eo = existing_owner; if eo != &Some(pid) { let is_lent = { if let Some(existing_owner) = eo { system_services .get_process(*existing_owner) .unwrap() .activate() .unwrap(); let is_lent = if let Ok(Some(owned_address)) = MemoryMapping::current().phys_to_virt(phys) { crate::arch::mem::page_is_lent(owned_address as *mut u8) } else { false }; system_services.get_process(pid).unwrap().activate().unwrap(); is_lent } else { false } }; println!( "!!! 0x{:08x} is owned by {} ({}) but is mapped to {} ({}) -- {}", phys, eo.map(|v| v.get() as isize).unwrap_or(-1), eo.map(|v| system_services.process_name(v).unwrap_or("")) .unwrap_or(""), pid.get(), system_services.process_name(pid).unwrap_or(""), if is_lent { "page is lent" } else { "duplicate!" }, ); } if !crate::arch::mem::page_is_lent(virt as *mut u8) { if owner.is_none() { owner = Some((pid, virt)); } else { println!( "!!! DUPLICATE !!! Page {:08x} owned by both {} ({}) @ {:08x} and {} ({}) @ {:08x}", phys, owner.map(|v| v.0.get() as isize).unwrap_or(-1), owner .map(|v| system_services.process_name(v.0).unwrap_or("")) .unwrap_or(""), owner.map(|v| v.1).unwrap_or(0), pid.get(), system_services.process_name(pid).unwrap_or(""), virt, ); } } } } } } // Restore the previous PID system_services.get_process(current_pid).unwrap().activate().unwrap(); }) } } pub unsafe fn bzero(mut start: *mut T, end: *mut T) where T: Copy, { while start < end { // NOTE(volatile) to prevent this from being transformed into `memclr` core::ptr::write_volatile(start, core::mem::zeroed()); start = start.offset(1); } }