From 94cbbddaa6494ca9b155745c81e9f594b29ca76a Mon Sep 17 00:00:00 2001 From: bunnie Date: Thu, 12 May 2022 17:49:27 +0800 Subject: [PATCH] refactor follow-up scheduling vs smoltcp stack smoltcp requires some polling to be done in the future for it to work with low latency. It returns a time in the future for the next poll. This refactors the polling algorithm to: 1. use a fixed thread pool. This burns a little extra memory in exchange for not having to allocate/de-allocate threads every time a follow-up is needed. 2. use a short/long scheduler, where short delays (shorter than the scheduler quantum) are grouped together, and only a single follow-up is fired off within that time range if needed. Longer delays have a larger window (roughly +/- 100ms) of grouping, so fewer such events are also required in the distant future. Empirically the delays seem to fall into two groups, either "short" (less than 20ms) or "long" (around 700ms). This improved scheduler uses fewer threads and generates less messages, so instead of 16 threads and encountering exhaustion we have 9 and have yet to see more than 7 used even under the most strenuous tests (multiple while-true curl loops + fountain test + ping simultaneously). Some helpful debug is also printed now when scheduler runs out of slots, so we can see the distribution of pending delays. --- services/net/src/main.rs | 203 ++++++++++++++++++++++++++++----------- 1 file changed, 147 insertions(+), 56 deletions(-) diff --git a/services/net/src/main.rs b/services/net/src/main.rs index c4bbdae42..1f01ae7e7 100644 --- a/services/net/src/main.rs +++ b/services/net/src/main.rs @@ -38,11 +38,13 @@ use smoltcp::socket::{ }; use smoltcp::iface::SocketHandle; use smoltcp::time::{Duration, Instant}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; +use std::sync::mpsc::{channel, Sender}; use std::thread; +use std::cmp::Ordering as CmpOrdering; const PING_DEFAULT_TIMEOUT_MS: u32 = 10_000; -const MAX_DELAY_THREADS: u32 = 16; // limit the number of concurrent delay threads. Typically we have 1-2 running at any time, but DoS conditions could lead to many more. +const MAX_DELAY_THREADS: u32 = 9; const PING_IDENT: u16 = 0x22b; fn set_ipv4_addr(iface: &mut Interface<'_, DeviceT>, cidr: Ipv4Cidr) @@ -103,6 +105,32 @@ pub struct UdpStdState { pub expiry: Option, } +pub struct Wakeup { + pub tx_index: usize, + pub time: u64, +} +impl Ord for Wakeup { + fn cmp(&self, other: &Self) -> CmpOrdering { + self.time.cmp(&other.time) + } +} +impl PartialOrd for Wakeup { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl PartialEq for Wakeup { + fn eq(&self, other: &Self) -> bool { + self.time == other.time + } +} +impl Eq for Wakeup {} +pub struct WorkerState { + pub tx: Sender::, + pub is_busy: bool, + pub time_replica: u64, // this is just to help with debugging, nothing else +} + fn set_com_ints(com_int_list: &mut Vec) { com_int_list.clear(); com_int_list.push(ComIntSources::WlanIpConfigUpdate); @@ -182,7 +210,6 @@ fn xmain() -> ! { com_int_list.clear(); com.ints_get_active(&mut com_int_list).ok(); log::debug!("COM pending interrupts after enabling: {:?}", com_int_list); - let delay_threads = Arc::new(AtomicU32::new(0)); let mut net_config: Option = None; // ------------- libstd variant ----------- @@ -255,8 +282,52 @@ fn xmain() -> ! { let mut dns_ipv6_hook = XousScalarEndpoint::new(); let mut dns_allclear_hook = XousScalarEndpoint::new(); - // wakeup polling management - this is not reset on connection reset, because the stale timers are still running and there is no way to reset them mid-run - let wakeup_time: Arc::>> = Arc::new(Mutex::new(BTreeSet::new())); + // wakeup polling management - kick off worker threads to wake up a poll in the future for certain rx events required by smoltcp + // this is not reset on connection reset, because the stale timers are still + // running and there is no way to reset them mid-run. + let mut wakeup_time: BTreeSet = BTreeSet::new(); + let mut wakeup_workers = Vec::::new(); + for _ in 0..MAX_DELAY_THREADS { + let (tx, rx) = channel(); + wakeup_workers.push(WorkerState { + tx, + is_busy: false, + time_replica: 0, + }); + thread::spawn({ + let parent_conn = net_conn.clone(); + move || { + let tt = ticktimer_server::Ticktimer::new().unwrap(); + loop { + let target = rx.recv().unwrap(); + let now = tt.elapsed_ms(); + // arbitrary time can pass between the target issuance and receive + let wakeup = if now >= target { + 0 + } else { + target - now + }; + if wakeup > 100_000 { + log::warn!("long wakeup detected: {}", wakeup); + } + if wakeup != 0 { // only issue the sleep if the wakeup time is non-zero + tt.sleep_ms(wakeup as usize).unwrap(); + } + xous::try_send_message( + parent_conn, + Message::new_scalar( + Opcode::NetPump.to_usize().unwrap(), + (target >> 32) as usize, + (target & 0xffff_ffff) as usize, + 0, + 0, + ), + ) + .ok(); + } + } + }); + } log::trace!("ready to accept requests"); // register a suspend/resume listener @@ -1068,7 +1139,25 @@ fn xmain() -> ! { } } } - Some(Opcode::NetPump) => { + Some(Opcode::NetPump) => msg_scalar_unpack!(msg, wup_hi, wup_lo, _, _, { + // assume: if wup_hi/wup_lo == 0, then the wakeup is from a non-timer thread + let wakeup: u64 = (wup_hi as u64) << 32 | (wup_lo as u64); + if wakeup != 0 { + if let Some(wup) = wakeup_time.take(& Wakeup{tx_index: 0, time: wakeup}) { + wakeup_workers[wup.tx_index].is_busy = false; + } else { + log::warn!("wakeup time {} was not in the wakeup queue!", wakeup); + } + if wakeup_time.len() == 0 { + for (index, worker) in wakeup_workers.iter().enumerate() { + if worker.is_busy { + log::warn!("stranded worker at {}", index) + } + } + } + log::debug!("pending wakeups: {}", wakeup_time.len()); + } + let now = timer.elapsed_ms(); let timestamp = Instant::from_millis(now as i64); match iface.poll(timestamp) { @@ -1334,7 +1423,6 @@ fn xmain() -> ! { .recv() .expect("couldn't receive on socket despite asserting availability"); log::trace!("icmp payload: {:x?}", payload); - let now = timer.elapsed_ms(); for (connection, waiting_queue) in ping_destinations.iter_mut() { let remote_addr = connection.remote; @@ -1505,7 +1593,6 @@ fn xmain() -> ! { // this block handles ICMP retirement; it runs everytime we pump the block log::debug!("pump: icmp retirement"); { - let now = timer.elapsed_ms(); // notify the callback to drop its connection, because the queue is now empty // do this before we clear the queue, because we want the Drop message to come on the iteration // *after* the queue is empty. @@ -1581,57 +1668,61 @@ fn xmain() -> ! { // establish our next check-up interval log::debug!("pump: checkup"); - if delay_threads.load(Ordering::SeqCst) < MAX_DELAY_THREADS { - let now = timer.elapsed_ms(); - let timestamp = Instant::from_millis(now as i64); - if let Some(delay) = iface.poll_delay(timestamp) { - const RANGE_MS: u64 = 5; - let wakeup = now + delay.total_millis(); - let no_wakeup_scheduled = wakeup_time.lock().unwrap().range(wakeup - RANGE_MS..wakeup + RANGE_MS).count() == 0; - if no_wakeup_scheduled { - wakeup_time.lock().unwrap().insert(wakeup); - let prev_count = delay_threads.fetch_add(1, Ordering::SeqCst); - log::trace!( - "spawning wakeup at {}ms. New total threads: {}", - wakeup, - prev_count + 1 - ); - thread::spawn({ - let parent_conn = net_conn.clone(); - let delay_threads = delay_threads.clone(); - let wakeup = wakeup.clone(); - let now = now.clone(); - let wakeup_time = wakeup_time.clone(); - move || { - let tt = ticktimer_server::Ticktimer::new().unwrap(); - tt.sleep_ms((wakeup - now) as usize).unwrap(); - if !wakeup_time.lock().unwrap().remove(&wakeup) { - log::warn!("time {} was not in wakeup queue!", wakeup); - } - xous::try_send_message( - parent_conn, - Message::new_scalar( - Opcode::NetPump.to_usize().unwrap(), - 0, - 0, - 0, - 0, - ), - ) - .ok(); - let prev_count = delay_threads.fetch_sub(1, Ordering::SeqCst); - log::trace!( - "terminating checkup thread. New total threads: {}", - prev_count - 1 - ); - } - }); + let timestamp = Instant::from_millis(now as i64); + if let Some(delay) = iface.poll_delay(timestamp) { + const RANGE_MS: u64 = 100; // this will define the responsivity to "future" events that maybe 100's of ms away + let mut wakeup_needed = false; + // round all short delays to within the next quantum + if (delay.total_millis() as u32) < xous::BASE_QUANTA_MS { + if wakeup_time.range( + .. + Wakeup { tx_index: 0, time: now + xous::BASE_QUANTA_MS as u64 } + ).count() == 0 { + wakeup_needed = true; + } + } + let wakeup = now + delay.total_millis(); + // now check for longer delays over a longer window + if wakeup_time.range( + // "round up" to now if the window goes earlier than now + Wakeup { tx_index: 0, time: (wakeup - RANGE_MS).max(now) } + .. + Wakeup { tx_index: 0, time: wakeup + RANGE_MS } + ).count() == 0 { + wakeup_needed = true; + } + if wakeup_needed { + // find a free worker. this is an O(n) search right now, but generally (n) is small + let mut found = false; + for (index, worker) in wakeup_workers.iter_mut().enumerate() { + if !worker.is_busy { + worker.is_busy = true; + worker.time_replica = wakeup; + match worker.tx.send(wakeup) { + Err(e) => log::error!("{}: Couldn't send wakeup time to worker {}: {}", e, index, wakeup), + _ => {} + }; + wakeup_time.insert(Wakeup { + tx_index: index, + time: wakeup, + }); + found = true; + break; + } + } + if !found { + log::warn!("All wakeup threads occupied, punting on wakeup at {}ms in the future", delay.total_millis()); + let mut dbg = String::from("Now pending: "); + for (index, worker) in wakeup_workers.iter().enumerate() { + if worker.is_busy { + dbg.push_str(&format!("{}[{}], ", index, worker.time_replica as i64 - now as i64)); + } + } + log::info!("{}", dbg); } } - } else { - log::info!("Thread pool exhausted; deferring creation of new poll helpers"); } - } + }), Some(Opcode::GetIpv4Config) => { let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap())