detect when the WF200 goes catatonic and reset it

This is not the fix I was hoping for for connection stability,
but unfortunately this is where we are at.

The problem is that at some point the WF200 throws a "host interface
error"; but the error codes show no specific error. After throwing
this error, it can respond to some basic commands (such as the one
asking what the current status and error codes are), but higher level
APIs seem to time out. There's no command to bring the WF200 stack
to a sane state (like a warm reset or buffer flush) as far as I
can see, so, the solution is to detect the issue and send a reset
to the WF200. This keeps the link up in the long term, but
you will experience drop-outs about once every 2-3 hours for
about 20 seconds while the system detects and fixes the problem.

I think for this one, more data will be needed from customer units
to see if this could be just a local network issue with my config
or a more pervasive problem.
This commit is contained in:
bunnie 2022-01-15 15:13:51 +08:00
parent ea8aa0befb
commit fbd0bc542f

View file

@ -14,7 +14,9 @@ pub(crate) enum ConnectionManagerOpcode {
Quit,
}
#[allow(dead_code)]
const POLL_INTERVAL_MS: usize = 10_151; // stagger slightly off of an integer-seconds interval to even out loads
const POLL_INTERVAL_MS: usize = 20_151; // stagger slightly off of an integer-seconds interval to even out loads
#[allow(dead_code)]
const BOOT_POLL_INTERVAL_MS: usize = 5_758; // a slightly faster poll during boot so we acquire wifi faster once PDDB is mounted
pub(crate) fn connection_manager(sid: xous::SID, activity_interval: Arc<AtomicU32>) {
let tt = ticktimer_server::Ticktimer::new().unwrap();
@ -36,25 +38,34 @@ pub(crate) fn connection_manager(sid: xous::SID, activity_interval: Arc<AtomicU3
}
let mut run = rev_ok;
send_message(self_cid, Message::new_scalar(ConnectionManagerOpcode::Poll.to_usize().unwrap(), POLL_INTERVAL_MS, 0, 0, 0)).expect("couldn't kick off next poll");
let mut mounted = false;
let mut current_interval = BOOT_POLL_INTERVAL_MS;
send_message(self_cid, Message::new_scalar(ConnectionManagerOpcode::Poll.to_usize().unwrap(), current_interval, 0, 0, 0)).expect("couldn't kick off next poll");
loop {
let msg = xous::receive_message(sid).unwrap();
log::debug!("got msg: {:?}", msg);
match FromPrimitive::from_usize(msg.body.id()) {
Some(ConnectionManagerOpcode::Run) => msg_scalar_unpack!(msg, poll_interval, _, _, _, {
Some(ConnectionManagerOpcode::Run) => msg_scalar_unpack!(msg, _, _, _, _, {
if !run {
run = true;
send_message(self_cid, Message::new_scalar(ConnectionManagerOpcode::Poll.to_usize().unwrap(), poll_interval, 0, 0, 0)).expect("couldn't kick off next poll");
send_message(self_cid, Message::new_scalar(ConnectionManagerOpcode::Poll.to_usize().unwrap(), 0, 0, 0, 0)).expect("couldn't kick off next poll");
}
}),
Some(ConnectionManagerOpcode::Poll) => msg_scalar_unpack!(msg, next_poll, _, _, _, {
if activity_interval.fetch_add(POLL_INTERVAL_MS as u32, Ordering::SeqCst) > POLL_INTERVAL_MS as u32 {
Some(ConnectionManagerOpcode::Poll) => msg_scalar_unpack!(msg, _, _, _, _, {
if activity_interval.fetch_add(current_interval as u32, Ordering::SeqCst) > current_interval as u32 {
log::info!("wlan activity interval timeout");
// if the pddb isn't mounted, don't even bother checking -- we can't connect until we have a place to get keys
if pddb.is_mounted() && rev_ok {
mounted = true;
// the status check code is going to get refactored, so this is a "bare minimum" check
let status = com.wlan_status().unwrap();
let config = netmgr.get_ipv4_config();
if status.link_state == com_rs_ref::LinkState::WFXError {
com.wlan_leave().expect("couldn't issue leave command"); // this may not be received by the wf200, *but* it will also re-init the DHCP stack on the EC
netmgr.reset(); // this will clear our internal net state
// the wfx chip is wedged. kick it. This call has a built-in 2 second delay.
com.wifi_reset().expect("couldn't reset the wf200 chip");
}
let needs_reconnect =
if status.link_state == com_rs_ref::LinkState::Connected {
if let Some(config) = config { // check that the EC's view of the world is synchronized with our view
@ -102,8 +113,13 @@ pub(crate) fn connection_manager(sid: xous::SID, activity_interval: Arc<AtomicU3
}
}
if run {
tt.sleep_ms(next_poll).unwrap();
send_message(self_cid, Message::new_scalar(ConnectionManagerOpcode::Poll.to_usize().unwrap(), next_poll, 0, 0, 0)).expect("couldn't kick off next poll");
tt.sleep_ms(current_interval).unwrap();
if !mounted {
current_interval = BOOT_POLL_INTERVAL_MS;
} else {
current_interval = POLL_INTERVAL_MS;
}
send_message(self_cid, Message::new_scalar(ConnectionManagerOpcode::Poll.to_usize().unwrap(), 0, 0, 0, 0)).expect("couldn't kick off next poll");
}
}),
// stop is blocking because we need to ensure the previous poll has finished before moving on, otherwise,