diff --git a/Cargo.lock b/Cargo.lock index dd4e66c24..f15c27e06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1416,6 +1416,7 @@ name = "modals" version = "0.1.0" dependencies = [ "gam", + "locales", "log", "log-server", "num-derive", diff --git a/services/gam/locales/i18n.json b/services/gam/locales/i18n.json index 523b6b64e..2770844de 100644 --- a/services/gam/locales/i18n.json +++ b/services/gam/locales/i18n.json @@ -5,6 +5,48 @@ "zh": "[ 确定 ]", "en-tts": "Accept and close radio box" }, + "radio.selection_tts": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "Set selection to: " + }, + "radio.select_and_close_tts": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "Close radio box with selection of " + }, + "radio.selected_tts": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "Selected: " + }, + "checkbox.select_and_close_tts": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "Close checkbox with these items selected " + }, + "checkbox.uncheck": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "Unchecked " + }, + "checkbox.check": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "Checked " + }, + "progress.increment": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "tick" + }, "notification.dismiss": { "en": "[ Press any key ]", "ja": "[ 何かのキーを押してください。]", diff --git a/services/gam/src/modal/checkboxes.rs b/services/gam/src/modal/checkboxes.rs index a38065178..b588f45da 100644 --- a/services/gam/src/modal/checkboxes.rs +++ b/services/gam/src/modal/checkboxes.rs @@ -6,6 +6,8 @@ use xous_ipc::Buffer; use core::fmt::Write; use locales::t; +#[cfg(feature="tts")] +use tts_frontend::TtsFrontend; #[derive(Debug)] pub struct CheckBoxes { @@ -14,15 +16,21 @@ pub struct CheckBoxes { pub action_opcode: u32, pub action_payload: CheckBoxPayload, pub select_index: i16, + #[cfg(feature = "tts")] + pub tts: TtsFrontend, } impl CheckBoxes { pub fn new(action_conn: xous::CID, action_opcode: u32) -> Self { + #[cfg(feature="tts")] + let tts = TtsFrontend::new(&xous_names::XousNames::new().unwrap()).unwrap(); CheckBoxes { items: Vec::new(), action_conn, action_opcode, action_payload: CheckBoxPayload::new(), select_index: 0, + #[cfg(feature="tts")] + tts, } } pub fn add_item(&mut self, new_item: ItemName) { @@ -62,6 +70,10 @@ impl ActionApi for CheckBoxes { for item in self.items.iter() { let cur_y = at_height + cur_line * modal.line_height; if cur_line == self.select_index { + #[cfg(feature="tts")] + { + self.tts.tts_simple(item.as_str()).unwrap(); + } // draw the cursor tv.text.clear(); tv.bounds_computed = None; @@ -103,6 +115,15 @@ impl ActionApi for CheckBoxes { )); write!(tv, "\u{25B6}").unwrap(); // right arrow emoji. use unicode numbers, because text editors do funny shit with emojis modal.gam.post_textview(&mut tv).expect("couldn't post tv"); + #[cfg(feature="tts")] + { + self.tts.tts_blocking(t!("checkbox.select_and_close_tts", xous::LANG)).unwrap(); + for item in self.action_payload.payload().iter() { + if let Some(name) = item { + self.tts.tts_blocking(name.as_str()).unwrap(); + } + } + } } // draw the "OK" line tv.text.clear(); @@ -141,10 +162,21 @@ impl ActionApi for CheckBoxes { let item_name = self.items[self.select_index as usize].as_str(); if self.action_payload.contains(item_name) { self.action_payload.remove(item_name); + #[cfg(feature="tts")] + { + self.tts.tts_blocking(t!("checkbox.uncheck", xous::LANG)).unwrap(); + self.tts.tts_blocking(item_name).unwrap(); + } } else { if !self.action_payload.add(item_name) { log::warn!("Limit of {} items that can be checked hit, consider increasing MAX_ITEMS in gam/src/modal.rs", MAX_ITEMS); log::warn!("The attempted item '{}' was not selected.", item_name); + } else { + #[cfg(feature="tts")] + { + self.tts.tts_blocking(t!("checkbox.check", xous::LANG)).unwrap(); + self.tts.tts_blocking(item_name).unwrap(); + } } } } else { // the OK button select diff --git a/services/gam/src/modal/radiobuttons.rs b/services/gam/src/modal/radiobuttons.rs index 602c5a993..81d16f1b0 100644 --- a/services/gam/src/modal/radiobuttons.rs +++ b/services/gam/src/modal/radiobuttons.rs @@ -6,6 +6,8 @@ use xous_ipc::Buffer; use core::fmt::Write; use locales::t; +#[cfg(feature="tts")] +use tts_frontend::TtsFrontend; #[derive(Debug)] pub struct RadioButtons { @@ -15,9 +17,13 @@ pub struct RadioButtons { pub action_payload: RadioButtonPayload, // the current "radio button" selection pub select_index: i16, // the current candidate to be selected pub is_password: bool, + #[cfg(feature = "tts")] + pub tts: TtsFrontend, } impl RadioButtons { pub fn new(action_conn: xous::CID, action_opcode: u32) -> Self { + #[cfg(feature="tts")] + let tts = TtsFrontend::new(&xous_names::XousNames::new().unwrap()).unwrap(); RadioButtons { items: Vec::new(), action_conn, @@ -25,6 +31,8 @@ impl RadioButtons { action_payload: RadioButtonPayload::new(""), select_index: 0, is_password: false, + #[cfg(feature="tts")] + tts, } } pub fn add_item(&mut self, new_item: ItemName) { @@ -77,6 +85,10 @@ impl ActionApi for RadioButtons { for item in self.items.iter() { let cur_y = at_height + cur_line * modal.line_height + modal.margin * 2; if cur_line == self.select_index { + #[cfg(feature="tts")] + { + self.tts.tts_simple(item.as_str()).unwrap(); + } // draw the cursor tv.text.clear(); tv.bounds_computed = None; @@ -118,6 +130,11 @@ impl ActionApi for RadioButtons { )); write!(tv, "\u{25B6}").unwrap(); // right arrow emoji. use unicode numbers, because text editors do funny shit with emojis modal.gam.post_textview(&mut tv).expect("couldn't post tv"); + #[cfg(feature="tts")] + { + self.tts.tts_blocking(t!("radio.select_and_close_tts", xous::LANG)).unwrap(); + self.tts.tts_blocking(self.action_payload.as_str()).unwrap(); + } } // draw the "OK" line tv.text.clear(); @@ -154,6 +171,11 @@ impl ActionApi for RadioButtons { '∴' | '\u{d}' => { if self.select_index < self.items.len() as i16 { self.action_payload = RadioButtonPayload::new(self.items[self.select_index as usize].as_str()); + #[cfg(feature="tts")] + { + self.tts.tts_blocking(t!("radio.selection_tts", xous::LANG)).unwrap(); + self.tts.tts_simple(self.items[self.select_index as usize].as_str()).unwrap(); + } } else { // the OK button select let buf = Buffer::into_buf(self.action_payload).expect("couldn't convert message to payload"); buf.send(self.action_conn, self.action_opcode).map(|_| ()).expect("couldn't send action message"); diff --git a/services/modals/Cargo.toml b/services/modals/Cargo.toml index dbf3f1bcd..43466a4ec 100644 --- a/services/modals/Cargo.toml +++ b/services/modals/Cargo.toml @@ -19,6 +19,7 @@ rkyv = {version = "0.4.3", features = ["const_generics"], default-features = fal gam = {path="../gam"} trng = {path="../trng"} tts-frontend = {path="../tts"} +locales = {path = "../../locales"} [target.'cfg(not(any(windows,unix)))'.dependencies] utralib = { path = "../../utralib"} @@ -26,4 +27,4 @@ utralib = { path = "../../utralib"} [features] ux_tests = [] tts = [] -default = [] +default = ["ux_tests"] diff --git a/services/modals/src/main.rs b/services/modals/src/main.rs index f0589bb75..1081f04d3 100644 --- a/services/modals/src/main.rs +++ b/services/modals/src/main.rs @@ -16,6 +16,10 @@ use std::thread; use gam::modal::*; #[cfg(feature="tts")] use tts_frontend::TtsFrontend; +#[cfg(feature="tts")] +use locales::t; +#[cfg(feature="tts")] +const TICK_INTERVAL: u64 = 2500; use num_traits::*; @@ -76,12 +80,15 @@ fn xmain() -> ! { #[cfg(feature="tts")] let tts = TtsFrontend::new(&xns).unwrap(); + #[cfg(feature="tts")] + let mut last_tick = tt.elapsed_ms(); let op = Arc::new(Mutex::new(RendererState::None)); // create a thread that just handles the redrawing requests let _redraw_handle = thread::spawn({ let op = Arc::clone(&op); move || { + let tt = ticktimer_server::Ticktimer::new().unwrap(); // build the core data structure here let text_action = TextEntry { is_password: false, @@ -178,7 +185,10 @@ fn xmain() -> ! { } fixed_items.clear(); #[cfg(feature="tts")] - tts.tts_simple(config.prompt.as_str().unwrap()).unwrap(); + { + tts.tts_blocking(t!("modals.radiobutton", xous::LANG)).unwrap(); + tts.tts_blocking(config.prompt.as_str().unwrap()).unwrap(); + } renderer_modal.modify( Some(ActionType::RadioButtons(radiobuttons)), Some(config.prompt.as_str().unwrap()), false, @@ -196,7 +206,11 @@ fn xmain() -> ! { } fixed_items.clear(); #[cfg(feature="tts")] - tts.tts_simple(config.prompt.as_str().unwrap()).unwrap(); + #[cfg(feature="tts")] + { + tts.tts_blocking(t!("modals.checkbox", xous::LANG)).unwrap(); + tts.tts_blocking(config.prompt.as_str().unwrap()).unwrap(); + } renderer_modal.modify( Some(ActionType::CheckBoxes(checkbox)), Some(config.prompt.as_str().unwrap()), false, @@ -238,11 +252,17 @@ fn xmain() -> ! { Some(RendererOp::UpdateProgress) => msg_scalar_unpack!(msg, current, _, _, _, { let new_percentage = compute_checked_percentage( current as u32, start_work, end_work); - log::trace!("percentage: {}, current: {}, start: {}, end: {}", new_percentage, current, start_work, end_work); - if new_percentage != last_percentage { + log::trace!("percentage: {}, current: {}, start: {}, end: {}", new_percentage, current, start_work, end_work); + if new_percentage != last_percentage { last_percentage = new_percentage; progress_action.set_state(last_percentage); - + #[cfg(feature="tts")] + { + if tt.elapsed_ms() - last_tick > TICK_INTERVAL { + tts.tts_blocking(t!("progress.increment", xous::LANG)).unwrap(); + last_tick = tt.elapsed_ms(); + } + } renderer_modal.modify( Some(ActionType::Slider(progress_action)), None, false, diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 98ed7d455..076eb1e20 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -468,6 +468,7 @@ fn xmain() -> ! { } }); // spawn a delayed mount command, shortly after boot. There's too much going on at boot, and it blocks other things from coming up. + /* #[cfg(not(any(windows, unix)))] // skip this step if running in hosted mode let _ = thread::spawn({ let my_cid = my_cid.clone(); @@ -478,7 +479,7 @@ fn xmain() -> ! { Message::new_blocking_scalar(Opcode::TryMount.to_usize().unwrap(), 0, 0, 0, 0) ).expect("couldn't send mount request"); } - }); + });*/ // main server loop let mut key_list = Vec::::new(); // storage for key lists let mut key_token: Option<[u32; 4]> = None; diff --git a/services/status/locales/i18n.json b/services/status/locales/i18n.json index 84dfb1794..f773642ce 100644 --- a/services/status/locales/i18n.json +++ b/services/status/locales/i18n.json @@ -13,10 +13,10 @@ }, "stats.uptime": { "translator-note": "This needs to be a very short string, 2 chars max. Trailing space is necessary for English due to proportional font.", - "ja": "稼働 ", + "ja": "稼働", "en": "Up ", - "zh": "运行 ", - "en-tts": "Uptime" + "zh": "运行", + "en-tts": "Up" }, "secnote.usb_unlock": { "en": " USB unlocked", diff --git a/services/tts/src/api.rs b/services/tts/src/api.rs index 36d18caf4..1084e992a 100644 --- a/services/tts/src/api.rs +++ b/services/tts/src/api.rs @@ -6,8 +6,6 @@ pub(crate) enum Opcode { TextToSpeech, /// Non-interruptable conversion of a string to audible speech. Blocks until the phrase is finished. TextToSpeechBlocking, - /// Callback for the codec crate - CodecCb, /// Stops audio playback immediately. Does not stop wave generation. CodecStop, /// Set words per minute diff --git a/services/tts/src/main.rs b/services/tts/src/main.rs index e5c6412fb..694ddec37 100644 --- a/services/tts/src/main.rs +++ b/services/tts/src/main.rs @@ -21,6 +21,11 @@ pub(crate) enum WaveOp { Return, Quit, } +#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)] +pub(crate) enum CallbackOp { + Callback, + Quit, +} #[xous::xous_main] fn xmain() -> ! { @@ -30,19 +35,12 @@ fn xmain() -> ! { let xns = xous_names::XousNames::new().unwrap(); let tts_sid = xns.register_name(api::SERVER_NAME_TTS, None).expect("can't register server"); - let tts_cid = xous::connect(tts_sid).unwrap(); + // let tts_cid = xous::connect(tts_sid).unwrap(); log::trace!("registered with NS -- {:?}", tts_sid); let tts_be = TtsBackend::new(&xns).unwrap(); let tt = ticktimer_server::Ticktimer::new().unwrap(); - let mut codec = codec::Codec::new(&xns).unwrap(); - codec.setup_8k_stream().expect("couldn't setup stream"); - tt.sleep_ms(50).unwrap(); - codec.set_speaker_volume(VolumeOps::Set, Some(0.0)).unwrap(); - codec.set_headphone_volume(VolumeOps::RestoreDefault, None).unwrap(); - codec.hook_frame_callback(Opcode::CodecCb.to_u32().unwrap(), tts_cid).unwrap(); - let mut frame_count = 0; let wav_sid = xous::create_server().unwrap(); let wav_cid = xous::connect(wav_sid).unwrap(); @@ -93,6 +91,83 @@ fn xmain() -> ! { } } }); + let cb_sid = xous::create_server().unwrap(); + let cb_cid = xous::connect(cb_sid).unwrap(); + let just_initiated = Arc::new(AtomicBool::new(false)); + std::thread::spawn({ + let wavbuf = wavbuf.clone(); + let synth_done = synth_done.clone(); + let just_initiated = just_initiated.clone(); + move || { + let mut codec = codec::Codec::new(&xous_names::XousNames::new().unwrap()).unwrap(); + let mut frame_count = 0; + loop { + let msg = xous::receive_message(cb_sid).unwrap(); + match FromPrimitive::from_usize(msg.body.id()) { + Some(CallbackOp::Callback) => msg_scalar_unpack!(msg, free_play, _available_rec, _, routing_id, { + if routing_id == codec::AUDIO_CB_ROUTING_ID { + let mut frames: FrameRing = FrameRing::new(); + let frames_to_push = if frames.writeable_count() < free_play { + frames.writeable_count() + } else { + free_play + }; + frame_count += frames_to_push as u32; + log::debug!("f{} p{}", frame_count, frames_to_push); + let mut locked_buf = wavbuf.lock().unwrap(); + if just_initiated.load(Ordering::SeqCst) { + // prevent stutter if the synth buffer isn't ready yet and we got an early fill request from the codec + if locked_buf.len() < codec::FIFO_DEPTH { + for _ in 0..frames_to_push { + let frame: [u32; codec::FIFO_DEPTH] = [ZERO_PCM as u32 | (ZERO_PCM as u32) << 16; codec::FIFO_DEPTH]; + frames.nq_frame(frame).unwrap(); + } + codec.swap_frames(&mut frames).unwrap(); + continue; + } else { + just_initiated.store(false, Ordering::SeqCst); + } + } + for _ in 0..frames_to_push { + let mut frame: [u32; codec::FIFO_DEPTH] = [ZERO_PCM as u32 | (ZERO_PCM as u32) << 16; codec::FIFO_DEPTH]; + if locked_buf.len() >= frame.len() || synth_done.load(Ordering::SeqCst) { + for sample in frame.iter_mut() { + let samp = locked_buf.pop_front().unwrap_or(ZERO_PCM); + let left = samp as u16; + let right = samp as u16; + *sample = right as u32 | (left as u32) << 16; + } + frames.nq_frame(frame).unwrap(); + } else { + log::trace!("ran out of frames during tts fill"); + break; + } + } + codec.swap_frames(&mut frames).unwrap(); + // detect if the buffer is empty and the synthesizer has indicated it's finished + if (locked_buf.len() == 0) && synth_done.load(Ordering::SeqCst) { + codec.pause().unwrap(); + } + } + }), + Some(CallbackOp::Quit) => { + xous::return_scalar(msg.sender, 1).unwrap(); + break; + }, + None => { + log::error!("couldn't convert opcode: {:?}", msg); + } + } + } + } + }); + let mut codec = codec::Codec::new(&xns).unwrap(); + codec.setup_8k_stream().expect("couldn't setup stream"); + tt.sleep_ms(50).unwrap(); + codec.set_speaker_volume(VolumeOps::Set, Some(0.0)).unwrap(); + codec.set_headphone_volume(VolumeOps::RestoreDefault, None).unwrap(); + codec.hook_frame_callback(CallbackOp::Callback.to_u32().unwrap(), cb_cid).unwrap(); + let mut wpm = DEFAULT_WPM; tts_be.tts_config( wav_sid.to_array(), @@ -100,82 +175,39 @@ fn xmain() -> ! { None, Some(wpm) ).unwrap(); - let mut just_initiated = false; loop { let msg = xous::receive_message(tts_sid).unwrap(); match FromPrimitive::from_usize(msg.body.id()) { Some(Opcode::TextToSpeech) => { let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) }; let msg = buffer.to_original::().unwrap(); - log::debug!("tts front end got string {}", msg.text.as_str().unwrap()); + log::info!("tts front end got string {}", msg.text.as_str().unwrap()); wavbuf.lock().unwrap().clear(); // this will truncate any buffered audio that is playing synth_done.store(false, Ordering::SeqCst); tts_be.tts_simple(msg.text.as_str().unwrap()).unwrap(); - just_initiated = true; + just_initiated.store(true, Ordering::SeqCst); log::debug!("resuming codec"); codec.resume().unwrap(); }, Some(Opcode::TextToSpeechBlocking) => { let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) }; let msg = buffer.to_original::().unwrap(); - log::debug!("tts blocking front end got string {}", msg.text.as_str().unwrap()); + log::info!("tts blocking front end got string {}", msg.text.as_str().unwrap()); wavbuf.lock().unwrap().clear(); // this will truncate any buffered audio that is playing synth_done.store(false, Ordering::SeqCst); tts_be.tts_simple(msg.text.as_str().unwrap()).unwrap(); - just_initiated = true; + just_initiated.store(true, Ordering::SeqCst); log::debug!("resuming codec (blocking)"); codec.resume().unwrap(); while !synth_done.load(Ordering::SeqCst) { + // this is done fairly "fast" because the synth buf fills quickly + tt.sleep_ms(WAIT_INTERVAL).unwrap(); + } + while codec.is_running().unwrap() { + // this actually waits until the playing is fully done tt.sleep_ms(WAIT_INTERVAL).unwrap(); } }, - Some(Opcode::CodecCb) => msg_scalar_unpack!(msg, free_play, _available_rec, _, routing_id, { - if routing_id == codec::AUDIO_CB_ROUTING_ID { - let mut frames: FrameRing = FrameRing::new(); - let frames_to_push = if frames.writeable_count() < free_play { - frames.writeable_count() - } else { - free_play - }; - frame_count += frames_to_push as u32; - - log::debug!("f{} p{}", frame_count, frames_to_push); - let mut locked_buf = wavbuf.lock().unwrap(); - if just_initiated { - // prevent stutter if the synth buffer isn't ready yet and we got an early fill request from the codec - if locked_buf.len() < codec::FIFO_DEPTH { - for _ in 0..frames_to_push { - let frame: [u32; codec::FIFO_DEPTH] = [ZERO_PCM as u32 | (ZERO_PCM as u32) << 16; codec::FIFO_DEPTH]; - frames.nq_frame(frame).unwrap(); - } - codec.swap_frames(&mut frames).unwrap(); - continue; - } else { - just_initiated = false; - } - } - for _ in 0..frames_to_push { - let mut frame: [u32; codec::FIFO_DEPTH] = [ZERO_PCM as u32 | (ZERO_PCM as u32) << 16; codec::FIFO_DEPTH]; - if locked_buf.len() >= frame.len() || synth_done.load(Ordering::SeqCst) { - for sample in frame.iter_mut() { - let samp = locked_buf.pop_front().unwrap_or(ZERO_PCM); - let left = samp as u16; - let right = samp as u16; - *sample = right as u32 | (left as u32) << 16; - } - frames.nq_frame(frame).unwrap(); - } else { - log::trace!("ran out of frames during tts fill"); - break; - } - } - codec.swap_frames(&mut frames).unwrap(); - // detect if the buffer is empty and the synthesizer has indicated it's finished - if (locked_buf.len() == 0) && synth_done.load(Ordering::SeqCst) { - codec.pause().unwrap(); - } - } - }), Some(Opcode::CodecStop) => { log::info!("stop called. Immediate stop and loss of audio data."); codec.abort().unwrap(); @@ -188,7 +220,11 @@ fn xmain() -> ! { send_message(wav_cid, Message::new_blocking_scalar(WaveOp::Quit.to_usize().unwrap(), 0, 0, 0, 0) ).expect("couldn't send quit to callback handler"); + send_message(cb_cid, + Message::new_blocking_scalar(WaveOp::Quit.to_usize().unwrap(), 0, 0, 0, 0) + ).expect("couldn't send quit to callback handler"); unsafe{xous::disconnect(wav_cid).ok()}; + unsafe{xous::disconnect(cb_cid).ok()}; log::warn!("Quit received, goodbye world!"); break; }, diff --git a/xous-rs/src/locale.rs b/xous-rs/src/locale.rs index ed67705bc..d18dfa847 100644 --- a/xous-rs/src/locale.rs +++ b/xous-rs/src/locale.rs @@ -1 +1 @@ -pub const LANG: &str = "en"; +pub const LANG: &str = "en-tts";