integrate TTS into more dialog boxes and modals

also tighten up the `blocking` method for TTS for prompts that
absolutely should not be truncated or overriden.
This commit is contained in:
bunnie 2022-03-10 23:40:35 +08:00
parent 2b33371a36
commit e8b13fcb44
11 changed files with 226 additions and 73 deletions

1
Cargo.lock generated
View file

@ -1416,6 +1416,7 @@ name = "modals"
version = "0.1.0"
dependencies = [
"gam",
"locales",
"log",
"log-server",
"num-derive",

View file

@ -5,6 +5,48 @@
"zh": "[ 确定 ]",
"en-tts": "Accept and close radio box"
},
"radio.selection_tts": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "Set selection to: "
},
"radio.select_and_close_tts": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "Close radio box with selection of "
},
"radio.selected_tts": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "Selected: "
},
"checkbox.select_and_close_tts": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "Close checkbox with these items selected "
},
"checkbox.uncheck": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "Unchecked "
},
"checkbox.check": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "Checked "
},
"progress.increment": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "tick"
},
"notification.dismiss": {
"en": "[ Press any key ]",
"ja": "[ 何かのキーを押してください。]",

View file

@ -6,6 +6,8 @@ use xous_ipc::Buffer;
use core::fmt::Write;
use locales::t;
#[cfg(feature="tts")]
use tts_frontend::TtsFrontend;
#[derive(Debug)]
pub struct CheckBoxes {
@ -14,15 +16,21 @@ pub struct CheckBoxes {
pub action_opcode: u32,
pub action_payload: CheckBoxPayload,
pub select_index: i16,
#[cfg(feature = "tts")]
pub tts: TtsFrontend,
}
impl CheckBoxes {
pub fn new(action_conn: xous::CID, action_opcode: u32) -> Self {
#[cfg(feature="tts")]
let tts = TtsFrontend::new(&xous_names::XousNames::new().unwrap()).unwrap();
CheckBoxes {
items: Vec::new(),
action_conn,
action_opcode,
action_payload: CheckBoxPayload::new(),
select_index: 0,
#[cfg(feature="tts")]
tts,
}
}
pub fn add_item(&mut self, new_item: ItemName) {
@ -62,6 +70,10 @@ impl ActionApi for CheckBoxes {
for item in self.items.iter() {
let cur_y = at_height + cur_line * modal.line_height;
if cur_line == self.select_index {
#[cfg(feature="tts")]
{
self.tts.tts_simple(item.as_str()).unwrap();
}
// draw the cursor
tv.text.clear();
tv.bounds_computed = None;
@ -103,6 +115,15 @@ impl ActionApi for CheckBoxes {
));
write!(tv, "\u{25B6}").unwrap(); // right arrow emoji. use unicode numbers, because text editors do funny shit with emojis
modal.gam.post_textview(&mut tv).expect("couldn't post tv");
#[cfg(feature="tts")]
{
self.tts.tts_blocking(t!("checkbox.select_and_close_tts", xous::LANG)).unwrap();
for item in self.action_payload.payload().iter() {
if let Some(name) = item {
self.tts.tts_blocking(name.as_str()).unwrap();
}
}
}
}
// draw the "OK" line
tv.text.clear();
@ -141,10 +162,21 @@ impl ActionApi for CheckBoxes {
let item_name = self.items[self.select_index as usize].as_str();
if self.action_payload.contains(item_name) {
self.action_payload.remove(item_name);
#[cfg(feature="tts")]
{
self.tts.tts_blocking(t!("checkbox.uncheck", xous::LANG)).unwrap();
self.tts.tts_blocking(item_name).unwrap();
}
} else {
if !self.action_payload.add(item_name) {
log::warn!("Limit of {} items that can be checked hit, consider increasing MAX_ITEMS in gam/src/modal.rs", MAX_ITEMS);
log::warn!("The attempted item '{}' was not selected.", item_name);
} else {
#[cfg(feature="tts")]
{
self.tts.tts_blocking(t!("checkbox.check", xous::LANG)).unwrap();
self.tts.tts_blocking(item_name).unwrap();
}
}
}
} else { // the OK button select

View file

@ -6,6 +6,8 @@ use xous_ipc::Buffer;
use core::fmt::Write;
use locales::t;
#[cfg(feature="tts")]
use tts_frontend::TtsFrontend;
#[derive(Debug)]
pub struct RadioButtons {
@ -15,9 +17,13 @@ pub struct RadioButtons {
pub action_payload: RadioButtonPayload, // the current "radio button" selection
pub select_index: i16, // the current candidate to be selected
pub is_password: bool,
#[cfg(feature = "tts")]
pub tts: TtsFrontend,
}
impl RadioButtons {
pub fn new(action_conn: xous::CID, action_opcode: u32) -> Self {
#[cfg(feature="tts")]
let tts = TtsFrontend::new(&xous_names::XousNames::new().unwrap()).unwrap();
RadioButtons {
items: Vec::new(),
action_conn,
@ -25,6 +31,8 @@ impl RadioButtons {
action_payload: RadioButtonPayload::new(""),
select_index: 0,
is_password: false,
#[cfg(feature="tts")]
tts,
}
}
pub fn add_item(&mut self, new_item: ItemName) {
@ -77,6 +85,10 @@ impl ActionApi for RadioButtons {
for item in self.items.iter() {
let cur_y = at_height + cur_line * modal.line_height + modal.margin * 2;
if cur_line == self.select_index {
#[cfg(feature="tts")]
{
self.tts.tts_simple(item.as_str()).unwrap();
}
// draw the cursor
tv.text.clear();
tv.bounds_computed = None;
@ -118,6 +130,11 @@ impl ActionApi for RadioButtons {
));
write!(tv, "\u{25B6}").unwrap(); // right arrow emoji. use unicode numbers, because text editors do funny shit with emojis
modal.gam.post_textview(&mut tv).expect("couldn't post tv");
#[cfg(feature="tts")]
{
self.tts.tts_blocking(t!("radio.select_and_close_tts", xous::LANG)).unwrap();
self.tts.tts_blocking(self.action_payload.as_str()).unwrap();
}
}
// draw the "OK" line
tv.text.clear();
@ -154,6 +171,11 @@ impl ActionApi for RadioButtons {
'∴' | '\u{d}' => {
if self.select_index < self.items.len() as i16 {
self.action_payload = RadioButtonPayload::new(self.items[self.select_index as usize].as_str());
#[cfg(feature="tts")]
{
self.tts.tts_blocking(t!("radio.selection_tts", xous::LANG)).unwrap();
self.tts.tts_simple(self.items[self.select_index as usize].as_str()).unwrap();
}
} else { // the OK button select
let buf = Buffer::into_buf(self.action_payload).expect("couldn't convert message to payload");
buf.send(self.action_conn, self.action_opcode).map(|_| ()).expect("couldn't send action message");

View file

@ -19,6 +19,7 @@ rkyv = {version = "0.4.3", features = ["const_generics"], default-features = fal
gam = {path="../gam"}
trng = {path="../trng"}
tts-frontend = {path="../tts"}
locales = {path = "../../locales"}
[target.'cfg(not(any(windows,unix)))'.dependencies]
utralib = { path = "../../utralib"}
@ -26,4 +27,4 @@ utralib = { path = "../../utralib"}
[features]
ux_tests = []
tts = []
default = []
default = ["ux_tests"]

View file

@ -16,6 +16,10 @@ use std::thread;
use gam::modal::*;
#[cfg(feature="tts")]
use tts_frontend::TtsFrontend;
#[cfg(feature="tts")]
use locales::t;
#[cfg(feature="tts")]
const TICK_INTERVAL: u64 = 2500;
use num_traits::*;
@ -76,12 +80,15 @@ fn xmain() -> ! {
#[cfg(feature="tts")]
let tts = TtsFrontend::new(&xns).unwrap();
#[cfg(feature="tts")]
let mut last_tick = tt.elapsed_ms();
let op = Arc::new(Mutex::new(RendererState::None));
// create a thread that just handles the redrawing requests
let _redraw_handle = thread::spawn({
let op = Arc::clone(&op);
move || {
let tt = ticktimer_server::Ticktimer::new().unwrap();
// build the core data structure here
let text_action = TextEntry {
is_password: false,
@ -178,7 +185,10 @@ fn xmain() -> ! {
}
fixed_items.clear();
#[cfg(feature="tts")]
tts.tts_simple(config.prompt.as_str().unwrap()).unwrap();
{
tts.tts_blocking(t!("modals.radiobutton", xous::LANG)).unwrap();
tts.tts_blocking(config.prompt.as_str().unwrap()).unwrap();
}
renderer_modal.modify(
Some(ActionType::RadioButtons(radiobuttons)),
Some(config.prompt.as_str().unwrap()), false,
@ -196,7 +206,11 @@ fn xmain() -> ! {
}
fixed_items.clear();
#[cfg(feature="tts")]
tts.tts_simple(config.prompt.as_str().unwrap()).unwrap();
#[cfg(feature="tts")]
{
tts.tts_blocking(t!("modals.checkbox", xous::LANG)).unwrap();
tts.tts_blocking(config.prompt.as_str().unwrap()).unwrap();
}
renderer_modal.modify(
Some(ActionType::CheckBoxes(checkbox)),
Some(config.prompt.as_str().unwrap()), false,
@ -238,11 +252,17 @@ fn xmain() -> ! {
Some(RendererOp::UpdateProgress) => msg_scalar_unpack!(msg, current, _, _, _, {
let new_percentage = compute_checked_percentage(
current as u32, start_work, end_work);
log::trace!("percentage: {}, current: {}, start: {}, end: {}", new_percentage, current, start_work, end_work);
if new_percentage != last_percentage {
log::trace!("percentage: {}, current: {}, start: {}, end: {}", new_percentage, current, start_work, end_work);
if new_percentage != last_percentage {
last_percentage = new_percentage;
progress_action.set_state(last_percentage);
#[cfg(feature="tts")]
{
if tt.elapsed_ms() - last_tick > TICK_INTERVAL {
tts.tts_blocking(t!("progress.increment", xous::LANG)).unwrap();
last_tick = tt.elapsed_ms();
}
}
renderer_modal.modify(
Some(ActionType::Slider(progress_action)),
None, false,

View file

@ -468,6 +468,7 @@ fn xmain() -> ! {
}
});
// spawn a delayed mount command, shortly after boot. There's too much going on at boot, and it blocks other things from coming up.
/*
#[cfg(not(any(windows, unix)))] // skip this step if running in hosted mode
let _ = thread::spawn({
let my_cid = my_cid.clone();
@ -478,7 +479,7 @@ fn xmain() -> ! {
Message::new_blocking_scalar(Opcode::TryMount.to_usize().unwrap(), 0, 0, 0, 0)
).expect("couldn't send mount request");
}
});
});*/
// main server loop
let mut key_list = Vec::<String>::new(); // storage for key lists
let mut key_token: Option<[u32; 4]> = None;

View file

@ -13,10 +13,10 @@
},
"stats.uptime": {
"translator-note": "This needs to be a very short string, 2 chars max. Trailing space is necessary for English due to proportional font.",
"ja": "稼働 ",
"ja": "稼働",
"en": "Up ",
"zh": "运行 ",
"en-tts": "Uptime"
"zh": "运行",
"en-tts": "Up"
},
"secnote.usb_unlock": {
"en": " USB unlocked",

View file

@ -6,8 +6,6 @@ pub(crate) enum Opcode {
TextToSpeech,
/// Non-interruptable conversion of a string to audible speech. Blocks until the phrase is finished.
TextToSpeechBlocking,
/// Callback for the codec crate
CodecCb,
/// Stops audio playback immediately. Does not stop wave generation.
CodecStop,
/// Set words per minute

View file

@ -21,6 +21,11 @@ pub(crate) enum WaveOp {
Return,
Quit,
}
#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)]
pub(crate) enum CallbackOp {
Callback,
Quit,
}
#[xous::xous_main]
fn xmain() -> ! {
@ -30,19 +35,12 @@ fn xmain() -> ! {
let xns = xous_names::XousNames::new().unwrap();
let tts_sid = xns.register_name(api::SERVER_NAME_TTS, None).expect("can't register server");
let tts_cid = xous::connect(tts_sid).unwrap();
// let tts_cid = xous::connect(tts_sid).unwrap();
log::trace!("registered with NS -- {:?}", tts_sid);
let tts_be = TtsBackend::new(&xns).unwrap();
let tt = ticktimer_server::Ticktimer::new().unwrap();
let mut codec = codec::Codec::new(&xns).unwrap();
codec.setup_8k_stream().expect("couldn't setup stream");
tt.sleep_ms(50).unwrap();
codec.set_speaker_volume(VolumeOps::Set, Some(0.0)).unwrap();
codec.set_headphone_volume(VolumeOps::RestoreDefault, None).unwrap();
codec.hook_frame_callback(Opcode::CodecCb.to_u32().unwrap(), tts_cid).unwrap();
let mut frame_count = 0;
let wav_sid = xous::create_server().unwrap();
let wav_cid = xous::connect(wav_sid).unwrap();
@ -93,6 +91,83 @@ fn xmain() -> ! {
}
}
});
let cb_sid = xous::create_server().unwrap();
let cb_cid = xous::connect(cb_sid).unwrap();
let just_initiated = Arc::new(AtomicBool::new(false));
std::thread::spawn({
let wavbuf = wavbuf.clone();
let synth_done = synth_done.clone();
let just_initiated = just_initiated.clone();
move || {
let mut codec = codec::Codec::new(&xous_names::XousNames::new().unwrap()).unwrap();
let mut frame_count = 0;
loop {
let msg = xous::receive_message(cb_sid).unwrap();
match FromPrimitive::from_usize(msg.body.id()) {
Some(CallbackOp::Callback) => msg_scalar_unpack!(msg, free_play, _available_rec, _, routing_id, {
if routing_id == codec::AUDIO_CB_ROUTING_ID {
let mut frames: FrameRing = FrameRing::new();
let frames_to_push = if frames.writeable_count() < free_play {
frames.writeable_count()
} else {
free_play
};
frame_count += frames_to_push as u32;
log::debug!("f{} p{}", frame_count, frames_to_push);
let mut locked_buf = wavbuf.lock().unwrap();
if just_initiated.load(Ordering::SeqCst) {
// prevent stutter if the synth buffer isn't ready yet and we got an early fill request from the codec
if locked_buf.len() < codec::FIFO_DEPTH {
for _ in 0..frames_to_push {
let frame: [u32; codec::FIFO_DEPTH] = [ZERO_PCM as u32 | (ZERO_PCM as u32) << 16; codec::FIFO_DEPTH];
frames.nq_frame(frame).unwrap();
}
codec.swap_frames(&mut frames).unwrap();
continue;
} else {
just_initiated.store(false, Ordering::SeqCst);
}
}
for _ in 0..frames_to_push {
let mut frame: [u32; codec::FIFO_DEPTH] = [ZERO_PCM as u32 | (ZERO_PCM as u32) << 16; codec::FIFO_DEPTH];
if locked_buf.len() >= frame.len() || synth_done.load(Ordering::SeqCst) {
for sample in frame.iter_mut() {
let samp = locked_buf.pop_front().unwrap_or(ZERO_PCM);
let left = samp as u16;
let right = samp as u16;
*sample = right as u32 | (left as u32) << 16;
}
frames.nq_frame(frame).unwrap();
} else {
log::trace!("ran out of frames during tts fill");
break;
}
}
codec.swap_frames(&mut frames).unwrap();
// detect if the buffer is empty and the synthesizer has indicated it's finished
if (locked_buf.len() == 0) && synth_done.load(Ordering::SeqCst) {
codec.pause().unwrap();
}
}
}),
Some(CallbackOp::Quit) => {
xous::return_scalar(msg.sender, 1).unwrap();
break;
},
None => {
log::error!("couldn't convert opcode: {:?}", msg);
}
}
}
}
});
let mut codec = codec::Codec::new(&xns).unwrap();
codec.setup_8k_stream().expect("couldn't setup stream");
tt.sleep_ms(50).unwrap();
codec.set_speaker_volume(VolumeOps::Set, Some(0.0)).unwrap();
codec.set_headphone_volume(VolumeOps::RestoreDefault, None).unwrap();
codec.hook_frame_callback(CallbackOp::Callback.to_u32().unwrap(), cb_cid).unwrap();
let mut wpm = DEFAULT_WPM;
tts_be.tts_config(
wav_sid.to_array(),
@ -100,82 +175,39 @@ fn xmain() -> ! {
None,
Some(wpm)
).unwrap();
let mut just_initiated = false;
loop {
let msg = xous::receive_message(tts_sid).unwrap();
match FromPrimitive::from_usize(msg.body.id()) {
Some(Opcode::TextToSpeech) => {
let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) };
let msg = buffer.to_original::<TtsFrontendMsg, _>().unwrap();
log::debug!("tts front end got string {}", msg.text.as_str().unwrap());
log::info!("tts front end got string {}", msg.text.as_str().unwrap());
wavbuf.lock().unwrap().clear(); // this will truncate any buffered audio that is playing
synth_done.store(false, Ordering::SeqCst);
tts_be.tts_simple(msg.text.as_str().unwrap()).unwrap();
just_initiated = true;
just_initiated.store(true, Ordering::SeqCst);
log::debug!("resuming codec");
codec.resume().unwrap();
},
Some(Opcode::TextToSpeechBlocking) => {
let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) };
let msg = buffer.to_original::<TtsFrontendMsg, _>().unwrap();
log::debug!("tts blocking front end got string {}", msg.text.as_str().unwrap());
log::info!("tts blocking front end got string {}", msg.text.as_str().unwrap());
wavbuf.lock().unwrap().clear(); // this will truncate any buffered audio that is playing
synth_done.store(false, Ordering::SeqCst);
tts_be.tts_simple(msg.text.as_str().unwrap()).unwrap();
just_initiated = true;
just_initiated.store(true, Ordering::SeqCst);
log::debug!("resuming codec (blocking)");
codec.resume().unwrap();
while !synth_done.load(Ordering::SeqCst) {
// this is done fairly "fast" because the synth buf fills quickly
tt.sleep_ms(WAIT_INTERVAL).unwrap();
}
while codec.is_running().unwrap() {
// this actually waits until the playing is fully done
tt.sleep_ms(WAIT_INTERVAL).unwrap();
}
},
Some(Opcode::CodecCb) => msg_scalar_unpack!(msg, free_play, _available_rec, _, routing_id, {
if routing_id == codec::AUDIO_CB_ROUTING_ID {
let mut frames: FrameRing = FrameRing::new();
let frames_to_push = if frames.writeable_count() < free_play {
frames.writeable_count()
} else {
free_play
};
frame_count += frames_to_push as u32;
log::debug!("f{} p{}", frame_count, frames_to_push);
let mut locked_buf = wavbuf.lock().unwrap();
if just_initiated {
// prevent stutter if the synth buffer isn't ready yet and we got an early fill request from the codec
if locked_buf.len() < codec::FIFO_DEPTH {
for _ in 0..frames_to_push {
let frame: [u32; codec::FIFO_DEPTH] = [ZERO_PCM as u32 | (ZERO_PCM as u32) << 16; codec::FIFO_DEPTH];
frames.nq_frame(frame).unwrap();
}
codec.swap_frames(&mut frames).unwrap();
continue;
} else {
just_initiated = false;
}
}
for _ in 0..frames_to_push {
let mut frame: [u32; codec::FIFO_DEPTH] = [ZERO_PCM as u32 | (ZERO_PCM as u32) << 16; codec::FIFO_DEPTH];
if locked_buf.len() >= frame.len() || synth_done.load(Ordering::SeqCst) {
for sample in frame.iter_mut() {
let samp = locked_buf.pop_front().unwrap_or(ZERO_PCM);
let left = samp as u16;
let right = samp as u16;
*sample = right as u32 | (left as u32) << 16;
}
frames.nq_frame(frame).unwrap();
} else {
log::trace!("ran out of frames during tts fill");
break;
}
}
codec.swap_frames(&mut frames).unwrap();
// detect if the buffer is empty and the synthesizer has indicated it's finished
if (locked_buf.len() == 0) && synth_done.load(Ordering::SeqCst) {
codec.pause().unwrap();
}
}
}),
Some(Opcode::CodecStop) => {
log::info!("stop called. Immediate stop and loss of audio data.");
codec.abort().unwrap();
@ -188,7 +220,11 @@ fn xmain() -> ! {
send_message(wav_cid,
Message::new_blocking_scalar(WaveOp::Quit.to_usize().unwrap(), 0, 0, 0, 0)
).expect("couldn't send quit to callback handler");
send_message(cb_cid,
Message::new_blocking_scalar(WaveOp::Quit.to_usize().unwrap(), 0, 0, 0, 0)
).expect("couldn't send quit to callback handler");
unsafe{xous::disconnect(wav_cid).ok()};
unsafe{xous::disconnect(cb_cid).ok()};
log::warn!("Quit received, goodbye world!");
break;
},

View file

@ -1 +1 @@
pub const LANG: &str = "en";
pub const LANG: &str = "en-tts";