From b246a369d4f542bcfb5eadc40c94b4441dd55f74 Mon Sep 17 00:00:00 2001 From: bunnie Date: Fri, 11 Mar 2022 02:37:48 +0800 Subject: [PATCH] lay in another layer of tts functionality now shellchat can talk, and password entries are a thing. next up: debug oom issues in STT when we start doing really big strings. --- Cargo.lock | 23 ++++++++++ Cargo.toml | 1 + services/gam/src/modal/textentry.rs | 12 +++++ services/ime-frontend/Cargo.toml | 3 ++ services/ime-frontend/locales/i18n.json | 6 +++ services/ime-frontend/src/main.rs | 44 +++++++++--------- services/ime-plugin-tts/Cargo.toml | 30 ++++++++++++ services/ime-plugin-tts/src/lib.rs | 6 +++ services/ime-plugin-tts/src/main.rs | 61 +++++++++++++++++++++++++ services/pddb/Cargo.toml | 3 ++ services/shellchat/Cargo.toml | 3 ++ services/shellchat/locales/i18n.json | 14 ++++++ services/shellchat/src/main.rs | 33 +++++++++++++ xtask/src/main.rs | 2 + 14 files changed, 218 insertions(+), 23 deletions(-) create mode 100644 services/ime-plugin-tts/Cargo.toml create mode 100644 services/ime-plugin-tts/src/lib.rs create mode 100644 services/ime-plugin-tts/src/main.rs create mode 100644 services/shellchat/locales/i18n.json diff --git a/Cargo.lock b/Cargo.lock index f15c27e06..7de4171de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1097,6 +1097,7 @@ dependencies = [ "num-traits", "rkyv", "ticktimer-server", + "tts-frontend", "utralib", "xous", "xous-ipc 0.9.7", @@ -1134,6 +1135,25 @@ dependencies = [ "xous-names", ] +[[package]] +name = "ime-plugin-tts" +version = "0.1.0" +dependencies = [ + "ime-plugin-api", + "locales", + "log", + "log-server", + "num-derive", + "num-traits", + "rkyv", + "ticktimer-server", + "tts-frontend", + "utralib", + "xous", + "xous-ipc 0.9.7", + "xous-names", +] + [[package]] name = "indenter" version = "0.3.3" @@ -1614,6 +1634,7 @@ dependencies = [ "susres", "ticktimer-server", "trng", + "tts-frontend", "utralib", "xous", "xous-ipc 0.9.7", @@ -2205,9 +2226,11 @@ dependencies = [ "hex", "ime-plugin-api", "ime-plugin-shell", + "ime-plugin-tts", "jtag", "keyboard", "llio", + "locales", "log", "log-server", "modals", diff --git a/Cargo.toml b/Cargo.toml index 71a7758b6..a0dd4452a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,7 @@ members = [ "services/benchmark-target", "services/ime-frontend", "services/ime-plugin-shell", + "services/ime-plugin-tts", "services/rkyv-test-server", "services/rkyv-test-client", "services/shellchat", diff --git a/services/gam/src/modal/textentry.rs b/services/gam/src/modal/textentry.rs index bf6339e53..49bef608c 100644 --- a/services/gam/src/modal/textentry.rs +++ b/services/gam/src/modal/textentry.rs @@ -264,6 +264,12 @@ impl ActionApi for TextEntry { // ignore null messages } '\u{8}' => { // backspace + #[cfg(feature="tts")] + { + let xns = xous_names::XousNames::new().unwrap(); + let tts = tts_frontend::TtsFrontend::new(&xns).unwrap(); + tts.tts_blocking(locales::t!("input.delete-tts", xous::LANG)).unwrap(); + } // coded in a conservative manner to avoid temporary allocations that can leave the plaintext on the stack if self.action_payload.0.len() > 0 { // don't backspace if we have no string. let mut temp_str = String::<256>::from_str(self.action_payload.0.as_str().unwrap()); @@ -277,6 +283,12 @@ impl ActionApi for TextEntry { } } _ => { // text entry + #[cfg(feature="tts")] + { + let xns = xous_names::XousNames::new().unwrap(); + let tts = tts_frontend::TtsFrontend::new(&xns).unwrap(); + tts.tts_blocking(&k.to_string()).unwrap(); + } self.action_payload.0.push(k).expect("ran out of space storing password"); log::trace!("****update payload: {}", self.action_payload.0); } diff --git a/services/ime-frontend/Cargo.toml b/services/ime-frontend/Cargo.toml index 4ed1a6e73..f41556beb 100644 --- a/services/ime-frontend/Cargo.toml +++ b/services/ime-frontend/Cargo.toml @@ -23,9 +23,12 @@ num-derive = {version = "0.3.3", default-features = false} num-traits = {version = "0.2.14", default-features = false} rkyv = {version = "0.4.3", default-features = false, features = ["const_generics"]} +tts-frontend = {path="../tts"} + [target.'cfg(not(any(windows,unix)))'.dependencies] utralib = {path = "../../utralib"} [features] +tts = [] debugprint = [] default = [] # "debugprint" diff --git a/services/ime-frontend/locales/i18n.json b/services/ime-frontend/locales/i18n.json index 63edfb22c..59e3e1fc9 100644 --- a/services/ime-frontend/locales/i18n.json +++ b/services/ime-frontend/locales/i18n.json @@ -1,4 +1,10 @@ { + "input.delete-tts": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "delete" + }, "input.greeting": { "en": "Ready for input...", "ja": "入力可能...", diff --git a/services/ime-frontend/src/main.rs b/services/ime-frontend/src/main.rs index f204a9bf3..0e20752fb 100644 --- a/services/ime-frontend/src/main.rs +++ b/services/ime-frontend/src/main.rs @@ -20,6 +20,8 @@ use xous::{CID, msg_scalar_unpack}; use core::fmt::Write; use locales::t; +#[cfg(feature = "tts")] +use tts_frontend::*; /// max number of prediction options to track/render const MAX_PREDICTION_OPTIONS: usize = 4; @@ -63,6 +65,8 @@ struct InputTracker { /// render the predictions. Slightly awkward because this code comes from before we had libstd pred_options: [Option; MAX_PREDICTION_OPTIONS], + #[cfg(feature = "tts")] + tts: TtsFrontend, } impl InputTracker { @@ -84,6 +88,8 @@ impl InputTracker { last_height: 0, was_grown: false, pred_options: Default::default(), + #[cfg(feature="tts")] + tts: TtsFrontend::new(xns).unwrap(), } } pub fn set_gam_token(&mut self, token: [u32; 4]) { @@ -302,6 +308,8 @@ impl InputTracker { do_redraw = true; } '\u{0008}' => { // backspace + #[cfg(feature="tts")] + self.tts.tts_simple(t!("input.delete-tts", xous::LANG)).unwrap(); if (self.characters > 0) && (self.insertion == self.characters) { if debug1{info!("simple backspace case")} self.line.pop(); @@ -356,7 +364,7 @@ impl InputTracker { } '\u{000d}' => { // carriage return let mut ret = xous_ipc::String::<4000>::new(); - write!(ret, "{}", self.line.as_str()).expect("couldn't copy input ilne to output"); + write!(ret, "{}", self.line.as_str()).expect("couldn't copy input line to output"); retstring = Some(ret); if let Some(trigger) = self.pred_triggers { @@ -406,8 +414,7 @@ impl InputTracker { update_predictor = true; } self.last_trigger_char = Some(self.insertion); - } - if trigger.punctuation && k.is_ascii_punctuation() { + } else if trigger.punctuation && k.is_ascii_punctuation() { if self.pred_phrase.len() > 0 { self.predictor.unwrap().feedback_picked( xous_ipc::String::<4000>::from_str(&self.pred_phrase)).expect("couldn't send feedback to predictor"); @@ -419,6 +426,13 @@ impl InputTracker { } } if self.insertion == self.characters { + #[cfg(feature="tts")] + { + if !k.is_ascii_whitespace() && !k.is_ascii_punctuation() { + // this is disastisfying in how slow it is + // self.tts.tts_simple(&k.to_string()).unwrap(); + } + } self.line.push(k); if let Some(trigger) = self.pred_triggers { if !(trigger.punctuation && k.is_ascii_punctuation() || @@ -687,26 +701,10 @@ fn xmain() -> ! { if tracker.is_init() { msg_scalar_unpack!(msg, k1, k2, k3, k4, { let keys = [ - if let Some(a) = core::char::from_u32(k1 as u32) { - a - } else { - '\u{0000}' - }, - if let Some(a) = core::char::from_u32(k2 as u32) { - a - } else { - '\u{0000}' - }, - if let Some(a) = core::char::from_u32(k3 as u32) { - a - } else { - '\u{0000}' - }, - if let Some(a) = core::char::from_u32(k4 as u32) { - a - } else { - '\u{0000}' - }, + core::char::from_u32(k1 as u32).unwrap_or('\u{0000}'), + core::char::from_u32(k2 as u32).unwrap_or('\u{0000}'), + core::char::from_u32(k3 as u32).unwrap_or('\u{0000}'), + core::char::from_u32(k4 as u32).unwrap_or('\u{0000}'), ]; log::trace!("tracking keys: {:?}", keys); if keys[0] == '😊' { diff --git a/services/ime-plugin-tts/Cargo.toml b/services/ime-plugin-tts/Cargo.toml new file mode 100644 index 000000000..40d924e35 --- /dev/null +++ b/services/ime-plugin-tts/Cargo.toml @@ -0,0 +1,30 @@ +[package] +authors = ["bunnie "] +description = "IME Text to Speech Plugin" +edition = "2018" +name = "ime-plugin-tts" +version = "0.1.0" + +# Dependency policy: fully specify dependencies to the minor version number +[dependencies] +ime-plugin-api = {path = "../ime-plugin-api"} +log = "0.4.14" +log-server = {path = "../log-server"} +ticktimer-server = {path = "../ticktimer-server"} +xous = {path = "../../xous-rs"} +xous-ipc = {path = "../../xous-ipc"} +xous-names = {path = "../xous-names"} + +num-derive = {version = "0.3.3", default-features = false} +num-traits = {version = "0.2.14", default-features = false} +rkyv = {version = "0.4.3", default-features = false, features = ["const_generics"]} + +tts-frontend = {path="../tts"} +locales = {path = "../../locales"} + +[target.'cfg(not(any(windows,unix)))'.dependencies] +utralib = {path = "../../utralib"} + +[features] +debugprint = [] +default = [] # "debugprint" diff --git a/services/ime-plugin-tts/src/lib.rs b/services/ime-plugin-tts/src/lib.rs new file mode 100644 index 000000000..1c0da06df --- /dev/null +++ b/services/ime-plugin-tts/src/lib.rs @@ -0,0 +1,6 @@ +#![cfg_attr(target_os = "none", no_std)] + +pub const SERVER_NAME_IME_PLUGIN_TTS: &str = "_IME TTS plugin_"; + +// just inherit all the default from the ime_plugin_api +pub use ime_plugin_api::*; diff --git a/services/ime-plugin-tts/src/main.rs b/services/ime-plugin-tts/src/main.rs new file mode 100644 index 000000000..f6176e6a7 --- /dev/null +++ b/services/ime-plugin-tts/src/main.rs @@ -0,0 +1,61 @@ +#![cfg_attr(target_os = "none", no_std)] +#![cfg_attr(target_os = "none", no_main)] + +use ime_plugin_api::*; + +use xous_ipc::{String, Buffer}; +use num_traits::FromPrimitive; +use tts_frontend::*; + +#[xous::xous_main] +fn xmain() -> ! { + log_server::init_wait().unwrap(); + log::set_max_level(log::LevelFilter::Info); + log::info!("my PID is {}", xous::process::id()); + + let xns = xous_names::XousNames::new().unwrap(); + // one connection only, should be the GAM + let ime_sh_sid = xns.register_name(ime_plugin_tts::SERVER_NAME_IME_PLUGIN_TTS, Some(1)).expect("can't register server"); + log::trace!("registered with NS -- {:?}", ime_sh_sid); + let tts = TtsFrontend::new(&xns).unwrap(); + + let mytriggers = PredictionTriggers { + newline: true, + punctuation: true, + whitespace: true, + }; + + log::trace!("ready to accept requests"); + loop { + let mut msg = xous::receive_message(ime_sh_sid).unwrap(); + match FromPrimitive::from_usize(msg.body.id()) { + Some(Opcode::Input) => { + } + Some(Opcode::Picked) => { + let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) }; + let s = buffer.as_flat::, _>().unwrap(); + tts.tts_simple(s.as_str()).unwrap(); + } + Some(Opcode::Prediction) => { + let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) }; + let mut prediction: Prediction = buffer.to_original::().unwrap(); + prediction.valid = false; + buffer.replace(Return::Prediction(prediction)).expect("couldn't return Prediction"); + } + Some(Opcode::Unpick) => { + } + Some(Opcode::GetPredictionTriggers) => { + xous::return_scalar(msg.sender, mytriggers.into()).expect("couldn't return GetPredictionTriggers"); + } + Some(Opcode::Quit) => { + log::error!("received quit, goodbye!"); break; + } + None => {log::error!("unknown Opcode");} + } + } + log::trace!("main loop exit, destroying servers"); + xns.unregister_server(ime_sh_sid).unwrap(); + xous::destroy_server(ime_sh_sid).unwrap(); + log::trace!("quitting"); + xous::terminate_process(0) +} diff --git a/services/pddb/Cargo.toml b/services/pddb/Cargo.toml index 97c4e6257..867783abc 100644 --- a/services/pddb/Cargo.toml +++ b/services/pddb/Cargo.toml @@ -28,6 +28,7 @@ bitfield = "0.13.2" aes-gcm-siv = "0.10.3" llio = {path="../llio"} subtle = {version = "2.4.1", default-features = false} +tts-frontend = {path="../tts"} # passwords sha2 = {path = "../engine-sha512"} @@ -61,4 +62,6 @@ mbbb = [] deterministic = [] # this flag adds CI tests to the build ci = [] +# this feature is for text-to-speech support +tts = [] default = ["mbbb"] diff --git a/services/shellchat/Cargo.toml b/services/shellchat/Cargo.toml index 0e354b69d..315f378e0 100644 --- a/services/shellchat/Cargo.toml +++ b/services/shellchat/Cargo.toml @@ -13,6 +13,7 @@ gam = {path = "../gam"} graphics-server = {path = "../graphics-server"} ime-plugin-api = {path = "../ime-plugin-api"} ime-plugin-shell = {path = "../ime-plugin-shell"} +ime-plugin-tts = {path = "../ime-plugin-tts"} llio = {path = "../llio"} log = "0.4.14" log-server = {path = "../log-server"} @@ -36,7 +37,9 @@ net = {path="../net"} dns = {path="../dns"} pddb = {path="../pddb"} modals = {path="../modals"} + tts-frontend = {path="../tts"} +locales = {path = "../../locales"} trng = {path = "../trng"} diff --git a/services/shellchat/locales/i18n.json b/services/shellchat/locales/i18n.json new file mode 100644 index 000000000..efb02137f --- /dev/null +++ b/services/shellchat/locales/i18n.json @@ -0,0 +1,14 @@ +{ + "shellchat.input-tts": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "got " + }, + "shellchat.output-tts": { + "en": "", + "ja": "", + "zh": "", + "en-tts": "response " + } +} \ No newline at end of file diff --git a/services/shellchat/src/main.rs b/services/shellchat/src/main.rs index a0279cca4..7d417a2c6 100644 --- a/services/shellchat/src/main.rs +++ b/services/shellchat/src/main.rs @@ -45,6 +45,11 @@ use cmds::*; mod oqc_test; +#[cfg(feature="tts")] +use locales::t; +#[cfg(feature="tts")] +use tts_frontend::*; + #[derive(Debug)] struct History { // the history record @@ -80,6 +85,8 @@ struct Repl { // our security token for making changes to our record on the GAM token: [u32; 4], + #[cfg(feature="tts")] + tts: TtsFrontend, } impl Repl{ fn new(xns: &xous_names::XousNames, sid: xous::SID) -> Self { @@ -88,7 +95,10 @@ impl Repl{ let token = gam.register_ux(UxRegistration { app_name: xous_ipc::String::<128>::from_str(gam::APP_NAME_SHELLCHAT), ux_type: gam::UxType::Chat, + #[cfg(not(feature="tts"))] predictor: Some(xous_ipc::String::<64>::from_str(ime_plugin_shell::SERVER_NAME_IME_PLUGIN_SHELL)), + #[cfg(feature="tts")] + predictor: Some(xous_ipc::String::<64>::from_str(ime_plugin_tts::SERVER_NAME_IME_PLUGIN_TTS)), listener: sid.to_array(), // note disclosure of our SID to the GAM -- the secret is now shared with the GAM! redraw_id: ShellOpcode::Redraw.to_u32().unwrap(), gotinput_id: Some(ShellOpcode::Line.to_u32().unwrap()), @@ -116,6 +126,8 @@ impl Repl{ bubble_space: 4, env: CmdEnv::new(xns), token: token.unwrap(), + #[cfg(feature="tts")] + tts: TtsFrontend::new(xns).unwrap(), } } @@ -163,6 +175,12 @@ impl Repl{ if let Some(local) = &self.input { log::trace!("processing line: {}", local); if let Some(res) = self.env.dispatch(Some(&mut xous_ipc::String::<1024>::from_str(&local)), None).expect("command dispatch failed") { + #[cfg(feature="tts")] + { + let mut output = t!("shellchat.output-tts", xous::LANG).to_string(); + output.push_str(res.as_str().unwrap_or("UTF-8 error")); + self.tts.tts_simple(&output).unwrap(); + } let output_history = History { text: String::from(res.as_str().unwrap_or("UTF-8 Error")), is_input: false @@ -174,6 +192,12 @@ impl Repl{ } else if let Some(msg) = &self.msg { log::trace!("processing callback msg: {:?}", msg); if let Some(res) = self.env.dispatch(None, Some(msg)).expect("callback failed") { + #[cfg(feature="tts")] + { + let mut output = t!("shellchat.output-tts", xous::LANG).to_string(); + output.push_str(res.as_str().unwrap_or("UTF-8 error")); + self.tts.tts_simple(&output).unwrap(); + } let output_history = History { text: String::from(res.as_str().unwrap_or("UTF-8 Error")), is_input: false @@ -297,6 +321,9 @@ fn xmain() -> ! { let shch_sid = xns.register_name(SERVER_NAME_SHELLCHAT, None).expect("can't register server"); //log::trace!("registered with NS -- {:?}", shch_sid); + #[cfg(feature="tts")] + let tts = TtsFrontend::new(&xns).unwrap(); + let mut repl = Repl::new(&xns, shch_sid); let mut update_repl = false; let mut was_callback = false; @@ -311,6 +338,12 @@ fn xmain() -> ! { let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) }; let s = buffer.as_flat::, _>().unwrap(); log::trace!("shell got input line: {}", s.as_str()); + #[cfg(feature="tts")] + { + let mut input = t!("shellchat.input-tts", xous::LANG).to_string(); + input.push_str(s.as_str()); + tts.tts_simple(&input).unwrap(); + } repl.input(s.as_str()).expect("REPL couldn't accept input string"); update_repl = true; // set a flag, instead of calling here, so message can drop and calling server is released was_callback = false; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 86be8dcf3..f944801fd 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -230,6 +230,8 @@ fn main() -> Result<(), Box> { pkgs.push(app); } pkgs.push("tts-frontend"); + pkgs.push("ime-plugin-tts"); + pkgs.retain(|&pkg| pkg != "ime-plugin-shell"); generate_app_menus(&apps); build_hw_image(false, Some("./precursors/soc.svd".to_string()),