lay in another layer of tts functionality

now shellchat can talk, and password entries are a thing.

next up: debug oom issues in STT when we start doing really big
strings.
This commit is contained in:
bunnie 2022-03-11 02:37:48 +08:00
parent c44f794dfe
commit b246a369d4
14 changed files with 218 additions and 23 deletions

23
Cargo.lock generated
View file

@ -1097,6 +1097,7 @@ dependencies = [
"num-traits",
"rkyv",
"ticktimer-server",
"tts-frontend",
"utralib",
"xous",
"xous-ipc 0.9.7",
@ -1134,6 +1135,25 @@ dependencies = [
"xous-names",
]
[[package]]
name = "ime-plugin-tts"
version = "0.1.0"
dependencies = [
"ime-plugin-api",
"locales",
"log",
"log-server",
"num-derive",
"num-traits",
"rkyv",
"ticktimer-server",
"tts-frontend",
"utralib",
"xous",
"xous-ipc 0.9.7",
"xous-names",
]
[[package]]
name = "indenter"
version = "0.3.3"
@ -1614,6 +1634,7 @@ dependencies = [
"susres",
"ticktimer-server",
"trng",
"tts-frontend",
"utralib",
"xous",
"xous-ipc 0.9.7",
@ -2205,9 +2226,11 @@ dependencies = [
"hex",
"ime-plugin-api",
"ime-plugin-shell",
"ime-plugin-tts",
"jtag",
"keyboard",
"llio",
"locales",
"log",
"log-server",
"modals",

View file

@ -58,6 +58,7 @@ members = [
"services/benchmark-target",
"services/ime-frontend",
"services/ime-plugin-shell",
"services/ime-plugin-tts",
"services/rkyv-test-server",
"services/rkyv-test-client",
"services/shellchat",

View file

@ -264,6 +264,12 @@ impl ActionApi for TextEntry {
// ignore null messages
}
'\u{8}' => { // backspace
#[cfg(feature="tts")]
{
let xns = xous_names::XousNames::new().unwrap();
let tts = tts_frontend::TtsFrontend::new(&xns).unwrap();
tts.tts_blocking(locales::t!("input.delete-tts", xous::LANG)).unwrap();
}
// coded in a conservative manner to avoid temporary allocations that can leave the plaintext on the stack
if self.action_payload.0.len() > 0 { // don't backspace if we have no string.
let mut temp_str = String::<256>::from_str(self.action_payload.0.as_str().unwrap());
@ -277,6 +283,12 @@ impl ActionApi for TextEntry {
}
}
_ => { // text entry
#[cfg(feature="tts")]
{
let xns = xous_names::XousNames::new().unwrap();
let tts = tts_frontend::TtsFrontend::new(&xns).unwrap();
tts.tts_blocking(&k.to_string()).unwrap();
}
self.action_payload.0.push(k).expect("ran out of space storing password");
log::trace!("****update payload: {}", self.action_payload.0);
}

View file

@ -23,9 +23,12 @@ num-derive = {version = "0.3.3", default-features = false}
num-traits = {version = "0.2.14", default-features = false}
rkyv = {version = "0.4.3", default-features = false, features = ["const_generics"]}
tts-frontend = {path="../tts"}
[target.'cfg(not(any(windows,unix)))'.dependencies]
utralib = {path = "../../utralib"}
[features]
tts = []
debugprint = []
default = [] # "debugprint"

View file

@ -1,4 +1,10 @@
{
"input.delete-tts": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "delete"
},
"input.greeting": {
"en": "Ready for input...",
"ja": "入力可能...",

View file

@ -20,6 +20,8 @@ use xous::{CID, msg_scalar_unpack};
use core::fmt::Write;
use locales::t;
#[cfg(feature = "tts")]
use tts_frontend::*;
/// max number of prediction options to track/render
const MAX_PREDICTION_OPTIONS: usize = 4;
@ -63,6 +65,8 @@ struct InputTracker {
/// render the predictions. Slightly awkward because this code comes from before we had libstd
pred_options: [Option<String>; MAX_PREDICTION_OPTIONS],
#[cfg(feature = "tts")]
tts: TtsFrontend,
}
impl InputTracker {
@ -84,6 +88,8 @@ impl InputTracker {
last_height: 0,
was_grown: false,
pred_options: Default::default(),
#[cfg(feature="tts")]
tts: TtsFrontend::new(xns).unwrap(),
}
}
pub fn set_gam_token(&mut self, token: [u32; 4]) {
@ -302,6 +308,8 @@ impl InputTracker {
do_redraw = true;
}
'\u{0008}' => { // backspace
#[cfg(feature="tts")]
self.tts.tts_simple(t!("input.delete-tts", xous::LANG)).unwrap();
if (self.characters > 0) && (self.insertion == self.characters) {
if debug1{info!("simple backspace case")}
self.line.pop();
@ -356,7 +364,7 @@ impl InputTracker {
}
'\u{000d}' => { // carriage return
let mut ret = xous_ipc::String::<4000>::new();
write!(ret, "{}", self.line.as_str()).expect("couldn't copy input ilne to output");
write!(ret, "{}", self.line.as_str()).expect("couldn't copy input line to output");
retstring = Some(ret);
if let Some(trigger) = self.pred_triggers {
@ -406,8 +414,7 @@ impl InputTracker {
update_predictor = true;
}
self.last_trigger_char = Some(self.insertion);
}
if trigger.punctuation && k.is_ascii_punctuation() {
} else if trigger.punctuation && k.is_ascii_punctuation() {
if self.pred_phrase.len() > 0 {
self.predictor.unwrap().feedback_picked(
xous_ipc::String::<4000>::from_str(&self.pred_phrase)).expect("couldn't send feedback to predictor");
@ -419,6 +426,13 @@ impl InputTracker {
}
}
if self.insertion == self.characters {
#[cfg(feature="tts")]
{
if !k.is_ascii_whitespace() && !k.is_ascii_punctuation() {
// this is disastisfying in how slow it is
// self.tts.tts_simple(&k.to_string()).unwrap();
}
}
self.line.push(k);
if let Some(trigger) = self.pred_triggers {
if !(trigger.punctuation && k.is_ascii_punctuation() ||
@ -687,26 +701,10 @@ fn xmain() -> ! {
if tracker.is_init() {
msg_scalar_unpack!(msg, k1, k2, k3, k4, {
let keys = [
if let Some(a) = core::char::from_u32(k1 as u32) {
a
} else {
'\u{0000}'
},
if let Some(a) = core::char::from_u32(k2 as u32) {
a
} else {
'\u{0000}'
},
if let Some(a) = core::char::from_u32(k3 as u32) {
a
} else {
'\u{0000}'
},
if let Some(a) = core::char::from_u32(k4 as u32) {
a
} else {
'\u{0000}'
},
core::char::from_u32(k1 as u32).unwrap_or('\u{0000}'),
core::char::from_u32(k2 as u32).unwrap_or('\u{0000}'),
core::char::from_u32(k3 as u32).unwrap_or('\u{0000}'),
core::char::from_u32(k4 as u32).unwrap_or('\u{0000}'),
];
log::trace!("tracking keys: {:?}", keys);
if keys[0] == '😊' {

View file

@ -0,0 +1,30 @@
[package]
authors = ["bunnie <bunnie@kosagi.com>"]
description = "IME Text to Speech Plugin"
edition = "2018"
name = "ime-plugin-tts"
version = "0.1.0"
# Dependency policy: fully specify dependencies to the minor version number
[dependencies]
ime-plugin-api = {path = "../ime-plugin-api"}
log = "0.4.14"
log-server = {path = "../log-server"}
ticktimer-server = {path = "../ticktimer-server"}
xous = {path = "../../xous-rs"}
xous-ipc = {path = "../../xous-ipc"}
xous-names = {path = "../xous-names"}
num-derive = {version = "0.3.3", default-features = false}
num-traits = {version = "0.2.14", default-features = false}
rkyv = {version = "0.4.3", default-features = false, features = ["const_generics"]}
tts-frontend = {path="../tts"}
locales = {path = "../../locales"}
[target.'cfg(not(any(windows,unix)))'.dependencies]
utralib = {path = "../../utralib"}
[features]
debugprint = []
default = [] # "debugprint"

View file

@ -0,0 +1,6 @@
#![cfg_attr(target_os = "none", no_std)]
pub const SERVER_NAME_IME_PLUGIN_TTS: &str = "_IME TTS plugin_";
// just inherit all the default from the ime_plugin_api
pub use ime_plugin_api::*;

View file

@ -0,0 +1,61 @@
#![cfg_attr(target_os = "none", no_std)]
#![cfg_attr(target_os = "none", no_main)]
use ime_plugin_api::*;
use xous_ipc::{String, Buffer};
use num_traits::FromPrimitive;
use tts_frontend::*;
#[xous::xous_main]
fn xmain() -> ! {
log_server::init_wait().unwrap();
log::set_max_level(log::LevelFilter::Info);
log::info!("my PID is {}", xous::process::id());
let xns = xous_names::XousNames::new().unwrap();
// one connection only, should be the GAM
let ime_sh_sid = xns.register_name(ime_plugin_tts::SERVER_NAME_IME_PLUGIN_TTS, Some(1)).expect("can't register server");
log::trace!("registered with NS -- {:?}", ime_sh_sid);
let tts = TtsFrontend::new(&xns).unwrap();
let mytriggers = PredictionTriggers {
newline: true,
punctuation: true,
whitespace: true,
};
log::trace!("ready to accept requests");
loop {
let mut msg = xous::receive_message(ime_sh_sid).unwrap();
match FromPrimitive::from_usize(msg.body.id()) {
Some(Opcode::Input) => {
}
Some(Opcode::Picked) => {
let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) };
let s = buffer.as_flat::<String::<4000>, _>().unwrap();
tts.tts_simple(s.as_str()).unwrap();
}
Some(Opcode::Prediction) => {
let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
let mut prediction: Prediction = buffer.to_original::<Prediction, _>().unwrap();
prediction.valid = false;
buffer.replace(Return::Prediction(prediction)).expect("couldn't return Prediction");
}
Some(Opcode::Unpick) => {
}
Some(Opcode::GetPredictionTriggers) => {
xous::return_scalar(msg.sender, mytriggers.into()).expect("couldn't return GetPredictionTriggers");
}
Some(Opcode::Quit) => {
log::error!("received quit, goodbye!"); break;
}
None => {log::error!("unknown Opcode");}
}
}
log::trace!("main loop exit, destroying servers");
xns.unregister_server(ime_sh_sid).unwrap();
xous::destroy_server(ime_sh_sid).unwrap();
log::trace!("quitting");
xous::terminate_process(0)
}

View file

@ -28,6 +28,7 @@ bitfield = "0.13.2"
aes-gcm-siv = "0.10.3"
llio = {path="../llio"}
subtle = {version = "2.4.1", default-features = false}
tts-frontend = {path="../tts"}
# passwords
sha2 = {path = "../engine-sha512"}
@ -61,4 +62,6 @@ mbbb = []
deterministic = []
# this flag adds CI tests to the build
ci = []
# this feature is for text-to-speech support
tts = []
default = ["mbbb"]

View file

@ -13,6 +13,7 @@ gam = {path = "../gam"}
graphics-server = {path = "../graphics-server"}
ime-plugin-api = {path = "../ime-plugin-api"}
ime-plugin-shell = {path = "../ime-plugin-shell"}
ime-plugin-tts = {path = "../ime-plugin-tts"}
llio = {path = "../llio"}
log = "0.4.14"
log-server = {path = "../log-server"}
@ -36,7 +37,9 @@ net = {path="../net"}
dns = {path="../dns"}
pddb = {path="../pddb"}
modals = {path="../modals"}
tts-frontend = {path="../tts"}
locales = {path = "../../locales"}
trng = {path = "../trng"}

View file

@ -0,0 +1,14 @@
{
"shellchat.input-tts": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "got "
},
"shellchat.output-tts": {
"en": "",
"ja": "",
"zh": "",
"en-tts": "response "
}
}

View file

@ -45,6 +45,11 @@ use cmds::*;
mod oqc_test;
#[cfg(feature="tts")]
use locales::t;
#[cfg(feature="tts")]
use tts_frontend::*;
#[derive(Debug)]
struct History {
// the history record
@ -80,6 +85,8 @@ struct Repl {
// our security token for making changes to our record on the GAM
token: [u32; 4],
#[cfg(feature="tts")]
tts: TtsFrontend,
}
impl Repl{
fn new(xns: &xous_names::XousNames, sid: xous::SID) -> Self {
@ -88,7 +95,10 @@ impl Repl{
let token = gam.register_ux(UxRegistration {
app_name: xous_ipc::String::<128>::from_str(gam::APP_NAME_SHELLCHAT),
ux_type: gam::UxType::Chat,
#[cfg(not(feature="tts"))]
predictor: Some(xous_ipc::String::<64>::from_str(ime_plugin_shell::SERVER_NAME_IME_PLUGIN_SHELL)),
#[cfg(feature="tts")]
predictor: Some(xous_ipc::String::<64>::from_str(ime_plugin_tts::SERVER_NAME_IME_PLUGIN_TTS)),
listener: sid.to_array(), // note disclosure of our SID to the GAM -- the secret is now shared with the GAM!
redraw_id: ShellOpcode::Redraw.to_u32().unwrap(),
gotinput_id: Some(ShellOpcode::Line.to_u32().unwrap()),
@ -116,6 +126,8 @@ impl Repl{
bubble_space: 4,
env: CmdEnv::new(xns),
token: token.unwrap(),
#[cfg(feature="tts")]
tts: TtsFrontend::new(xns).unwrap(),
}
}
@ -163,6 +175,12 @@ impl Repl{
if let Some(local) = &self.input {
log::trace!("processing line: {}", local);
if let Some(res) = self.env.dispatch(Some(&mut xous_ipc::String::<1024>::from_str(&local)), None).expect("command dispatch failed") {
#[cfg(feature="tts")]
{
let mut output = t!("shellchat.output-tts", xous::LANG).to_string();
output.push_str(res.as_str().unwrap_or("UTF-8 error"));
self.tts.tts_simple(&output).unwrap();
}
let output_history = History {
text: String::from(res.as_str().unwrap_or("UTF-8 Error")),
is_input: false
@ -174,6 +192,12 @@ impl Repl{
} else if let Some(msg) = &self.msg {
log::trace!("processing callback msg: {:?}", msg);
if let Some(res) = self.env.dispatch(None, Some(msg)).expect("callback failed") {
#[cfg(feature="tts")]
{
let mut output = t!("shellchat.output-tts", xous::LANG).to_string();
output.push_str(res.as_str().unwrap_or("UTF-8 error"));
self.tts.tts_simple(&output).unwrap();
}
let output_history = History {
text: String::from(res.as_str().unwrap_or("UTF-8 Error")),
is_input: false
@ -297,6 +321,9 @@ fn xmain() -> ! {
let shch_sid = xns.register_name(SERVER_NAME_SHELLCHAT, None).expect("can't register server");
//log::trace!("registered with NS -- {:?}", shch_sid);
#[cfg(feature="tts")]
let tts = TtsFrontend::new(&xns).unwrap();
let mut repl = Repl::new(&xns, shch_sid);
let mut update_repl = false;
let mut was_callback = false;
@ -311,6 +338,12 @@ fn xmain() -> ! {
let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) };
let s = buffer.as_flat::<xous_ipc::String<4000>, _>().unwrap();
log::trace!("shell got input line: {}", s.as_str());
#[cfg(feature="tts")]
{
let mut input = t!("shellchat.input-tts", xous::LANG).to_string();
input.push_str(s.as_str());
tts.tts_simple(&input).unwrap();
}
repl.input(s.as_str()).expect("REPL couldn't accept input string");
update_repl = true; // set a flag, instead of calling here, so message can drop and calling server is released
was_callback = false;

View file

@ -230,6 +230,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
pkgs.push(app);
}
pkgs.push("tts-frontend");
pkgs.push("ime-plugin-tts");
pkgs.retain(|&pkg| pkg != "ime-plugin-shell");
generate_app_menus(&apps);
build_hw_image(false,
Some("./precursors/soc.svd".to_string()),