mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-16 08:38:26 +00:00
lay in another layer of tts functionality
now shellchat can talk, and password entries are a thing. next up: debug oom issues in STT when we start doing really big strings.
This commit is contained in:
parent
c44f794dfe
commit
b246a369d4
14 changed files with 218 additions and 23 deletions
23
Cargo.lock
generated
23
Cargo.lock
generated
|
|
@ -1097,6 +1097,7 @@ dependencies = [
|
|||
"num-traits",
|
||||
"rkyv",
|
||||
"ticktimer-server",
|
||||
"tts-frontend",
|
||||
"utralib",
|
||||
"xous",
|
||||
"xous-ipc 0.9.7",
|
||||
|
|
@ -1134,6 +1135,25 @@ dependencies = [
|
|||
"xous-names",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ime-plugin-tts"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ime-plugin-api",
|
||||
"locales",
|
||||
"log",
|
||||
"log-server",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"rkyv",
|
||||
"ticktimer-server",
|
||||
"tts-frontend",
|
||||
"utralib",
|
||||
"xous",
|
||||
"xous-ipc 0.9.7",
|
||||
"xous-names",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indenter"
|
||||
version = "0.3.3"
|
||||
|
|
@ -1614,6 +1634,7 @@ dependencies = [
|
|||
"susres",
|
||||
"ticktimer-server",
|
||||
"trng",
|
||||
"tts-frontend",
|
||||
"utralib",
|
||||
"xous",
|
||||
"xous-ipc 0.9.7",
|
||||
|
|
@ -2205,9 +2226,11 @@ dependencies = [
|
|||
"hex",
|
||||
"ime-plugin-api",
|
||||
"ime-plugin-shell",
|
||||
"ime-plugin-tts",
|
||||
"jtag",
|
||||
"keyboard",
|
||||
"llio",
|
||||
"locales",
|
||||
"log",
|
||||
"log-server",
|
||||
"modals",
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ members = [
|
|||
"services/benchmark-target",
|
||||
"services/ime-frontend",
|
||||
"services/ime-plugin-shell",
|
||||
"services/ime-plugin-tts",
|
||||
"services/rkyv-test-server",
|
||||
"services/rkyv-test-client",
|
||||
"services/shellchat",
|
||||
|
|
|
|||
|
|
@ -264,6 +264,12 @@ impl ActionApi for TextEntry {
|
|||
// ignore null messages
|
||||
}
|
||||
'\u{8}' => { // backspace
|
||||
#[cfg(feature="tts")]
|
||||
{
|
||||
let xns = xous_names::XousNames::new().unwrap();
|
||||
let tts = tts_frontend::TtsFrontend::new(&xns).unwrap();
|
||||
tts.tts_blocking(locales::t!("input.delete-tts", xous::LANG)).unwrap();
|
||||
}
|
||||
// coded in a conservative manner to avoid temporary allocations that can leave the plaintext on the stack
|
||||
if self.action_payload.0.len() > 0 { // don't backspace if we have no string.
|
||||
let mut temp_str = String::<256>::from_str(self.action_payload.0.as_str().unwrap());
|
||||
|
|
@ -277,6 +283,12 @@ impl ActionApi for TextEntry {
|
|||
}
|
||||
}
|
||||
_ => { // text entry
|
||||
#[cfg(feature="tts")]
|
||||
{
|
||||
let xns = xous_names::XousNames::new().unwrap();
|
||||
let tts = tts_frontend::TtsFrontend::new(&xns).unwrap();
|
||||
tts.tts_blocking(&k.to_string()).unwrap();
|
||||
}
|
||||
self.action_payload.0.push(k).expect("ran out of space storing password");
|
||||
log::trace!("****update payload: {}", self.action_payload.0);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,9 +23,12 @@ num-derive = {version = "0.3.3", default-features = false}
|
|||
num-traits = {version = "0.2.14", default-features = false}
|
||||
rkyv = {version = "0.4.3", default-features = false, features = ["const_generics"]}
|
||||
|
||||
tts-frontend = {path="../tts"}
|
||||
|
||||
[target.'cfg(not(any(windows,unix)))'.dependencies]
|
||||
utralib = {path = "../../utralib"}
|
||||
|
||||
[features]
|
||||
tts = []
|
||||
debugprint = []
|
||||
default = [] # "debugprint"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
{
|
||||
"input.delete-tts": {
|
||||
"en": "",
|
||||
"ja": "",
|
||||
"zh": "",
|
||||
"en-tts": "delete"
|
||||
},
|
||||
"input.greeting": {
|
||||
"en": "Ready for input...",
|
||||
"ja": "入力可能...",
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ use xous::{CID, msg_scalar_unpack};
|
|||
use core::fmt::Write;
|
||||
|
||||
use locales::t;
|
||||
#[cfg(feature = "tts")]
|
||||
use tts_frontend::*;
|
||||
|
||||
/// max number of prediction options to track/render
|
||||
const MAX_PREDICTION_OPTIONS: usize = 4;
|
||||
|
|
@ -63,6 +65,8 @@ struct InputTracker {
|
|||
|
||||
/// render the predictions. Slightly awkward because this code comes from before we had libstd
|
||||
pred_options: [Option<String>; MAX_PREDICTION_OPTIONS],
|
||||
#[cfg(feature = "tts")]
|
||||
tts: TtsFrontend,
|
||||
}
|
||||
|
||||
impl InputTracker {
|
||||
|
|
@ -84,6 +88,8 @@ impl InputTracker {
|
|||
last_height: 0,
|
||||
was_grown: false,
|
||||
pred_options: Default::default(),
|
||||
#[cfg(feature="tts")]
|
||||
tts: TtsFrontend::new(xns).unwrap(),
|
||||
}
|
||||
}
|
||||
pub fn set_gam_token(&mut self, token: [u32; 4]) {
|
||||
|
|
@ -302,6 +308,8 @@ impl InputTracker {
|
|||
do_redraw = true;
|
||||
}
|
||||
'\u{0008}' => { // backspace
|
||||
#[cfg(feature="tts")]
|
||||
self.tts.tts_simple(t!("input.delete-tts", xous::LANG)).unwrap();
|
||||
if (self.characters > 0) && (self.insertion == self.characters) {
|
||||
if debug1{info!("simple backspace case")}
|
||||
self.line.pop();
|
||||
|
|
@ -356,7 +364,7 @@ impl InputTracker {
|
|||
}
|
||||
'\u{000d}' => { // carriage return
|
||||
let mut ret = xous_ipc::String::<4000>::new();
|
||||
write!(ret, "{}", self.line.as_str()).expect("couldn't copy input ilne to output");
|
||||
write!(ret, "{}", self.line.as_str()).expect("couldn't copy input line to output");
|
||||
retstring = Some(ret);
|
||||
|
||||
if let Some(trigger) = self.pred_triggers {
|
||||
|
|
@ -406,8 +414,7 @@ impl InputTracker {
|
|||
update_predictor = true;
|
||||
}
|
||||
self.last_trigger_char = Some(self.insertion);
|
||||
}
|
||||
if trigger.punctuation && k.is_ascii_punctuation() {
|
||||
} else if trigger.punctuation && k.is_ascii_punctuation() {
|
||||
if self.pred_phrase.len() > 0 {
|
||||
self.predictor.unwrap().feedback_picked(
|
||||
xous_ipc::String::<4000>::from_str(&self.pred_phrase)).expect("couldn't send feedback to predictor");
|
||||
|
|
@ -419,6 +426,13 @@ impl InputTracker {
|
|||
}
|
||||
}
|
||||
if self.insertion == self.characters {
|
||||
#[cfg(feature="tts")]
|
||||
{
|
||||
if !k.is_ascii_whitespace() && !k.is_ascii_punctuation() {
|
||||
// this is disastisfying in how slow it is
|
||||
// self.tts.tts_simple(&k.to_string()).unwrap();
|
||||
}
|
||||
}
|
||||
self.line.push(k);
|
||||
if let Some(trigger) = self.pred_triggers {
|
||||
if !(trigger.punctuation && k.is_ascii_punctuation() ||
|
||||
|
|
@ -687,26 +701,10 @@ fn xmain() -> ! {
|
|||
if tracker.is_init() {
|
||||
msg_scalar_unpack!(msg, k1, k2, k3, k4, {
|
||||
let keys = [
|
||||
if let Some(a) = core::char::from_u32(k1 as u32) {
|
||||
a
|
||||
} else {
|
||||
'\u{0000}'
|
||||
},
|
||||
if let Some(a) = core::char::from_u32(k2 as u32) {
|
||||
a
|
||||
} else {
|
||||
'\u{0000}'
|
||||
},
|
||||
if let Some(a) = core::char::from_u32(k3 as u32) {
|
||||
a
|
||||
} else {
|
||||
'\u{0000}'
|
||||
},
|
||||
if let Some(a) = core::char::from_u32(k4 as u32) {
|
||||
a
|
||||
} else {
|
||||
'\u{0000}'
|
||||
},
|
||||
core::char::from_u32(k1 as u32).unwrap_or('\u{0000}'),
|
||||
core::char::from_u32(k2 as u32).unwrap_or('\u{0000}'),
|
||||
core::char::from_u32(k3 as u32).unwrap_or('\u{0000}'),
|
||||
core::char::from_u32(k4 as u32).unwrap_or('\u{0000}'),
|
||||
];
|
||||
log::trace!("tracking keys: {:?}", keys);
|
||||
if keys[0] == '😊' {
|
||||
|
|
|
|||
30
services/ime-plugin-tts/Cargo.toml
Normal file
30
services/ime-plugin-tts/Cargo.toml
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
[package]
|
||||
authors = ["bunnie <bunnie@kosagi.com>"]
|
||||
description = "IME Text to Speech Plugin"
|
||||
edition = "2018"
|
||||
name = "ime-plugin-tts"
|
||||
version = "0.1.0"
|
||||
|
||||
# Dependency policy: fully specify dependencies to the minor version number
|
||||
[dependencies]
|
||||
ime-plugin-api = {path = "../ime-plugin-api"}
|
||||
log = "0.4.14"
|
||||
log-server = {path = "../log-server"}
|
||||
ticktimer-server = {path = "../ticktimer-server"}
|
||||
xous = {path = "../../xous-rs"}
|
||||
xous-ipc = {path = "../../xous-ipc"}
|
||||
xous-names = {path = "../xous-names"}
|
||||
|
||||
num-derive = {version = "0.3.3", default-features = false}
|
||||
num-traits = {version = "0.2.14", default-features = false}
|
||||
rkyv = {version = "0.4.3", default-features = false, features = ["const_generics"]}
|
||||
|
||||
tts-frontend = {path="../tts"}
|
||||
locales = {path = "../../locales"}
|
||||
|
||||
[target.'cfg(not(any(windows,unix)))'.dependencies]
|
||||
utralib = {path = "../../utralib"}
|
||||
|
||||
[features]
|
||||
debugprint = []
|
||||
default = [] # "debugprint"
|
||||
6
services/ime-plugin-tts/src/lib.rs
Normal file
6
services/ime-plugin-tts/src/lib.rs
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
#![cfg_attr(target_os = "none", no_std)]
|
||||
|
||||
pub const SERVER_NAME_IME_PLUGIN_TTS: &str = "_IME TTS plugin_";
|
||||
|
||||
// just inherit all the default from the ime_plugin_api
|
||||
pub use ime_plugin_api::*;
|
||||
61
services/ime-plugin-tts/src/main.rs
Normal file
61
services/ime-plugin-tts/src/main.rs
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
#![cfg_attr(target_os = "none", no_std)]
|
||||
#![cfg_attr(target_os = "none", no_main)]
|
||||
|
||||
use ime_plugin_api::*;
|
||||
|
||||
use xous_ipc::{String, Buffer};
|
||||
use num_traits::FromPrimitive;
|
||||
use tts_frontend::*;
|
||||
|
||||
#[xous::xous_main]
|
||||
fn xmain() -> ! {
|
||||
log_server::init_wait().unwrap();
|
||||
log::set_max_level(log::LevelFilter::Info);
|
||||
log::info!("my PID is {}", xous::process::id());
|
||||
|
||||
let xns = xous_names::XousNames::new().unwrap();
|
||||
// one connection only, should be the GAM
|
||||
let ime_sh_sid = xns.register_name(ime_plugin_tts::SERVER_NAME_IME_PLUGIN_TTS, Some(1)).expect("can't register server");
|
||||
log::trace!("registered with NS -- {:?}", ime_sh_sid);
|
||||
let tts = TtsFrontend::new(&xns).unwrap();
|
||||
|
||||
let mytriggers = PredictionTriggers {
|
||||
newline: true,
|
||||
punctuation: true,
|
||||
whitespace: true,
|
||||
};
|
||||
|
||||
log::trace!("ready to accept requests");
|
||||
loop {
|
||||
let mut msg = xous::receive_message(ime_sh_sid).unwrap();
|
||||
match FromPrimitive::from_usize(msg.body.id()) {
|
||||
Some(Opcode::Input) => {
|
||||
}
|
||||
Some(Opcode::Picked) => {
|
||||
let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) };
|
||||
let s = buffer.as_flat::<String::<4000>, _>().unwrap();
|
||||
tts.tts_simple(s.as_str()).unwrap();
|
||||
}
|
||||
Some(Opcode::Prediction) => {
|
||||
let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
|
||||
let mut prediction: Prediction = buffer.to_original::<Prediction, _>().unwrap();
|
||||
prediction.valid = false;
|
||||
buffer.replace(Return::Prediction(prediction)).expect("couldn't return Prediction");
|
||||
}
|
||||
Some(Opcode::Unpick) => {
|
||||
}
|
||||
Some(Opcode::GetPredictionTriggers) => {
|
||||
xous::return_scalar(msg.sender, mytriggers.into()).expect("couldn't return GetPredictionTriggers");
|
||||
}
|
||||
Some(Opcode::Quit) => {
|
||||
log::error!("received quit, goodbye!"); break;
|
||||
}
|
||||
None => {log::error!("unknown Opcode");}
|
||||
}
|
||||
}
|
||||
log::trace!("main loop exit, destroying servers");
|
||||
xns.unregister_server(ime_sh_sid).unwrap();
|
||||
xous::destroy_server(ime_sh_sid).unwrap();
|
||||
log::trace!("quitting");
|
||||
xous::terminate_process(0)
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ bitfield = "0.13.2"
|
|||
aes-gcm-siv = "0.10.3"
|
||||
llio = {path="../llio"}
|
||||
subtle = {version = "2.4.1", default-features = false}
|
||||
tts-frontend = {path="../tts"}
|
||||
|
||||
# passwords
|
||||
sha2 = {path = "../engine-sha512"}
|
||||
|
|
@ -61,4 +62,6 @@ mbbb = []
|
|||
deterministic = []
|
||||
# this flag adds CI tests to the build
|
||||
ci = []
|
||||
# this feature is for text-to-speech support
|
||||
tts = []
|
||||
default = ["mbbb"]
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ gam = {path = "../gam"}
|
|||
graphics-server = {path = "../graphics-server"}
|
||||
ime-plugin-api = {path = "../ime-plugin-api"}
|
||||
ime-plugin-shell = {path = "../ime-plugin-shell"}
|
||||
ime-plugin-tts = {path = "../ime-plugin-tts"}
|
||||
llio = {path = "../llio"}
|
||||
log = "0.4.14"
|
||||
log-server = {path = "../log-server"}
|
||||
|
|
@ -36,7 +37,9 @@ net = {path="../net"}
|
|||
dns = {path="../dns"}
|
||||
pddb = {path="../pddb"}
|
||||
modals = {path="../modals"}
|
||||
|
||||
tts-frontend = {path="../tts"}
|
||||
locales = {path = "../../locales"}
|
||||
|
||||
trng = {path = "../trng"}
|
||||
|
||||
|
|
|
|||
14
services/shellchat/locales/i18n.json
Normal file
14
services/shellchat/locales/i18n.json
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"shellchat.input-tts": {
|
||||
"en": "",
|
||||
"ja": "",
|
||||
"zh": "",
|
||||
"en-tts": "got "
|
||||
},
|
||||
"shellchat.output-tts": {
|
||||
"en": "",
|
||||
"ja": "",
|
||||
"zh": "",
|
||||
"en-tts": "response "
|
||||
}
|
||||
}
|
||||
|
|
@ -45,6 +45,11 @@ use cmds::*;
|
|||
|
||||
mod oqc_test;
|
||||
|
||||
#[cfg(feature="tts")]
|
||||
use locales::t;
|
||||
#[cfg(feature="tts")]
|
||||
use tts_frontend::*;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct History {
|
||||
// the history record
|
||||
|
|
@ -80,6 +85,8 @@ struct Repl {
|
|||
|
||||
// our security token for making changes to our record on the GAM
|
||||
token: [u32; 4],
|
||||
#[cfg(feature="tts")]
|
||||
tts: TtsFrontend,
|
||||
}
|
||||
impl Repl{
|
||||
fn new(xns: &xous_names::XousNames, sid: xous::SID) -> Self {
|
||||
|
|
@ -88,7 +95,10 @@ impl Repl{
|
|||
let token = gam.register_ux(UxRegistration {
|
||||
app_name: xous_ipc::String::<128>::from_str(gam::APP_NAME_SHELLCHAT),
|
||||
ux_type: gam::UxType::Chat,
|
||||
#[cfg(not(feature="tts"))]
|
||||
predictor: Some(xous_ipc::String::<64>::from_str(ime_plugin_shell::SERVER_NAME_IME_PLUGIN_SHELL)),
|
||||
#[cfg(feature="tts")]
|
||||
predictor: Some(xous_ipc::String::<64>::from_str(ime_plugin_tts::SERVER_NAME_IME_PLUGIN_TTS)),
|
||||
listener: sid.to_array(), // note disclosure of our SID to the GAM -- the secret is now shared with the GAM!
|
||||
redraw_id: ShellOpcode::Redraw.to_u32().unwrap(),
|
||||
gotinput_id: Some(ShellOpcode::Line.to_u32().unwrap()),
|
||||
|
|
@ -116,6 +126,8 @@ impl Repl{
|
|||
bubble_space: 4,
|
||||
env: CmdEnv::new(xns),
|
||||
token: token.unwrap(),
|
||||
#[cfg(feature="tts")]
|
||||
tts: TtsFrontend::new(xns).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -163,6 +175,12 @@ impl Repl{
|
|||
if let Some(local) = &self.input {
|
||||
log::trace!("processing line: {}", local);
|
||||
if let Some(res) = self.env.dispatch(Some(&mut xous_ipc::String::<1024>::from_str(&local)), None).expect("command dispatch failed") {
|
||||
#[cfg(feature="tts")]
|
||||
{
|
||||
let mut output = t!("shellchat.output-tts", xous::LANG).to_string();
|
||||
output.push_str(res.as_str().unwrap_or("UTF-8 error"));
|
||||
self.tts.tts_simple(&output).unwrap();
|
||||
}
|
||||
let output_history = History {
|
||||
text: String::from(res.as_str().unwrap_or("UTF-8 Error")),
|
||||
is_input: false
|
||||
|
|
@ -174,6 +192,12 @@ impl Repl{
|
|||
} else if let Some(msg) = &self.msg {
|
||||
log::trace!("processing callback msg: {:?}", msg);
|
||||
if let Some(res) = self.env.dispatch(None, Some(msg)).expect("callback failed") {
|
||||
#[cfg(feature="tts")]
|
||||
{
|
||||
let mut output = t!("shellchat.output-tts", xous::LANG).to_string();
|
||||
output.push_str(res.as_str().unwrap_or("UTF-8 error"));
|
||||
self.tts.tts_simple(&output).unwrap();
|
||||
}
|
||||
let output_history = History {
|
||||
text: String::from(res.as_str().unwrap_or("UTF-8 Error")),
|
||||
is_input: false
|
||||
|
|
@ -297,6 +321,9 @@ fn xmain() -> ! {
|
|||
let shch_sid = xns.register_name(SERVER_NAME_SHELLCHAT, None).expect("can't register server");
|
||||
//log::trace!("registered with NS -- {:?}", shch_sid);
|
||||
|
||||
#[cfg(feature="tts")]
|
||||
let tts = TtsFrontend::new(&xns).unwrap();
|
||||
|
||||
let mut repl = Repl::new(&xns, shch_sid);
|
||||
let mut update_repl = false;
|
||||
let mut was_callback = false;
|
||||
|
|
@ -311,6 +338,12 @@ fn xmain() -> ! {
|
|||
let buffer = unsafe { Buffer::from_memory_message(msg.body.memory_message().unwrap()) };
|
||||
let s = buffer.as_flat::<xous_ipc::String<4000>, _>().unwrap();
|
||||
log::trace!("shell got input line: {}", s.as_str());
|
||||
#[cfg(feature="tts")]
|
||||
{
|
||||
let mut input = t!("shellchat.input-tts", xous::LANG).to_string();
|
||||
input.push_str(s.as_str());
|
||||
tts.tts_simple(&input).unwrap();
|
||||
}
|
||||
repl.input(s.as_str()).expect("REPL couldn't accept input string");
|
||||
update_repl = true; // set a flag, instead of calling here, so message can drop and calling server is released
|
||||
was_callback = false;
|
||||
|
|
|
|||
|
|
@ -230,6 +230,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
pkgs.push(app);
|
||||
}
|
||||
pkgs.push("tts-frontend");
|
||||
pkgs.push("ime-plugin-tts");
|
||||
pkgs.retain(|&pkg| pkg != "ime-plugin-shell");
|
||||
generate_app_menus(&apps);
|
||||
build_hw_image(false,
|
||||
Some("./precursors/soc.svd".to_string()),
|
||||
|
|
|
|||
Loading…
Reference in a new issue