From feffc2e5be38d9034f5151b9fd0500345267d3a1 Mon Sep 17 00:00:00 2001 From: Tom Marble Date: Fri, 13 Jan 2023 16:52:28 -0600 Subject: [PATCH] WIP: Fix mtxcli excessive memory usage #287 Adds migrations framework - Migration for v0.9.11-0120: update the filter Filter - narrow filter to only the fields and events for the current room Introduces "read only" variables (e.g. __version). Signed-off-by: Tom Marble --- apps/mtxcli/Cargo.toml | 2 +- apps/mtxcli/src/cmds.rs | 108 +++++++++++------- apps/mtxcli/src/cmds/migrations.rs | 69 +++++++++++ .../src/cmds/migrations/v0_9_11_0120.rs | 26 +++++ apps/mtxcli/src/cmds/web.rs | 79 +++++++------ 5 files changed, 205 insertions(+), 79 deletions(-) create mode 100644 apps/mtxcli/src/cmds/migrations.rs create mode 100644 apps/mtxcli/src/cmds/migrations/v0_9_11_0120.rs diff --git a/apps/mtxcli/Cargo.toml b/apps/mtxcli/Cargo.toml index 159c77fbc..4fad08b42 100644 --- a/apps/mtxcli/Cargo.toml +++ b/apps/mtxcli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mtxcli" -version = "0.1.0" +version = "0.6.0" authors = ["bunnie "] edition = "2018" description = "Matrix chat" diff --git a/apps/mtxcli/src/cmds.rs b/apps/mtxcli/src/cmds.rs index eee2c6f0b..53dd2c4a9 100644 --- a/apps/mtxcli/src/cmds.rs +++ b/apps/mtxcli/src/cmds.rs @@ -5,10 +5,11 @@ use locales::t; use std::collections::HashMap; use std::fs::File; -use std::io::{Read, Write as StdWrite, Error}; +use std::io::{Read, Write as StdWrite, Error, ErrorKind}; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; // to help gen_txn_id +mod migrations; use migrations::run_migrations; mod url; mod web; @@ -35,6 +36,24 @@ const SERVER_MATRIX: &str = "https://matrix.org"; const EMPTY: &str = ""; const MTX_TIMEOUT: i32 = 300; // ms +/// Returns a version string which is more likely to string compare +/// correctly vs. another version. FFI please see +/// https://git-scm.com/docs/git-describe#_examples +fn get_version(ticktimer: &ticktimer_server::Ticktimer) -> String { + let xous_version = ticktimer.get_version(); + let v: Vec<&str> = xous_version.split('-').collect(); + if v.len() > 2 { + let n = v[1].parse::().expect("could not parse version"); + let version = format!("{}-{:04}",v[0], n); + log::info!("version={}=", version); + version + } else { + log::error!("ERROR, couldn't find version from xous_version: {}", + xous_version); + xous_version + } +} + /////////////////////////// Common items to all commands pub trait ShellCmdApi<'a> { // user implemented: @@ -90,6 +109,7 @@ pub struct CommonEnv { filter: String, since: String, first_line: bool, + version: String, } impl CommonEnv { @@ -103,24 +123,29 @@ impl CommonEnv { } pub fn set(&mut self, key: &str, value: &str) -> Result<(), Error> { - log::info!("set '{}' = '{}'", key, value); - let mut keypath = PathBuf::new(); - keypath.push(MTXCLI_DICT); - if std::fs::metadata(&keypath).is_ok() { // keypath exists - log::info!("dict '{}' exists", MTXCLI_DICT); + if key.starts_with("__") { + Err(Error::new(ErrorKind::PermissionDenied, + "may not set a variable beginning with __ ")) } else { - log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT); - std::fs::create_dir_all(&keypath)?; + log::info!("set '{}' = '{}'", key, value); + let mut keypath = PathBuf::new(); + keypath.push(MTXCLI_DICT); + if std::fs::metadata(&keypath).is_ok() { // keypath exists + log::info!("dict '{}' exists", MTXCLI_DICT); + } else { + log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT); + std::fs::create_dir_all(&keypath)?; + } + keypath.push(key); + File::create(keypath)?.write_all(value.as_bytes())?; + match key { // special case side effects + USER_KEY => { self.set_user(value); } + PASSWORD_KEY => { self.set_password(); } + ROOM_KEY => { self.set_room(); } + _ => { } + } + Ok(()) } - keypath.push(key); - File::create(keypath)?.write_all(value.as_bytes())?; - match key { // special case side effects - USER_KEY => { self.set_user(value); } - PASSWORD_KEY => { self.set_password(); } - ROOM_KEY => { self.set_room(); } - _ => { } - } - Ok(()) } // will log on error (vs. panic) @@ -174,22 +199,27 @@ impl CommonEnv { } pub fn unset(&mut self, key: &str) -> Result<(), Error> { - // log::info!("unset '{}'", key); - let mut keypath = PathBuf::new(); - keypath.push(MTXCLI_DICT); - if std::fs::metadata(&keypath).is_ok() { // keypath exists - log::info!("dict '{}' exists", MTXCLI_DICT); + if key.starts_with("__") { + Err(Error::new(ErrorKind::PermissionDenied, + "may not unset a variable beginning with __ ")) } else { - log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT); - std::fs::create_dir_all(&keypath)?; - } - keypath.push(key); - if std::fs::metadata(&keypath).is_ok() { // keypath exists - log::info!("dict:key = '{}:{}' exists.. deleting it", MTXCLI_DICT, key); + log::info!("unset '{}'", key); + let mut keypath = PathBuf::new(); + keypath.push(MTXCLI_DICT); + if std::fs::metadata(&keypath).is_ok() { // keypath exists + log::info!("dict '{}' exists", MTXCLI_DICT); + } else { + log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT); + std::fs::create_dir_all(&keypath)?; + } + keypath.push(key); + if std::fs::metadata(&keypath).is_ok() { // keypath exists + log::info!("dict:key = '{}:{}' exists.. deleting it", MTXCLI_DICT, key); - std::fs::remove_file(keypath)?; + std::fs::remove_file(keypath)?; + } + Ok(()) } - Ok(()) } // will log on error (vs. panic) @@ -205,15 +235,9 @@ impl CommonEnv { } } - pub fn xous_version(&self) -> String { - // Currently breaks in hosted mode: - // self.ticktimer.get_version() - "v0.9.11".to_string() - } - pub fn get(&mut self, key: &str) -> Result, Error> { if key.eq(CURRENT_VERSION_KEY) { - Ok(Some(self.xous_version())) + Ok(Some(self.version.clone())) } else { let mut keypath = PathBuf::new(); keypath.push(MTXCLI_DICT); @@ -383,7 +407,8 @@ impl CommonEnv { if self.filter.len() > 0 { true } else { - if let Some(new_filter) = web::get_filter(&self.user, &self.server, &self.token) { + if let Some(new_filter) = web::get_filter(&self.user, &self.server, + &self.room_id, &self.token) { self.set_debug(FILTER_KEY, &new_filter); self.filter = new_filter; true @@ -458,8 +483,9 @@ pub struct CmdEnv { impl CmdEnv { pub fn new(xns: &xous_names::XousNames) -> CmdEnv { let ticktimer = ticktimer_server::Ticktimer::new().expect("Couldn't connect to Ticktimer"); + let version = get_version(&ticktimer); log::info!("creating CommonEnv"); - let common = CommonEnv { + let mut common = CommonEnv { llio: llio::Llio::new(&xns), com: com::Com::new(&xns).expect("could't connect to COM"), codec: codec::Codec::new(&xns).expect("couldn't connect to CODEC"), @@ -477,9 +503,11 @@ impl CmdEnv { room_id: EMPTY.to_string(), filter: EMPTY.to_string(), since: EMPTY.to_string(), - first_line: true + first_line: true, + version, }; log::info!("done creating CommonEnv"); + run_migrations(&mut common); CmdEnv { common_env: common, lastverb: XousString::<256>::new(), diff --git a/apps/mtxcli/src/cmds/migrations.rs b/apps/mtxcli/src/cmds/migrations.rs new file mode 100644 index 000000000..6fa7cd953 --- /dev/null +++ b/apps/mtxcli/src/cmds/migrations.rs @@ -0,0 +1,69 @@ +//! Migrations +//! +//! Runs migrations up to the current version + +use std::io::Error; +use crate::cmds::{CommonEnv,VERSION_KEY}; + +mod v0_9_11_0120; use v0_9_11_0120::*; + +const DEFAULT_VERSION: &str = "0"; + +pub trait MigrationApi<'a> { + // created with migration_api! macro + // returns my version + fn version(&self) -> &'static str; + // checks if the migration should be applied + fn applies(&self, version: &str) -> bool; + // run the migration + fn process(&self, common: &mut CommonEnv) -> Result; +} + +// the argument to this macro is the command verb +#[macro_export] +macro_rules! migration_api { + ($version:expr) => { // NOTE the expr will have literal quotes + fn version(&self) -> &'static str { + $version + } + + fn applies(&self, version: &str) -> bool { + if version < self.version() { + true + } else { + false + } + } + }; +} + +/// Run migrations as needed +pub fn run_migrations(common: &mut CommonEnv) { + let version = common.get_default(VERSION_KEY, DEFAULT_VERSION); + if version.ne(&common.version) { + log::info!("Running migrations from version {} to {}", + version, common.version); + let mut migrations: Vec> = Vec::new(); + migrations.push(Box::new(V0_9_11_0120::new())); + for migration in migrations.iter() { + if migration.applies(&version) { + match migration.process(common) { + Ok(boolean) => { + log::info!("migration competed: {}: {}", + migration.version(), boolean); + if boolean { + common.set(VERSION_KEY, migration.version()) + .expect("cannot set _version"); + } + }, + Err(e) => { + log::error!("error running migration: {}: {:?}", + migration.version(), e); + } + } + } + } + common.set(VERSION_KEY, &common.version.clone()) + .expect("cannot set _version"); + } +} diff --git a/apps/mtxcli/src/cmds/migrations/v0_9_11_0120.rs b/apps/mtxcli/src/cmds/migrations/v0_9_11_0120.rs new file mode 100644 index 000000000..9a1151383 --- /dev/null +++ b/apps/mtxcli/src/cmds/migrations/v0_9_11_0120.rs @@ -0,0 +1,26 @@ +use std::io::Error; + +use crate::cmds::{CommonEnv,FILTER_KEY}; +use crate::cmds::migrations::MigrationApi; +use crate::migration_api; + +#[derive(Debug)] +pub struct V0_9_11_0120 { +} +impl V0_9_11_0120 { + pub fn new() -> Self { + V0_9_11_0120 { + } + } +} + +impl<'a> MigrationApi<'a> for V0_9_11_0120 { + migration_api!("v0.9.11-0120"); + + fn process(&self, common: &mut CommonEnv) -> Result { + log::info!("Running migration for: {}", self.version()); + // we need to provoke the creation of a new/updated filter + common.unset(FILTER_KEY)?; + Ok(true) + } +} diff --git a/apps/mtxcli/src/cmds/web.rs b/apps/mtxcli/src/cmds/web.rs index 6cd988972..258c09212 100644 --- a/apps/mtxcli/src/cmds/web.rs +++ b/apps/mtxcli/src/cmds/web.rs @@ -29,8 +29,8 @@ fn serialize(object: &T) -> Option { Ok(value) => { Some(value) }, - Err(_) => { - // println!("FAILED TO SERIALIZE"); + Err(e) => { + log::error!("ERROR in serialize: {:?}", e); None } } @@ -39,10 +39,6 @@ fn serialize(object: &T) -> Option { pub fn handle_response(maybe_response: Result) -> Option { match maybe_response { Ok(response) => { - // DEBUG - // if let Ok(json) = response.into_string() { - // println!("JSON = \n{}\n", json); - // None if let Ok(body) = response.into_json() { Some(body) } else { @@ -55,27 +51,22 @@ pub fn handle_response(maybe_response: Result) -> Op code (such as 400, 500 etc) */ let err_body = response.into_string().unwrap(); log::error!("ERROR code {} err_body = {}", code, err_body); - // let status_text = "the status text"; - // Err(ureq::Error::Status(code, ureq::Response::new(code, status_text, &body).unwrap())) None } - Err(_) => { - log::error!("Unknown Error"); - // Err(e) + Err(e) => { + log::error!("ERROR in handle_response: {:?}", e); None } } } pub fn get_json(url: &str) -> Result { - // println!("getting json from {}", &url); ureq::get(&url) .set(ACCEPT, ACCEPT_JSON) .call() } pub fn get_json_auth(url: &str, token:&str) -> Result { - // println!("getting json auth from {}", &url); let mut authorization = String::from(BEARER); authorization.push_str(token); ureq::get(&url) @@ -85,14 +76,12 @@ pub fn get_json_auth(url: &str, token:&str) -> Result Result { - // println!("post json to {}", &url); ureq::post(&url) .set(ACCEPT, ACCEPT_JSON) .send_string(request_body) } pub fn post_string_auth(url: &str, request_body: &str, token: &str) -> Result { - // println!("post json auth from {}", &url); let mut authorization = String::from(BEARER); authorization.push_str(token); ureq::post(&url) @@ -102,7 +91,6 @@ pub fn post_string_auth(url: &str, request_body: &str, token: &str) -> Result Result { - // println!("put json auth from {}", &url); let mut authorization = String::from(BEARER); authorization.push_str(token); ureq::put(&url) @@ -225,13 +213,17 @@ pub fn get_room_id(server: &str, room_server: &str, token: &str) -> Option, } impl EventFilter { pub fn new(limit: i32) -> Self { + let mut not_types: Vec = Vec::new(); + not_types.push("*".to_string()); EventFilter { - limit + limit, + not_types, } } } @@ -240,15 +232,19 @@ impl EventFilter { struct RoomEventFilter { limit: i32, types: Vec, + rooms: Vec, } impl RoomEventFilter { - pub fn new(limit: i32, type_0: &str) -> Self { + pub fn new(limit: i32, room_id: &str, type_0: &str) -> Self { let mut types = Vec::new(); types.push(type_0.to_string()); + let mut rooms: Vec = Vec::new(); + rooms.push(room_id.to_string()); RoomEventFilter { limit, types, + rooms, } } } @@ -256,21 +252,25 @@ impl RoomEventFilter { #[derive(Serialize, Deserialize)] struct RoomFilter { account_data: EventFilter, // Should be RoomEventFilter - state: EventFilter, // Should be StateFilter ephemeral: EventFilter, + rooms: Vec, + state: EventFilter, // Should be StateFilter timeline: RoomEventFilter, } impl RoomFilter { - pub fn new() -> Self { + pub fn new(room_id: &str) -> Self { let account_data = EventFilter::new(0); - let state = EventFilter::new(0); let ephemeral = EventFilter::new(0); - let timeline = RoomEventFilter::new(10, "m.room.message"); + let mut rooms: Vec = Vec::new(); + rooms.push(room_id.to_string()); + let state = EventFilter::new(0); + let timeline = RoomEventFilter::new(10, room_id, "m.room.message"); RoomFilter { account_data, - state, ephemeral, + rooms, + state, timeline, } } @@ -278,38 +278,44 @@ impl RoomFilter { #[derive(Serialize, Deserialize)] struct FilterRequest { - presence: EventFilter, account_data: EventFilter, + event_fields: Vec, + presence: EventFilter, room: RoomFilter, } impl FilterRequest { - pub fn new() -> Self { - let presence = EventFilter::new(0); + pub fn new(room_id: &str) -> Self { let account_data = EventFilter::new(0); - let room = RoomFilter::new(); + let mut event_fields: Vec = Vec::new(); + event_fields.push("type".to_string()); + event_fields.push("sender".to_string()); + event_fields.push("content.body".to_string()); + let presence = EventFilter::new(0); + let room = RoomFilter::new(room_id); FilterRequest { - presence, account_data, - room + event_fields, + presence, + room, } } } -pub fn get_filter(user: &str, server: &str, token: &str) -> Option { +pub fn get_filter(user: &str, server: &str, room_id: &str, token: &str) + -> Option { let user_encoded = url::encode(user); let mut url = String::from(server); - url.push_str("/_matrix/client/r0/user/"); + url.push_str("/_matrix/client/v3/user/"); url.push_str(&user_encoded); url.push_str("/filter"); log::debug!("get_filter = {}", url); - let filter_request = FilterRequest::new(); + let filter_request = FilterRequest::new(room_id); if let Some(request_body) = serialize(&filter_request) { - // println!("filter_request = {}", request_body); if let Some(value) = handle_response(post_string_auth(&url, &request_body, token)) { if let Value::Object(body) = value { if let Some(Value::String(filter_id)) = body.get("filter_id") { - // log::debug!("filter_id = {}", filter_id); + log::debug!("filter_id = {}", filter_id); Some(filter_id.to_string()) } else { log::error!("invalid response for get_filter"); @@ -384,7 +390,6 @@ pub fn client_sync(server: &str, filter: &str, since: &str, timeout: i32, } log::debug!("client_sync = {}", url); if let Some(value) = handle_response(get_json_auth(&url, token)) { - // println!("SYNC = {:?}", value); if let Value::Object(body) = value { if let Some(Value::String(next_batch)) = body.get("next_batch") { Some((next_batch.to_string(), get_messages(body, room_id))) @@ -429,10 +434,8 @@ pub fn send_message(server: &str, room_id: &str, text: &str, txn_id: &str, token log::debug!("send_message = {}", url); let message_request = MessageRequest::new(text); if let Some(request_body) = serialize(&message_request) { - // println!("request_body = {}", request_body); if let Some(value) = handle_response(put_string_auth(&url, &request_body, token)) { if let Value::Object(_body) = value { - // println!("SENT = {:?}", body); true } else { log::error!("invalid response for send_message");