WIP: Fix mtxcli excessive memory usage #287

Adds migrations framework
- Migration for v0.9.11-0120: update the filter

Filter
- narrow filter to only the fields and events for the current room

Introduces "read only" variables (e.g. __version).

Signed-off-by: Tom Marble <tmarble@info9.net>
This commit is contained in:
Tom Marble 2023-01-13 16:52:28 -06:00 committed by Sean Cross
parent b4566bc22e
commit feffc2e5be
5 changed files with 205 additions and 79 deletions

View file

@ -1,6 +1,6 @@
[package]
name = "mtxcli"
version = "0.1.0"
version = "0.6.0"
authors = ["bunnie <bunnie@kosagi.com>"]
edition = "2018"
description = "Matrix chat"

View file

@ -5,10 +5,11 @@ use locales::t;
use std::collections::HashMap;
use std::fs::File;
use std::io::{Read, Write as StdWrite, Error};
use std::io::{Read, Write as StdWrite, Error, ErrorKind};
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH}; // to help gen_txn_id
mod migrations; use migrations::run_migrations;
mod url;
mod web;
@ -35,6 +36,24 @@ const SERVER_MATRIX: &str = "https://matrix.org";
const EMPTY: &str = "";
const MTX_TIMEOUT: i32 = 300; // ms
/// Returns a version string which is more likely to string compare
/// correctly vs. another version. FFI please see
/// https://git-scm.com/docs/git-describe#_examples
fn get_version(ticktimer: &ticktimer_server::Ticktimer) -> String {
let xous_version = ticktimer.get_version();
let v: Vec<&str> = xous_version.split('-').collect();
if v.len() > 2 {
let n = v[1].parse::<usize>().expect("could not parse version");
let version = format!("{}-{:04}",v[0], n);
log::info!("version={}=", version);
version
} else {
log::error!("ERROR, couldn't find version from xous_version: {}",
xous_version);
xous_version
}
}
/////////////////////////// Common items to all commands
pub trait ShellCmdApi<'a> {
// user implemented:
@ -90,6 +109,7 @@ pub struct CommonEnv {
filter: String,
since: String,
first_line: bool,
version: String,
}
impl CommonEnv {
@ -103,24 +123,29 @@ impl CommonEnv {
}
pub fn set(&mut self, key: &str, value: &str) -> Result<(), Error> {
log::info!("set '{}' = '{}'", key, value);
let mut keypath = PathBuf::new();
keypath.push(MTXCLI_DICT);
if std::fs::metadata(&keypath).is_ok() { // keypath exists
log::info!("dict '{}' exists", MTXCLI_DICT);
if key.starts_with("__") {
Err(Error::new(ErrorKind::PermissionDenied,
"may not set a variable beginning with __ "))
} else {
log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT);
std::fs::create_dir_all(&keypath)?;
log::info!("set '{}' = '{}'", key, value);
let mut keypath = PathBuf::new();
keypath.push(MTXCLI_DICT);
if std::fs::metadata(&keypath).is_ok() { // keypath exists
log::info!("dict '{}' exists", MTXCLI_DICT);
} else {
log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT);
std::fs::create_dir_all(&keypath)?;
}
keypath.push(key);
File::create(keypath)?.write_all(value.as_bytes())?;
match key { // special case side effects
USER_KEY => { self.set_user(value); }
PASSWORD_KEY => { self.set_password(); }
ROOM_KEY => { self.set_room(); }
_ => { }
}
Ok(())
}
keypath.push(key);
File::create(keypath)?.write_all(value.as_bytes())?;
match key { // special case side effects
USER_KEY => { self.set_user(value); }
PASSWORD_KEY => { self.set_password(); }
ROOM_KEY => { self.set_room(); }
_ => { }
}
Ok(())
}
// will log on error (vs. panic)
@ -174,22 +199,27 @@ impl CommonEnv {
}
pub fn unset(&mut self, key: &str) -> Result<(), Error> {
// log::info!("unset '{}'", key);
let mut keypath = PathBuf::new();
keypath.push(MTXCLI_DICT);
if std::fs::metadata(&keypath).is_ok() { // keypath exists
log::info!("dict '{}' exists", MTXCLI_DICT);
if key.starts_with("__") {
Err(Error::new(ErrorKind::PermissionDenied,
"may not unset a variable beginning with __ "))
} else {
log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT);
std::fs::create_dir_all(&keypath)?;
}
keypath.push(key);
if std::fs::metadata(&keypath).is_ok() { // keypath exists
log::info!("dict:key = '{}:{}' exists.. deleting it", MTXCLI_DICT, key);
log::info!("unset '{}'", key);
let mut keypath = PathBuf::new();
keypath.push(MTXCLI_DICT);
if std::fs::metadata(&keypath).is_ok() { // keypath exists
log::info!("dict '{}' exists", MTXCLI_DICT);
} else {
log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT);
std::fs::create_dir_all(&keypath)?;
}
keypath.push(key);
if std::fs::metadata(&keypath).is_ok() { // keypath exists
log::info!("dict:key = '{}:{}' exists.. deleting it", MTXCLI_DICT, key);
std::fs::remove_file(keypath)?;
std::fs::remove_file(keypath)?;
}
Ok(())
}
Ok(())
}
// will log on error (vs. panic)
@ -205,15 +235,9 @@ impl CommonEnv {
}
}
pub fn xous_version(&self) -> String {
// Currently breaks in hosted mode:
// self.ticktimer.get_version()
"v0.9.11".to_string()
}
pub fn get(&mut self, key: &str) -> Result<Option<String>, Error> {
if key.eq(CURRENT_VERSION_KEY) {
Ok(Some(self.xous_version()))
Ok(Some(self.version.clone()))
} else {
let mut keypath = PathBuf::new();
keypath.push(MTXCLI_DICT);
@ -383,7 +407,8 @@ impl CommonEnv {
if self.filter.len() > 0 {
true
} else {
if let Some(new_filter) = web::get_filter(&self.user, &self.server, &self.token) {
if let Some(new_filter) = web::get_filter(&self.user, &self.server,
&self.room_id, &self.token) {
self.set_debug(FILTER_KEY, &new_filter);
self.filter = new_filter;
true
@ -458,8 +483,9 @@ pub struct CmdEnv {
impl CmdEnv {
pub fn new(xns: &xous_names::XousNames) -> CmdEnv {
let ticktimer = ticktimer_server::Ticktimer::new().expect("Couldn't connect to Ticktimer");
let version = get_version(&ticktimer);
log::info!("creating CommonEnv");
let common = CommonEnv {
let mut common = CommonEnv {
llio: llio::Llio::new(&xns),
com: com::Com::new(&xns).expect("could't connect to COM"),
codec: codec::Codec::new(&xns).expect("couldn't connect to CODEC"),
@ -477,9 +503,11 @@ impl CmdEnv {
room_id: EMPTY.to_string(),
filter: EMPTY.to_string(),
since: EMPTY.to_string(),
first_line: true
first_line: true,
version,
};
log::info!("done creating CommonEnv");
run_migrations(&mut common);
CmdEnv {
common_env: common,
lastverb: XousString::<256>::new(),

View file

@ -0,0 +1,69 @@
//! Migrations
//!
//! Runs migrations up to the current version
use std::io::Error;
use crate::cmds::{CommonEnv,VERSION_KEY};
mod v0_9_11_0120; use v0_9_11_0120::*;
const DEFAULT_VERSION: &str = "0";
pub trait MigrationApi<'a> {
// created with migration_api! macro
// returns my version
fn version(&self) -> &'static str;
// checks if the migration should be applied
fn applies(&self, version: &str) -> bool;
// run the migration
fn process(&self, common: &mut CommonEnv) -> Result<bool, Error>;
}
// the argument to this macro is the command verb
#[macro_export]
macro_rules! migration_api {
($version:expr) => { // NOTE the expr will have literal quotes
fn version(&self) -> &'static str {
$version
}
fn applies(&self, version: &str) -> bool {
if version < self.version() {
true
} else {
false
}
}
};
}
/// Run migrations as needed
pub fn run_migrations(common: &mut CommonEnv) {
let version = common.get_default(VERSION_KEY, DEFAULT_VERSION);
if version.ne(&common.version) {
log::info!("Running migrations from version {} to {}",
version, common.version);
let mut migrations: Vec<Box<dyn MigrationApi>> = Vec::new();
migrations.push(Box::new(V0_9_11_0120::new()));
for migration in migrations.iter() {
if migration.applies(&version) {
match migration.process(common) {
Ok(boolean) => {
log::info!("migration competed: {}: {}",
migration.version(), boolean);
if boolean {
common.set(VERSION_KEY, migration.version())
.expect("cannot set _version");
}
},
Err(e) => {
log::error!("error running migration: {}: {:?}",
migration.version(), e);
}
}
}
}
common.set(VERSION_KEY, &common.version.clone())
.expect("cannot set _version");
}
}

View file

@ -0,0 +1,26 @@
use std::io::Error;
use crate::cmds::{CommonEnv,FILTER_KEY};
use crate::cmds::migrations::MigrationApi;
use crate::migration_api;
#[derive(Debug)]
pub struct V0_9_11_0120 {
}
impl V0_9_11_0120 {
pub fn new() -> Self {
V0_9_11_0120 {
}
}
}
impl<'a> MigrationApi<'a> for V0_9_11_0120 {
migration_api!("v0.9.11-0120");
fn process(&self, common: &mut CommonEnv) -> Result<bool, Error> {
log::info!("Running migration for: {}", self.version());
// we need to provoke the creation of a new/updated filter
common.unset(FILTER_KEY)?;
Ok(true)
}
}

View file

@ -29,8 +29,8 @@ fn serialize<T: ?Sized + Serialize>(object: &T) -> Option<String> {
Ok(value) => {
Some(value)
},
Err(_) => {
// println!("FAILED TO SERIALIZE");
Err(e) => {
log::error!("ERROR in serialize: {:?}", e);
None
}
}
@ -39,10 +39,6 @@ fn serialize<T: ?Sized + Serialize>(object: &T) -> Option<String> {
pub fn handle_response(maybe_response: Result<ureq::Response,ureq::Error>) -> Option<Value> {
match maybe_response {
Ok(response) => {
// DEBUG
// if let Ok(json) = response.into_string() {
// println!("JSON = \n{}\n", json);
// None
if let Ok(body) = response.into_json() {
Some(body)
} else {
@ -55,27 +51,22 @@ pub fn handle_response(maybe_response: Result<ureq::Response,ureq::Error>) -> Op
code (such as 400, 500 etc) */
let err_body = response.into_string().unwrap();
log::error!("ERROR code {} err_body = {}", code, err_body);
// let status_text = "the status text";
// Err(ureq::Error::Status(code, ureq::Response::new(code, status_text, &body).unwrap()))
None
}
Err(_) => {
log::error!("Unknown Error");
// Err(e)
Err(e) => {
log::error!("ERROR in handle_response: {:?}", e);
None
}
}
}
pub fn get_json(url: &str) -> Result<ureq::Response, ureq::Error> {
// println!("getting json from {}", &url);
ureq::get(&url)
.set(ACCEPT, ACCEPT_JSON)
.call()
}
pub fn get_json_auth(url: &str, token:&str) -> Result<ureq::Response, ureq::Error> {
// println!("getting json auth from {}", &url);
let mut authorization = String::from(BEARER);
authorization.push_str(token);
ureq::get(&url)
@ -85,14 +76,12 @@ pub fn get_json_auth(url: &str, token:&str) -> Result<ureq::Response, ureq::Erro
}
pub fn post_string(url: &str, request_body: &str) -> Result<ureq::Response, ureq::Error> {
// println!("post json to {}", &url);
ureq::post(&url)
.set(ACCEPT, ACCEPT_JSON)
.send_string(request_body)
}
pub fn post_string_auth(url: &str, request_body: &str, token: &str) -> Result<ureq::Response, ureq::Error> {
// println!("post json auth from {}", &url);
let mut authorization = String::from(BEARER);
authorization.push_str(token);
ureq::post(&url)
@ -102,7 +91,6 @@ pub fn post_string_auth(url: &str, request_body: &str, token: &str) -> Result<ur
}
pub fn put_string_auth(url: &str, request_body: &str, token: &str) -> Result<ureq::Response, ureq::Error> {
// println!("put json auth from {}", &url);
let mut authorization = String::from(BEARER);
authorization.push_str(token);
ureq::put(&url)
@ -225,13 +213,17 @@ pub fn get_room_id(server: &str, room_server: &str, token: &str) -> Option<Strin
#[derive(Serialize, Deserialize)]
struct EventFilter {
limit: i32
limit: i32,
not_types: Vec<String>,
}
impl EventFilter {
pub fn new(limit: i32) -> Self {
let mut not_types: Vec<String> = Vec::new();
not_types.push("*".to_string());
EventFilter {
limit
limit,
not_types,
}
}
}
@ -240,15 +232,19 @@ impl EventFilter {
struct RoomEventFilter {
limit: i32,
types: Vec<String>,
rooms: Vec<String>,
}
impl RoomEventFilter {
pub fn new(limit: i32, type_0: &str) -> Self {
pub fn new(limit: i32, room_id: &str, type_0: &str) -> Self {
let mut types = Vec::new();
types.push(type_0.to_string());
let mut rooms: Vec<String> = Vec::new();
rooms.push(room_id.to_string());
RoomEventFilter {
limit,
types,
rooms,
}
}
}
@ -256,21 +252,25 @@ impl RoomEventFilter {
#[derive(Serialize, Deserialize)]
struct RoomFilter {
account_data: EventFilter, // Should be RoomEventFilter
state: EventFilter, // Should be StateFilter
ephemeral: EventFilter,
rooms: Vec<String>,
state: EventFilter, // Should be StateFilter
timeline: RoomEventFilter,
}
impl RoomFilter {
pub fn new() -> Self {
pub fn new(room_id: &str) -> Self {
let account_data = EventFilter::new(0);
let state = EventFilter::new(0);
let ephemeral = EventFilter::new(0);
let timeline = RoomEventFilter::new(10, "m.room.message");
let mut rooms: Vec<String> = Vec::new();
rooms.push(room_id.to_string());
let state = EventFilter::new(0);
let timeline = RoomEventFilter::new(10, room_id, "m.room.message");
RoomFilter {
account_data,
state,
ephemeral,
rooms,
state,
timeline,
}
}
@ -278,38 +278,44 @@ impl RoomFilter {
#[derive(Serialize, Deserialize)]
struct FilterRequest {
presence: EventFilter,
account_data: EventFilter,
event_fields: Vec<String>,
presence: EventFilter,
room: RoomFilter,
}
impl FilterRequest {
pub fn new() -> Self {
let presence = EventFilter::new(0);
pub fn new(room_id: &str) -> Self {
let account_data = EventFilter::new(0);
let room = RoomFilter::new();
let mut event_fields: Vec<String> = Vec::new();
event_fields.push("type".to_string());
event_fields.push("sender".to_string());
event_fields.push("content.body".to_string());
let presence = EventFilter::new(0);
let room = RoomFilter::new(room_id);
FilterRequest {
presence,
account_data,
room
event_fields,
presence,
room,
}
}
}
pub fn get_filter(user: &str, server: &str, token: &str) -> Option<String> {
pub fn get_filter(user: &str, server: &str, room_id: &str, token: &str)
-> Option<String> {
let user_encoded = url::encode(user);
let mut url = String::from(server);
url.push_str("/_matrix/client/r0/user/");
url.push_str("/_matrix/client/v3/user/");
url.push_str(&user_encoded);
url.push_str("/filter");
log::debug!("get_filter = {}", url);
let filter_request = FilterRequest::new();
let filter_request = FilterRequest::new(room_id);
if let Some(request_body) = serialize(&filter_request) {
// println!("filter_request = {}", request_body);
if let Some(value) = handle_response(post_string_auth(&url, &request_body, token)) {
if let Value::Object(body) = value {
if let Some(Value::String(filter_id)) = body.get("filter_id") {
// log::debug!("filter_id = {}", filter_id);
log::debug!("filter_id = {}", filter_id);
Some(filter_id.to_string())
} else {
log::error!("invalid response for get_filter");
@ -384,7 +390,6 @@ pub fn client_sync(server: &str, filter: &str, since: &str, timeout: i32,
}
log::debug!("client_sync = {}", url);
if let Some(value) = handle_response(get_json_auth(&url, token)) {
// println!("SYNC = {:?}", value);
if let Value::Object(body) = value {
if let Some(Value::String(next_batch)) = body.get("next_batch") {
Some((next_batch.to_string(), get_messages(body, room_id)))
@ -429,10 +434,8 @@ pub fn send_message(server: &str, room_id: &str, text: &str, txn_id: &str, token
log::debug!("send_message = {}", url);
let message_request = MessageRequest::new(text);
if let Some(request_body) = serialize(&message_request) {
// println!("request_body = {}", request_body);
if let Some(value) = handle_response(put_string_auth(&url, &request_body, token)) {
if let Value::Object(_body) = value {
// println!("SENT = {:?}", body);
true
} else {
log::error!("invalid response for send_message");