mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-15 08:30:02 +00:00
WIP: Fix mtxcli excessive memory usage #287
Adds migrations framework - Migration for v0.9.11-0120: update the filter Filter - narrow filter to only the fields and events for the current room Introduces "read only" variables (e.g. __version). Signed-off-by: Tom Marble <tmarble@info9.net>
This commit is contained in:
parent
b4566bc22e
commit
feffc2e5be
5 changed files with 205 additions and 79 deletions
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "mtxcli"
|
||||
version = "0.1.0"
|
||||
version = "0.6.0"
|
||||
authors = ["bunnie <bunnie@kosagi.com>"]
|
||||
edition = "2018"
|
||||
description = "Matrix chat"
|
||||
|
|
|
|||
|
|
@ -5,10 +5,11 @@ use locales::t;
|
|||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Write as StdWrite, Error};
|
||||
use std::io::{Read, Write as StdWrite, Error, ErrorKind};
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH}; // to help gen_txn_id
|
||||
|
||||
mod migrations; use migrations::run_migrations;
|
||||
mod url;
|
||||
mod web;
|
||||
|
||||
|
|
@ -35,6 +36,24 @@ const SERVER_MATRIX: &str = "https://matrix.org";
|
|||
const EMPTY: &str = "";
|
||||
const MTX_TIMEOUT: i32 = 300; // ms
|
||||
|
||||
/// Returns a version string which is more likely to string compare
|
||||
/// correctly vs. another version. FFI please see
|
||||
/// https://git-scm.com/docs/git-describe#_examples
|
||||
fn get_version(ticktimer: &ticktimer_server::Ticktimer) -> String {
|
||||
let xous_version = ticktimer.get_version();
|
||||
let v: Vec<&str> = xous_version.split('-').collect();
|
||||
if v.len() > 2 {
|
||||
let n = v[1].parse::<usize>().expect("could not parse version");
|
||||
let version = format!("{}-{:04}",v[0], n);
|
||||
log::info!("version={}=", version);
|
||||
version
|
||||
} else {
|
||||
log::error!("ERROR, couldn't find version from xous_version: {}",
|
||||
xous_version);
|
||||
xous_version
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////// Common items to all commands
|
||||
pub trait ShellCmdApi<'a> {
|
||||
// user implemented:
|
||||
|
|
@ -90,6 +109,7 @@ pub struct CommonEnv {
|
|||
filter: String,
|
||||
since: String,
|
||||
first_line: bool,
|
||||
version: String,
|
||||
}
|
||||
impl CommonEnv {
|
||||
|
||||
|
|
@ -103,24 +123,29 @@ impl CommonEnv {
|
|||
}
|
||||
|
||||
pub fn set(&mut self, key: &str, value: &str) -> Result<(), Error> {
|
||||
log::info!("set '{}' = '{}'", key, value);
|
||||
let mut keypath = PathBuf::new();
|
||||
keypath.push(MTXCLI_DICT);
|
||||
if std::fs::metadata(&keypath).is_ok() { // keypath exists
|
||||
log::info!("dict '{}' exists", MTXCLI_DICT);
|
||||
if key.starts_with("__") {
|
||||
Err(Error::new(ErrorKind::PermissionDenied,
|
||||
"may not set a variable beginning with __ "))
|
||||
} else {
|
||||
log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT);
|
||||
std::fs::create_dir_all(&keypath)?;
|
||||
log::info!("set '{}' = '{}'", key, value);
|
||||
let mut keypath = PathBuf::new();
|
||||
keypath.push(MTXCLI_DICT);
|
||||
if std::fs::metadata(&keypath).is_ok() { // keypath exists
|
||||
log::info!("dict '{}' exists", MTXCLI_DICT);
|
||||
} else {
|
||||
log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT);
|
||||
std::fs::create_dir_all(&keypath)?;
|
||||
}
|
||||
keypath.push(key);
|
||||
File::create(keypath)?.write_all(value.as_bytes())?;
|
||||
match key { // special case side effects
|
||||
USER_KEY => { self.set_user(value); }
|
||||
PASSWORD_KEY => { self.set_password(); }
|
||||
ROOM_KEY => { self.set_room(); }
|
||||
_ => { }
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
keypath.push(key);
|
||||
File::create(keypath)?.write_all(value.as_bytes())?;
|
||||
match key { // special case side effects
|
||||
USER_KEY => { self.set_user(value); }
|
||||
PASSWORD_KEY => { self.set_password(); }
|
||||
ROOM_KEY => { self.set_room(); }
|
||||
_ => { }
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// will log on error (vs. panic)
|
||||
|
|
@ -174,22 +199,27 @@ impl CommonEnv {
|
|||
}
|
||||
|
||||
pub fn unset(&mut self, key: &str) -> Result<(), Error> {
|
||||
// log::info!("unset '{}'", key);
|
||||
let mut keypath = PathBuf::new();
|
||||
keypath.push(MTXCLI_DICT);
|
||||
if std::fs::metadata(&keypath).is_ok() { // keypath exists
|
||||
log::info!("dict '{}' exists", MTXCLI_DICT);
|
||||
if key.starts_with("__") {
|
||||
Err(Error::new(ErrorKind::PermissionDenied,
|
||||
"may not unset a variable beginning with __ "))
|
||||
} else {
|
||||
log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT);
|
||||
std::fs::create_dir_all(&keypath)?;
|
||||
}
|
||||
keypath.push(key);
|
||||
if std::fs::metadata(&keypath).is_ok() { // keypath exists
|
||||
log::info!("dict:key = '{}:{}' exists.. deleting it", MTXCLI_DICT, key);
|
||||
log::info!("unset '{}'", key);
|
||||
let mut keypath = PathBuf::new();
|
||||
keypath.push(MTXCLI_DICT);
|
||||
if std::fs::metadata(&keypath).is_ok() { // keypath exists
|
||||
log::info!("dict '{}' exists", MTXCLI_DICT);
|
||||
} else {
|
||||
log::info!("dict '{}' does NOT exist.. creating it", MTXCLI_DICT);
|
||||
std::fs::create_dir_all(&keypath)?;
|
||||
}
|
||||
keypath.push(key);
|
||||
if std::fs::metadata(&keypath).is_ok() { // keypath exists
|
||||
log::info!("dict:key = '{}:{}' exists.. deleting it", MTXCLI_DICT, key);
|
||||
|
||||
std::fs::remove_file(keypath)?;
|
||||
std::fs::remove_file(keypath)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// will log on error (vs. panic)
|
||||
|
|
@ -205,15 +235,9 @@ impl CommonEnv {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn xous_version(&self) -> String {
|
||||
// Currently breaks in hosted mode:
|
||||
// self.ticktimer.get_version()
|
||||
"v0.9.11".to_string()
|
||||
}
|
||||
|
||||
pub fn get(&mut self, key: &str) -> Result<Option<String>, Error> {
|
||||
if key.eq(CURRENT_VERSION_KEY) {
|
||||
Ok(Some(self.xous_version()))
|
||||
Ok(Some(self.version.clone()))
|
||||
} else {
|
||||
let mut keypath = PathBuf::new();
|
||||
keypath.push(MTXCLI_DICT);
|
||||
|
|
@ -383,7 +407,8 @@ impl CommonEnv {
|
|||
if self.filter.len() > 0 {
|
||||
true
|
||||
} else {
|
||||
if let Some(new_filter) = web::get_filter(&self.user, &self.server, &self.token) {
|
||||
if let Some(new_filter) = web::get_filter(&self.user, &self.server,
|
||||
&self.room_id, &self.token) {
|
||||
self.set_debug(FILTER_KEY, &new_filter);
|
||||
self.filter = new_filter;
|
||||
true
|
||||
|
|
@ -458,8 +483,9 @@ pub struct CmdEnv {
|
|||
impl CmdEnv {
|
||||
pub fn new(xns: &xous_names::XousNames) -> CmdEnv {
|
||||
let ticktimer = ticktimer_server::Ticktimer::new().expect("Couldn't connect to Ticktimer");
|
||||
let version = get_version(&ticktimer);
|
||||
log::info!("creating CommonEnv");
|
||||
let common = CommonEnv {
|
||||
let mut common = CommonEnv {
|
||||
llio: llio::Llio::new(&xns),
|
||||
com: com::Com::new(&xns).expect("could't connect to COM"),
|
||||
codec: codec::Codec::new(&xns).expect("couldn't connect to CODEC"),
|
||||
|
|
@ -477,9 +503,11 @@ impl CmdEnv {
|
|||
room_id: EMPTY.to_string(),
|
||||
filter: EMPTY.to_string(),
|
||||
since: EMPTY.to_string(),
|
||||
first_line: true
|
||||
first_line: true,
|
||||
version,
|
||||
};
|
||||
log::info!("done creating CommonEnv");
|
||||
run_migrations(&mut common);
|
||||
CmdEnv {
|
||||
common_env: common,
|
||||
lastverb: XousString::<256>::new(),
|
||||
|
|
|
|||
69
apps/mtxcli/src/cmds/migrations.rs
Normal file
69
apps/mtxcli/src/cmds/migrations.rs
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
//! Migrations
|
||||
//!
|
||||
//! Runs migrations up to the current version
|
||||
|
||||
use std::io::Error;
|
||||
use crate::cmds::{CommonEnv,VERSION_KEY};
|
||||
|
||||
mod v0_9_11_0120; use v0_9_11_0120::*;
|
||||
|
||||
const DEFAULT_VERSION: &str = "0";
|
||||
|
||||
pub trait MigrationApi<'a> {
|
||||
// created with migration_api! macro
|
||||
// returns my version
|
||||
fn version(&self) -> &'static str;
|
||||
// checks if the migration should be applied
|
||||
fn applies(&self, version: &str) -> bool;
|
||||
// run the migration
|
||||
fn process(&self, common: &mut CommonEnv) -> Result<bool, Error>;
|
||||
}
|
||||
|
||||
// the argument to this macro is the command verb
|
||||
#[macro_export]
|
||||
macro_rules! migration_api {
|
||||
($version:expr) => { // NOTE the expr will have literal quotes
|
||||
fn version(&self) -> &'static str {
|
||||
$version
|
||||
}
|
||||
|
||||
fn applies(&self, version: &str) -> bool {
|
||||
if version < self.version() {
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Run migrations as needed
|
||||
pub fn run_migrations(common: &mut CommonEnv) {
|
||||
let version = common.get_default(VERSION_KEY, DEFAULT_VERSION);
|
||||
if version.ne(&common.version) {
|
||||
log::info!("Running migrations from version {} to {}",
|
||||
version, common.version);
|
||||
let mut migrations: Vec<Box<dyn MigrationApi>> = Vec::new();
|
||||
migrations.push(Box::new(V0_9_11_0120::new()));
|
||||
for migration in migrations.iter() {
|
||||
if migration.applies(&version) {
|
||||
match migration.process(common) {
|
||||
Ok(boolean) => {
|
||||
log::info!("migration competed: {}: {}",
|
||||
migration.version(), boolean);
|
||||
if boolean {
|
||||
common.set(VERSION_KEY, migration.version())
|
||||
.expect("cannot set _version");
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
log::error!("error running migration: {}: {:?}",
|
||||
migration.version(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
common.set(VERSION_KEY, &common.version.clone())
|
||||
.expect("cannot set _version");
|
||||
}
|
||||
}
|
||||
26
apps/mtxcli/src/cmds/migrations/v0_9_11_0120.rs
Normal file
26
apps/mtxcli/src/cmds/migrations/v0_9_11_0120.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
use std::io::Error;
|
||||
|
||||
use crate::cmds::{CommonEnv,FILTER_KEY};
|
||||
use crate::cmds::migrations::MigrationApi;
|
||||
use crate::migration_api;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct V0_9_11_0120 {
|
||||
}
|
||||
impl V0_9_11_0120 {
|
||||
pub fn new() -> Self {
|
||||
V0_9_11_0120 {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MigrationApi<'a> for V0_9_11_0120 {
|
||||
migration_api!("v0.9.11-0120");
|
||||
|
||||
fn process(&self, common: &mut CommonEnv) -> Result<bool, Error> {
|
||||
log::info!("Running migration for: {}", self.version());
|
||||
// we need to provoke the creation of a new/updated filter
|
||||
common.unset(FILTER_KEY)?;
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
|
@ -29,8 +29,8 @@ fn serialize<T: ?Sized + Serialize>(object: &T) -> Option<String> {
|
|||
Ok(value) => {
|
||||
Some(value)
|
||||
},
|
||||
Err(_) => {
|
||||
// println!("FAILED TO SERIALIZE");
|
||||
Err(e) => {
|
||||
log::error!("ERROR in serialize: {:?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
|
@ -39,10 +39,6 @@ fn serialize<T: ?Sized + Serialize>(object: &T) -> Option<String> {
|
|||
pub fn handle_response(maybe_response: Result<ureq::Response,ureq::Error>) -> Option<Value> {
|
||||
match maybe_response {
|
||||
Ok(response) => {
|
||||
// DEBUG
|
||||
// if let Ok(json) = response.into_string() {
|
||||
// println!("JSON = \n{}\n", json);
|
||||
// None
|
||||
if let Ok(body) = response.into_json() {
|
||||
Some(body)
|
||||
} else {
|
||||
|
|
@ -55,27 +51,22 @@ pub fn handle_response(maybe_response: Result<ureq::Response,ureq::Error>) -> Op
|
|||
code (such as 400, 500 etc) */
|
||||
let err_body = response.into_string().unwrap();
|
||||
log::error!("ERROR code {} err_body = {}", code, err_body);
|
||||
// let status_text = "the status text";
|
||||
// Err(ureq::Error::Status(code, ureq::Response::new(code, status_text, &body).unwrap()))
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
log::error!("Unknown Error");
|
||||
// Err(e)
|
||||
Err(e) => {
|
||||
log::error!("ERROR in handle_response: {:?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_json(url: &str) -> Result<ureq::Response, ureq::Error> {
|
||||
// println!("getting json from {}", &url);
|
||||
ureq::get(&url)
|
||||
.set(ACCEPT, ACCEPT_JSON)
|
||||
.call()
|
||||
}
|
||||
|
||||
pub fn get_json_auth(url: &str, token:&str) -> Result<ureq::Response, ureq::Error> {
|
||||
// println!("getting json auth from {}", &url);
|
||||
let mut authorization = String::from(BEARER);
|
||||
authorization.push_str(token);
|
||||
ureq::get(&url)
|
||||
|
|
@ -85,14 +76,12 @@ pub fn get_json_auth(url: &str, token:&str) -> Result<ureq::Response, ureq::Erro
|
|||
}
|
||||
|
||||
pub fn post_string(url: &str, request_body: &str) -> Result<ureq::Response, ureq::Error> {
|
||||
// println!("post json to {}", &url);
|
||||
ureq::post(&url)
|
||||
.set(ACCEPT, ACCEPT_JSON)
|
||||
.send_string(request_body)
|
||||
}
|
||||
|
||||
pub fn post_string_auth(url: &str, request_body: &str, token: &str) -> Result<ureq::Response, ureq::Error> {
|
||||
// println!("post json auth from {}", &url);
|
||||
let mut authorization = String::from(BEARER);
|
||||
authorization.push_str(token);
|
||||
ureq::post(&url)
|
||||
|
|
@ -102,7 +91,6 @@ pub fn post_string_auth(url: &str, request_body: &str, token: &str) -> Result<ur
|
|||
}
|
||||
|
||||
pub fn put_string_auth(url: &str, request_body: &str, token: &str) -> Result<ureq::Response, ureq::Error> {
|
||||
// println!("put json auth from {}", &url);
|
||||
let mut authorization = String::from(BEARER);
|
||||
authorization.push_str(token);
|
||||
ureq::put(&url)
|
||||
|
|
@ -225,13 +213,17 @@ pub fn get_room_id(server: &str, room_server: &str, token: &str) -> Option<Strin
|
|||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct EventFilter {
|
||||
limit: i32
|
||||
limit: i32,
|
||||
not_types: Vec<String>,
|
||||
}
|
||||
|
||||
impl EventFilter {
|
||||
pub fn new(limit: i32) -> Self {
|
||||
let mut not_types: Vec<String> = Vec::new();
|
||||
not_types.push("*".to_string());
|
||||
EventFilter {
|
||||
limit
|
||||
limit,
|
||||
not_types,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -240,15 +232,19 @@ impl EventFilter {
|
|||
struct RoomEventFilter {
|
||||
limit: i32,
|
||||
types: Vec<String>,
|
||||
rooms: Vec<String>,
|
||||
}
|
||||
|
||||
impl RoomEventFilter {
|
||||
pub fn new(limit: i32, type_0: &str) -> Self {
|
||||
pub fn new(limit: i32, room_id: &str, type_0: &str) -> Self {
|
||||
let mut types = Vec::new();
|
||||
types.push(type_0.to_string());
|
||||
let mut rooms: Vec<String> = Vec::new();
|
||||
rooms.push(room_id.to_string());
|
||||
RoomEventFilter {
|
||||
limit,
|
||||
types,
|
||||
rooms,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -256,21 +252,25 @@ impl RoomEventFilter {
|
|||
#[derive(Serialize, Deserialize)]
|
||||
struct RoomFilter {
|
||||
account_data: EventFilter, // Should be RoomEventFilter
|
||||
state: EventFilter, // Should be StateFilter
|
||||
ephemeral: EventFilter,
|
||||
rooms: Vec<String>,
|
||||
state: EventFilter, // Should be StateFilter
|
||||
timeline: RoomEventFilter,
|
||||
}
|
||||
|
||||
impl RoomFilter {
|
||||
pub fn new() -> Self {
|
||||
pub fn new(room_id: &str) -> Self {
|
||||
let account_data = EventFilter::new(0);
|
||||
let state = EventFilter::new(0);
|
||||
let ephemeral = EventFilter::new(0);
|
||||
let timeline = RoomEventFilter::new(10, "m.room.message");
|
||||
let mut rooms: Vec<String> = Vec::new();
|
||||
rooms.push(room_id.to_string());
|
||||
let state = EventFilter::new(0);
|
||||
let timeline = RoomEventFilter::new(10, room_id, "m.room.message");
|
||||
RoomFilter {
|
||||
account_data,
|
||||
state,
|
||||
ephemeral,
|
||||
rooms,
|
||||
state,
|
||||
timeline,
|
||||
}
|
||||
}
|
||||
|
|
@ -278,38 +278,44 @@ impl RoomFilter {
|
|||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct FilterRequest {
|
||||
presence: EventFilter,
|
||||
account_data: EventFilter,
|
||||
event_fields: Vec<String>,
|
||||
presence: EventFilter,
|
||||
room: RoomFilter,
|
||||
}
|
||||
|
||||
impl FilterRequest {
|
||||
pub fn new() -> Self {
|
||||
let presence = EventFilter::new(0);
|
||||
pub fn new(room_id: &str) -> Self {
|
||||
let account_data = EventFilter::new(0);
|
||||
let room = RoomFilter::new();
|
||||
let mut event_fields: Vec<String> = Vec::new();
|
||||
event_fields.push("type".to_string());
|
||||
event_fields.push("sender".to_string());
|
||||
event_fields.push("content.body".to_string());
|
||||
let presence = EventFilter::new(0);
|
||||
let room = RoomFilter::new(room_id);
|
||||
FilterRequest {
|
||||
presence,
|
||||
account_data,
|
||||
room
|
||||
event_fields,
|
||||
presence,
|
||||
room,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_filter(user: &str, server: &str, token: &str) -> Option<String> {
|
||||
pub fn get_filter(user: &str, server: &str, room_id: &str, token: &str)
|
||||
-> Option<String> {
|
||||
let user_encoded = url::encode(user);
|
||||
let mut url = String::from(server);
|
||||
url.push_str("/_matrix/client/r0/user/");
|
||||
url.push_str("/_matrix/client/v3/user/");
|
||||
url.push_str(&user_encoded);
|
||||
url.push_str("/filter");
|
||||
log::debug!("get_filter = {}", url);
|
||||
let filter_request = FilterRequest::new();
|
||||
let filter_request = FilterRequest::new(room_id);
|
||||
if let Some(request_body) = serialize(&filter_request) {
|
||||
// println!("filter_request = {}", request_body);
|
||||
if let Some(value) = handle_response(post_string_auth(&url, &request_body, token)) {
|
||||
if let Value::Object(body) = value {
|
||||
if let Some(Value::String(filter_id)) = body.get("filter_id") {
|
||||
// log::debug!("filter_id = {}", filter_id);
|
||||
log::debug!("filter_id = {}", filter_id);
|
||||
Some(filter_id.to_string())
|
||||
} else {
|
||||
log::error!("invalid response for get_filter");
|
||||
|
|
@ -384,7 +390,6 @@ pub fn client_sync(server: &str, filter: &str, since: &str, timeout: i32,
|
|||
}
|
||||
log::debug!("client_sync = {}", url);
|
||||
if let Some(value) = handle_response(get_json_auth(&url, token)) {
|
||||
// println!("SYNC = {:?}", value);
|
||||
if let Value::Object(body) = value {
|
||||
if let Some(Value::String(next_batch)) = body.get("next_batch") {
|
||||
Some((next_batch.to_string(), get_messages(body, room_id)))
|
||||
|
|
@ -429,10 +434,8 @@ pub fn send_message(server: &str, room_id: &str, text: &str, txn_id: &str, token
|
|||
log::debug!("send_message = {}", url);
|
||||
let message_request = MessageRequest::new(text);
|
||||
if let Some(request_body) = serialize(&message_request) {
|
||||
// println!("request_body = {}", request_body);
|
||||
if let Some(value) = handle_response(put_string_auth(&url, &request_body, token)) {
|
||||
if let Value::Object(_body) = value {
|
||||
// println!("SENT = {:?}", body);
|
||||
true
|
||||
} else {
|
||||
log::error!("invalid response for send_message");
|
||||
|
|
|
|||
Loading…
Reference in a new issue