From f7e216bb25b7df352c314e37ff30a951ac73baa2 Mon Sep 17 00:00:00 2001 From: nworbnhoj Date: Sat, 18 Jun 2022 07:07:08 +1000 Subject: [PATCH 1/5] convert RGB image immediately to greyscale on import code simplification/reduction, and memory/processing saving --- services/gam/src/bitmap.rs | 61 +++++++++++--------------------------- services/gam/src/lib.rs | 2 +- services/modals/src/lib.rs | 4 +-- 3 files changed, 21 insertions(+), 46 deletions(-) diff --git a/services/gam/src/bitmap.rs b/services/gam/src/bitmap.rs index 960989744..f57dec177 100644 --- a/services/gam/src/bitmap.rs +++ b/services/gam/src/bitmap.rs @@ -291,50 +291,37 @@ impl From<&Img> for Bitmap { } /* - * Image as a minimal flat buffer of RGB pixels; accessible by (x, y) + * Image as a minimal flat buffer of grey u8 pixels; accessible by (x, y) * * author: nworbnhoj */ - - #[derive(Debug, Clone, Copy)] -pub struct RGB(pub u8, pub u8, pub u8); - -impl RGB { - const R: u32 = 2126; - const G: u32 = 7152; - const B: u32 = 722; - const BLACK: u32 = RGB::R + RGB::G + RGB::B; - /// chromatic conversion from RGB to grey - pub fn to_grey(&self) -> u8 { - let grey_r = RGB::R * self.0 as u32; - let grey_g = RGB::G * self.1 as u32; - let grey_b = RGB::B * self.2 as u32; - ((grey_r + grey_g + grey_b) / RGB::BLACK).try_into().unwrap() - } -} - #[derive(Debug, Clone)] pub struct Img { - pixels: Vec, + pixels: Vec, width: usize, } impl Img { pub fn new(buf: Vec, width: usize) -> Self { + const R: u32 = 2126; + const G: u32 = 7152; + const B: u32 = 722; + const BLACK: u32 = R + G + B; let px_len = buf.len() / 3; - let mut pixels: Vec = Vec::with_capacity(px_len); + let mut pixels: Vec = Vec::with_capacity(px_len); for px in 0..px_len { let b = px * 3; - pixels.push(RGB { - 0: buf[b], - 1: buf[b + 1], - 2: buf[b + 2], - }); + let grey_r = R * buf[b] as u32; + let grey_g = G * buf[b + 1] as u32; + let grey_b = B * buf[b + 2] as u32; + pixels.push(((grey_r + grey_g + grey_b) / BLACK) + .try_into() + .unwrap()); } Self { pixels, width } } - pub fn get(&self, x: usize, y: usize) -> Option<&RGB> { + pub fn get(&self, x: usize, y: usize) -> Option<&u8> { let i: usize = (y * self.width) + x; self.pixels.get(i) } @@ -344,33 +331,24 @@ impl Img { let height: usize = length / width; (width, height, length) } - pub fn into_raw(&self) -> Vec { - let len = self.pixels.len() as usize; - let mut bytes: Vec = Vec::with_capacity(len); - for px in &self.pixels { - bytes.push(px.0); - bytes.push(px.1); - bytes.push(px.2); - } - bytes + pub fn as_slice(&self) -> &[u8] { + self.pixels.as_slice() } } - /* * Dithering involves aplying a threshold to each pixel to round down to Black * or up to White. The residual error from this blunt instrument is diffused amongst * the surrounding pixels. So the luminosity lost by forcing a pixel down to Black, * results in the surrounding pixels incrementally more likely to round up to White. * Pixels are processed from left to right and then top to bottom. The residual - * error from each Black/White determination is carried-forward to pixels to the + * error from each Black/White determination is carried-forward to pixels to the * right and below as per the diffusion scheme. * https://tannerhelland.com/2012/12/28/dithering-eleven-algorithms-source-code.html * * author: nworbnhoj */ - /// Burkes dithering diffusion scheme was chosen for its modest resource /// requirements with impressive quality outcome. /// Burkes dithering. Div=32. @@ -388,7 +366,6 @@ const BURKES: [(isize, isize, i16); 7] = [ (2, 1, 2), ]; - struct Dither { // the width of the image to be dithered width: usize, @@ -463,7 +440,7 @@ impl Dither { for y in 0..height { for x in 0..width { let pixel = match image.get(x, y) { - Some(rgb) => self.pixel(rgb.to_grey()), + Some(grey) => self.pixel(*grey), None => PixelColor::Dark, }; pixels.push(pixel); @@ -474,8 +451,6 @@ impl Dither { } } - - #[cfg(test)] mod tests { use super::*; diff --git a/services/gam/src/lib.rs b/services/gam/src/lib.rs index 42a3f80d6..2ca97ab0b 100644 --- a/services/gam/src/lib.rs +++ b/services/gam/src/lib.rs @@ -13,7 +13,7 @@ pub use apps::*; #[cfg(feature="ditherpunk")] pub mod bitmap; #[cfg(feature="ditherpunk")] -pub use bitmap::{Bitmap, Img, RGB}; +pub use bitmap::{Bitmap, Img}; use graphics_server::api::{TextOp, TextView}; use graphics_server::api::{Gid, Line, Circle, RoundedRectangle, TokenClaim}; diff --git a/services/modals/src/lib.rs b/services/modals/src/lib.rs index cb0edf1bd..06bd68fbd 100644 --- a/services/modals/src/lib.rs +++ b/services/modals/src/lib.rs @@ -262,8 +262,8 @@ impl Modals { let fir_img = fast_image_resize::Image::from_vec_u8( NonZeroU32::new(width).unwrap(), NonZeroU32::new(height).unwrap(), - img.into_raw(), - fast_image_resize::PixelType::U8x3, + img.as_slice().to_vec(), + fast_image_resize::PixelType::U8, ) .unwrap(); From 409f696fdeac4ed572845f68ffe4cab0cb2a04f3 Mon Sep 17 00:00:00 2001 From: nworbnhoj Date: Sat, 18 Jun 2022 08:43:24 +1000 Subject: [PATCH 2/5] Implement gam::PixelType enables gam::Img to import different image formats: U8 & U8x3 for now --- services/gam/src/bitmap.rs | 36 +++++++++++++++++++------- services/gam/src/lib.rs | 2 +- services/modals/src/lib.rs | 4 ++- services/modals/src/tests.rs | 4 ++- services/shellchat/src/cmds/net_cmd.rs | 6 ++--- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/services/gam/src/bitmap.rs b/services/gam/src/bitmap.rs index f57dec177..e7022554c 100644 --- a/services/gam/src/bitmap.rs +++ b/services/gam/src/bitmap.rs @@ -290,6 +290,12 @@ impl From<&Img> for Bitmap { } } +pub enum PixelType { + U8, + U8x3, + U8x4, +} + /* * Image as a minimal flat buffer of grey u8 pixels; accessible by (x, y) * @@ -303,21 +309,33 @@ pub struct Img { } impl Img { - pub fn new(buf: Vec, width: usize) -> Self { + pub fn new(buf: Vec, width: usize, px_type: PixelType) -> Self { const R: u32 = 2126; const G: u32 = 7152; const B: u32 = 722; const BLACK: u32 = R + G + B; - let px_len = buf.len() / 3; + let px_len = match px_type { + PixelType::U8 => buf.len(), + PixelType::U8x3 => buf.len() / 3, + _ => 0, + }; let mut pixels: Vec = Vec::with_capacity(px_len); for px in 0..px_len { - let b = px * 3; - let grey_r = R * buf[b] as u32; - let grey_g = G * buf[b + 1] as u32; - let grey_b = B * buf[b + 2] as u32; - pixels.push(((grey_r + grey_g + grey_b) / BLACK) - .try_into() - .unwrap()); + let pixel: u8 = match px_type { + PixelType::U8 => buf[px], + PixelType::U8x3 => { + let b = px * 3; + let grey_r = R * buf[b] as u32; + let grey_g = G * buf[b + 1] as u32; + let grey_b = B * buf[b + 2] as u32; + ((grey_r + grey_g + grey_b) / BLACK).try_into().unwrap() + } + _ => { + log::warn!("unsupported PixelType"); + 0 + } + }; + pixels.push(pixel); } Self { pixels, width } } diff --git a/services/gam/src/lib.rs b/services/gam/src/lib.rs index 2ca97ab0b..ff6240b32 100644 --- a/services/gam/src/lib.rs +++ b/services/gam/src/lib.rs @@ -13,7 +13,7 @@ pub use apps::*; #[cfg(feature="ditherpunk")] pub mod bitmap; #[cfg(feature="ditherpunk")] -pub use bitmap::{Bitmap, Img}; +pub use bitmap::{Bitmap, Img, PixelType}; use graphics_server::api::{TextOp, TextView}; use graphics_server::api::{Gid, Line, Circle, RoundedRectangle, TokenClaim}; diff --git a/services/modals/src/lib.rs b/services/modals/src/lib.rs index 06bd68fbd..aa90ca5a5 100644 --- a/services/modals/src/lib.rs +++ b/services/modals/src/lib.rs @@ -12,6 +12,8 @@ use num_traits::*; use std::convert::TryInto; #[cfg(feature = "ditherpunk")] use std::num::NonZeroU32; +#[cfg(feature = "ditherpunk")] +use bitmap::PixelType; use xous::{send_message, Message, CID}; use xous_ipc::Buffer; @@ -280,7 +282,7 @@ impl Modals { resizer.resize(&fir_img.view(), &mut rsz_view).unwrap(); let width: usize = resized.width().get().try_into().unwrap(); - Img::new(resized.into_vec(), width ) + Img::new(resized.into_vec(), width, PixelType::U8x3) } pub fn start_progress( diff --git a/services/modals/src/tests.rs b/services/modals/src/tests.rs index 3bb10811e..a899a5a28 100644 --- a/services/modals/src/tests.rs +++ b/services/modals/src/tests.rs @@ -2,6 +2,8 @@ use super::*; use gam::*; use std::thread; use xous_names::XousNames; +#[cfg(feature = "ditherpunk")] +use bitmap::PixelType; const RADIO_TEST: [&'static str; 4] = ["zebra", "cow", "horse", "cat"]; @@ -196,7 +198,7 @@ fn clifford() -> Img { (rgb[j], rgb[j + 1], rgb[j + 2]) = (buf[i], buf[i], buf[i]); i += 1; } - Img::new(rgb, SIZE.try_into().unwrap()) + Img::new(rgb, SIZE.try_into().unwrap(), PixelType::U8x3) } fn test_validator(input: TextEntryPayload) -> Option> { diff --git a/services/shellchat/src/cmds/net_cmd.rs b/services/shellchat/src/cmds/net_cmd.rs index 16820b637..806568460 100644 --- a/services/shellchat/src/cmds/net_cmd.rs +++ b/services/shellchat/src/cmds/net_cmd.rs @@ -17,7 +17,7 @@ use dns::Dns; // necessary to work around https://github.com/rust-lang/rust/issu #[cfg(feature="ditherpunk")] use std::str::FromStr; #[cfg(feature="ditherpunk")] -use gam::Img; +use gam::{Img, PixelType}; pub struct NetCmd { callback_id: Option, @@ -389,7 +389,7 @@ impl<'a> ShellCmdApi<'a> for NetCmd { let (width, height) = (info.width, info.height); log::info!("image is {} x {}", width, height); log::info!("some image bytes: {:x?}", &buf[..64]); - let img = Img::new(buf, width as usize); + let img = Img::new(buf, width as usize, PixelType::U8x3); modals.show_image(&img).expect("show image modal failed"); write!(ret, "Image of {} bytes read successfully", content_length).unwrap(); @@ -700,4 +700,4 @@ impl Worker { thread: Some(thread), } } -} \ No newline at end of file +} From 81ebb74b589334d156304953c56402121a9eb6bd Mon Sep 17 00:00:00 2001 From: nworbnhoj Date: Sat, 18 Jun 2022 09:00:36 +1000 Subject: [PATCH 3/5] simplify clifford test image to greyscale U8 --- services/modals/src/tests.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/services/modals/src/tests.rs b/services/modals/src/tests.rs index f3770642d..6194386e1 100644 --- a/services/modals/src/tests.rs +++ b/services/modals/src/tests.rs @@ -191,14 +191,7 @@ fn clifford() -> Img { buf[i] -= 1; } } - let mut rgb = vec![0u8; 3 * buf.len()]; - let mut i: usize = 0; - while i < buf.len() { - let j = 3 * i; - (rgb[j], rgb[j + 1], rgb[j + 2]) = (buf[i], buf[i], buf[i]); - i += 1; - } - Img::new(rgb, SIZE.try_into().unwrap(), PixelType::U8x3) + Img::new(buf, SIZE.try_into().unwrap(), PixelType::U8) } fn test_validator(input: TextEntryPayload) -> Option> { From 2dbdb10704cbc74f048ac3a6538544c7bb50b3cc Mon Sep 17 00:00:00 2001 From: nworbnhoj Date: Sat, 18 Jun 2022 12:07:58 +1000 Subject: [PATCH 4/5] remove dependency on fast-image-resize crate --- services/gam/src/bitmap.rs | 25 +++++++++++++ services/modals/Cargo.toml | 4 +- services/modals/src/lib.rs | 76 +++++++++++++++----------------------- 3 files changed, 55 insertions(+), 50 deletions(-) diff --git a/services/gam/src/bitmap.rs b/services/gam/src/bitmap.rs index e7022554c..88077e241 100644 --- a/services/gam/src/bitmap.rs +++ b/services/gam/src/bitmap.rs @@ -352,6 +352,31 @@ impl Img { pub fn as_slice(&self) -> &[u8] { self.pixels.as_slice() } + // simply retains the closest pixel + pub fn resize(&self, width: usize, height: usize) -> Img { + let mut buf = vec![0u8; (width * height).try_into().unwrap()]; + let (self_width, self_height, _) = self.size(); + let x_scale = self_width as f32 / width as f32; + let y_scale = self_height as f32 / height as f32; + let self_x_base = x_scale * 0.5; + let self_y_base = y_scale * 0.5; + + // Pretabulate horizontal pixel positions + let mut self_x_tab: Vec = Vec::with_capacity(width); + for buf_x in 0..width { + self_x_tab.push(((self_x_base + x_scale * buf_x as f32) as usize).min(self_width - 1)); + } + + for buf_y in 0..height { + let self_y = ((self_y_base + y_scale * buf_y as f32) as usize).min(self_height - 1); + let self_row = self_y * self_width; + let buf_row = buf_y * width; + for (buf_x, self_x) in self_x_tab.iter().enumerate() { + buf[buf_row + buf_x] = self.pixels[self_row + self_x]; + } + } + Img::new(buf, width.try_into().unwrap(), PixelType::U8) + } } /* diff --git a/services/modals/Cargo.toml b/services/modals/Cargo.toml index cf7f5a986..e1932daee 100644 --- a/services/modals/Cargo.toml +++ b/services/modals/Cargo.toml @@ -22,12 +22,10 @@ tts-frontend = {path="../tts"} locales = {path = "../../locales"} bit_field = "0.9.0" -fast_image_resize = {version = "0.9.2", optional = true} - [target.'cfg(not(any(windows,unix)))'.dependencies] utralib = { path = "../../utralib"} [features] tts = [] -ditherpunk = ["fast_image_resize"] +ditherpunk = [] default = [] diff --git a/services/modals/src/lib.rs b/services/modals/src/lib.rs index 1488bd91a..4bdfaeb31 100644 --- a/services/modals/src/lib.rs +++ b/services/modals/src/lib.rs @@ -10,10 +10,6 @@ use gam::*; use num_traits::*; #[cfg(feature = "ditherpunk")] use std::convert::TryInto; -#[cfg(feature = "ditherpunk")] -use std::num::NonZeroU32; -#[cfg(feature = "ditherpunk")] -use bitmap::PixelType; use xous::{send_message, Message, CID}; use xous_ipc::Buffer; @@ -201,7 +197,8 @@ impl Modals { pub fn show_image(&self, img: &Img) -> Result<(), xous::Error> { self.lock(); // resize and/or rotate - let (modal_width, modal_height) = (gam::IMG_MODAL_WIDTH as f32, gam::IMG_MODAL_HEIGHT as f32); + let (modal_width, modal_height) = + (gam::IMG_MODAL_WIDTH as f32, gam::IMG_MODAL_HEIGHT as f32); let (w, h, _) = img.size(); let (img_width, img_height) = (w as f32, h as f32); @@ -209,13 +206,19 @@ impl Modals { let landscape_scale = (modal_width / img_height).min(modal_height / img_width); let mut bm = if portrait_scale >= 1.0 { Bitmap::from(img) - } else if landscape_scale >= 1.0 { + } else if landscape_scale >= 1.0 { Bitmap::from(img).rotate90() } else if portrait_scale >= landscape_scale { - let resized = Modals::resize_image(img, portrait_scale); + let resized = img.resize( + (portrait_scale * img_width) as usize, + (portrait_scale * img_height) as usize, + ); Bitmap::from(&resized) } else { - let resized = Modals::resize_image(img, landscape_scale); + let resized = img.resize( + (landscape_scale * img_width) as usize, + (landscape_scale * img_height) as usize, + ); Bitmap::from(&resized).rotate90() }; @@ -225,8 +228,10 @@ impl Modals { // center image in modal let center = Point::new( ((gam::IMG_MODAL_WIDTH - bm_width) / 2).try_into().unwrap(), - ((gam::IMG_MODAL_HEIGHT - bm_height) / 2).try_into().unwrap(), - ); + ((gam::IMG_MODAL_HEIGHT - bm_height) / 2) + .try_into() + .unwrap(), + ); bm.translate(center); let mut tiles: [Option; 6] = [None; 6]; @@ -248,38 +253,6 @@ impl Modals { Ok(()) } - #[allow(dead_code)] - #[cfg(feature = "ditherpunk")] - fn resize_image(img: &Img, scale: f32) -> Img { - let (w, h, _) = img.size(); - let (img_width, img_height) = (w as f32, h as f32); - let height: u32 = (scale * img_height).floor() as u32; - let width: u32 = (scale * img_width).floor() as u32; - - let fir_img = fast_image_resize::Image::from_vec_u8( - NonZeroU32::new(width).unwrap(), - NonZeroU32::new(height).unwrap(), - img.as_slice().to_vec(), - fast_image_resize::PixelType::U8, - ) - .unwrap(); - - // Create container for data of destination image - let mut resized = fast_image_resize::Image::new( - NonZeroU32::new(width).unwrap(), - NonZeroU32::new(height).unwrap(), - fir_img.pixel_type(), - ); - let mut rsz_view = resized.view_mut(); - - // resize image with fastest available algorithm - let mut resizer = fast_image_resize::Resizer::new(fast_image_resize::ResizeAlg::Nearest); - resizer.resize(&fir_img.view(), &mut rsz_view).unwrap(); - - let width: usize = resized.width().get().try_into().unwrap(); - Img::new(resized.into_vec(), width, PixelType::U8x3) - } - pub fn start_progress( &self, title: &str, @@ -532,10 +505,14 @@ impl Modals { fn unlock(&self) { self.have_lock.set(false); } - pub fn conn(&self) -> CID {self.conn} + pub fn conn(&self) -> CID { + self.conn + } /// Don't leak this token outside of your server, otherwise, another server can pretend to be you and /// steal your modal information! - pub fn token(&self) -> [u32; 4] {self.token} + pub fn token(&self) -> [u32; 4] { + self.token + } } use core::sync::atomic::{AtomicU32, Ordering}; @@ -563,7 +540,10 @@ impl Drop for Modals { /// /// This function is "broken out" so that it can be called from a thread without having /// to wrap a mutex around the primary Modals structure. -pub fn dynamic_notification_blocking_listener(token: [u32; 4], conn: CID) -> Result, xous::Error> { +pub fn dynamic_notification_blocking_listener( + token: [u32; 4], + conn: CID, +) -> Result, xous::Error> { match send_message( conn, Message::new_blocking_scalar( @@ -573,7 +553,9 @@ pub fn dynamic_notification_blocking_listener(token: [u32; 4], conn: CID) -> Res token[2] as usize, token[3] as usize, ), - ).expect("couldn't listen") { + ) + .expect("couldn't listen") + { xous::Result::Scalar2(is_some, code) => { if is_some == 1 { let c = char::from_u32(code as u32).unwrap_or('\u{0000}'); @@ -585,6 +567,6 @@ pub fn dynamic_notification_blocking_listener(token: [u32; 4], conn: CID) -> Res Ok(None) } } - _ => Err(xous::Error::InternalError) + _ => Err(xous::Error::InternalError), } } From e504e9b89c514e9c829278b6f8a7dde7e4789e21 Mon Sep 17 00:00:00 2001 From: bunnie Date: Sat, 18 Jun 2022 14:21:56 +0800 Subject: [PATCH 5/5] attempt to limit memory size used by PNG decoder still getting a heap allocation failure, something is looking for 256k and even when the heap is increased to a couple megabytes we get heap allocation failures. --- Cargo.lock | 11 ----------- services/shellchat/src/cmds/net_cmd.rs | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9c855e48..9ed5b57d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1133,16 +1133,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fast_image_resize" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf174e621dc44fc6118d0e3289a1286c166ac28599e15651a82da23a14a3425" -dependencies = [ - "num-traits", - "thiserror", -] - [[package]] name = "fastrand" version = "1.7.0" @@ -2061,7 +2051,6 @@ name = "modals" version = "0.1.0" dependencies = [ "bit_field 0.9.0", - "fast_image_resize", "gam", "locales", "log", diff --git a/services/shellchat/src/cmds/net_cmd.rs b/services/shellchat/src/cmds/net_cmd.rs index 806568460..c9a3dbd4f 100644 --- a/services/shellchat/src/cmds/net_cmd.rs +++ b/services/shellchat/src/cmds/net_cmd.rs @@ -380,11 +380,14 @@ impl<'a> ShellCmdApi<'a> for NetCmd { let modals = modals::Modals::new(&env.xns).unwrap(); // have to add 4 to the start because the index is at the beginning of the header sequence, // not the end of the sequence. + log::info!("heap size: {}", heap_usage()); log::info!("decoding png starting with: {:x?}", &received[start+4..start+4+64]); - let decoder = png::Decoder::new(&received[start+4..]); + let decoder = png::Decoder::new_with_limits(&received[start+4..], png::Limits { bytes: 400 * 1024 }); + log::info!("heap size: {}", heap_usage()); let mut reader = decoder.read_info().expect("failed to read png info"); let mut buf = vec![0; reader.output_buffer_size()]; let info = reader.next_frame(&mut buf).expect("failed to decode png"); + log::info!("heap size: {}", heap_usage()); let (width, height) = (info.width, info.height); log::info!("image is {} x {}", width, height); @@ -701,3 +704,17 @@ impl Worker { } } } + +#[cfg(feature="ditherpunk")] +fn heap_usage() -> usize { + match xous::rsyscall(xous::SysCall::IncreaseHeap(0, xous::MemoryFlags::R)).expect("couldn't get heap size") { + xous::Result::MemoryRange(m) => { + let usage = m.len(); + usage + } + _ => { + log::error!("Couldn't measure heap usage"); + 0 + }, + } +} \ No newline at end of file