diff --git a/services/bao-video/Cargo.toml b/services/bao-video/Cargo.toml index 10e6ae1f7..08caf25cd 100644 --- a/services/bao-video/Cargo.toml +++ b/services/bao-video/Cargo.toml @@ -30,7 +30,7 @@ modals = { path = "../modals", optional = true } # QR decoding libm = { version = "0.2.8" } nalgebra = { version = "0.33", default-features = false, features = ["libm"] } -rqrr = "0.8.0" +rqrr = { git = "https://github.com/betrusted-io/rqrr.git", branch = "rv32-opt" } base64 = { version = "0.22.1", optional = true } locales = { path = "../../locales" } diff --git a/services/bao-video/src/gfx.rs b/services/bao-video/src/gfx.rs index 46334f012..54b5eb00f 100644 --- a/services/bao-video/src/gfx.rs +++ b/services/bao-video/src/gfx.rs @@ -96,6 +96,7 @@ pub fn msg<'a>(fb: &mut dyn FrameBuffer, text: &'a str, ll_pos: Point, fg: Color ll_pos.x += x_update; } +#[allow(dead_code)] pub fn line(fb: &mut dyn FrameBuffer, l: Line, clip: Option, xor: bool) { let color: ColorNative; if l.style.stroke_color.is_some() { diff --git a/services/bao-video/src/homography.rs b/services/bao-video/src/homography.rs deleted file mode 100644 index 80539f51a..000000000 --- a/services/bao-video/src/homography.rs +++ /dev/null @@ -1,99 +0,0 @@ -use nalgebra::{Matrix3, SVD}; - -pub type Matrix8x9 = nalgebra::Matrix>; - -// This function calculates the homography matrix from 4 corresponding point pairs -pub fn find_homography(src_points: [(f32, f32); 4], dst_points: [(f32, f32); 4]) -> Option> { - // Prepare the A matrix (8x9) for 4 point pairs - let mut a = Matrix8x9::zeros(); - let mut b = nalgebra::Vector::from_column_slice(&[0.0f32; 9]); - - for (i, (src, dst)) in src_points.iter().zip(dst_points.iter()).enumerate() { - let (x, y) = *src; - let (xp, yp) = *dst; - let row1 = 2 * i; - let row2 = 2 * i + 1; - - a[(row1, 0)] = -x; - a[(row1, 1)] = -y; - a[(row1, 2)] = -1.0; - a[(row1, 6)] = x * xp; - a[(row1, 7)] = y * xp; - b[row1] = -xp; - - a[(row2, 3)] = -x; - a[(row2, 4)] = -y; - a[(row2, 5)] = -1.0; - a[(row2, 6)] = x * yp; - a[(row2, 7)] = y * yp; - b[row2] = -yp; - } - - // Solve the least-squares solution using Singular Value Decomposition (SVD) - let svd = SVD::new_unordered(a.clone(), true, true); - let solution = svd.solve(&b, f32::EPSILON); - - if let Ok(h) = solution { - // Construct the homography matrix from the result - let mut homography = Matrix3::identity(); - homography[(0, 0)] = h[0]; - homography[(0, 1)] = h[1]; - homography[(0, 2)] = h[2]; - homography[(1, 0)] = h[3]; - homography[(1, 1)] = h[4]; - homography[(1, 2)] = h[5]; - homography[(2, 0)] = h[6]; - homography[(2, 1)] = h[7]; - homography[(2, 2)] = 1.0; - - Some(homography) - } else { - None - } -} - -const HOM_FP_SHIFT: i32 = 18; -const HOM_FP_SCALE: i32 = 1 << HOM_FP_SHIFT; - -pub fn matrix3_to_fixp(h: Matrix3) -> Matrix3 { - Matrix3::new( - (h[(0, 0)] * HOM_FP_SCALE as f32) as i32, - (h[(0, 1)] * HOM_FP_SCALE as f32) as i32, - (h[(0, 2)] * HOM_FP_SCALE as f32) as i32, - (h[(1, 0)] * HOM_FP_SCALE as f32) as i32, - (h[(1, 1)] * HOM_FP_SCALE as f32) as i32, - (h[(1, 2)] * HOM_FP_SCALE as f32) as i32, - (h[(2, 0)] * HOM_FP_SCALE as f32) as i32, - (h[(2, 1)] * HOM_FP_SCALE as f32) as i32, - (h[(2, 2)] * HOM_FP_SCALE as f32) as i32, - ) -} - -pub fn apply_fixp_homography(homography: &Matrix3, point: (i32, i32)) -> (i32, i32) { - let (x, y) = point; - - // Transform the point using fixed-point arithmetic - let tx = homography[(0, 0)] * x + homography[(0, 1)] * y + homography[(0, 2)]; - let ty = homography[(1, 0)] * x + homography[(1, 1)] * y + homography[(1, 2)]; - let tw = homography[(2, 0)] * x + homography[(2, 1)] * y + homography[(2, 2)]; - - // Normalize by w to get the final coordinates - if tw != 0 { - let x_transformed = tx / tw; - let y_transformed = ty / tw; - - (x_transformed, y_transformed) - } else { - log::info!( - "Homography transformation would divide by 0: possible fixed-point underflow!\n\r {}, {}\n\r {} * {} + {} * {} + {}", - tx, - ty, - homography[(2, 0)], - x, - homography[(2, 1)], - y, - homography[(2, 2)] - ); - (0, 0) - } -} diff --git a/services/bao-video/src/main.rs b/services/bao-video/src/main.rs index fd2bf1a5b..ba4e62798 100644 --- a/services/bao-video/src/main.rs +++ b/services/bao-video/src/main.rs @@ -1,8 +1,6 @@ use ux_api::minigfx::*; mod gfx; -mod homography; -mod modules; #[cfg(feature = "board-baosec")] mod panic; mod qr; @@ -76,6 +74,7 @@ pub const IMAGE_HEIGHT: usize = 240; // off from ideal, which causes the data around that point to be interpreted incorrectly. #[cfg(feature = "b64-export")] +#[allow(dead_code)] fn encode_base64(input: &[u8]) -> String { general_purpose::STANDARD.encode(input) } /// This converts a frame of `[u8]` grayscale pixels that may be larger than the native @@ -313,8 +312,6 @@ pub fn wrapped_main(main_thread_token: MainThreadToken) -> ! { let mut modal_queue = VecDeque::::new(); let mut frames = 0; let mut frame = [0u8; IMAGE_WIDTH * IMAGE_HEIGHT]; - #[cfg(feature = "b64-export")] - let mut original = [0u8; IMAGE_WIDTH * IMAGE_HEIGHT]; let mut decode_success; let mut msg_opt = None; #[cfg(feature = "gfx-testing")] @@ -355,272 +352,57 @@ pub fn wrapped_main(main_thread_token: MainThreadToken) -> ! { let mut candidates = Vec::::new(); decode_success = false; log::debug!("------------- SEARCH {} -----------", frames); - let finder_width = + let _finder_width = qr::find_finders(&mut candidates, &frame, bw_thresh, IMAGE_WIDTH) as isize; + // blit raw camera fb to display + blit_to_display(&mut display, &frame, true, &mut bw_thresh); if candidates.len() == 3 { - #[cfg(feature = "b64-export")] - original.copy_from_slice(&frame); // make a backup copy for diagnostics - let candidates_orig = candidates.clone(); - let mut x_candidates: [Point; 3] = [Point::new(0, 0); 3]; - // apply homography to generate a new buffer for processing - let mut aligned = Vec::new(); - let mut qr_pixels: Option = None; - if let Some(mut qr_corners) = qr::QrCorners::from_finders( - &candidates.try_into().unwrap(), - Point::new(IMAGE_WIDTH as isize, IMAGE_HEIGHT as isize), - // add a search margin on the finder width - (finder_width + (qr::FINDER_SEARCH_MARGIN * finder_width) / (1 + 1 + 3 + 1 + 1)) - as usize, - ) { - let dims = Point::new(IMAGE_WIDTH as isize, IMAGE_HEIGHT as isize); - let mut il = qr::ImageRoi::new(&mut frame, dims, bw_thresh); - let (src, dst) = qr_corners.mapping(&mut il, qr::HOMOGRAPHY_MARGIN); - let mut src_f: [(f32, f32); 4] = [(0.0, 0.0); 4]; - let mut dst_f: [(f32, f32); 4] = [(0.0, 0.0); 4]; - let mut all_found = true; - for (s, s_f32) in src.iter().zip(src_f.iter_mut()) { - if let Some(p) = s { - *s_f32 = p.to_f32(); - } else { - all_found = false; + gfx::msg( + &mut display, + "Decoding...", + Point::new(0, 0), + Mono::White.into(), + Mono::Black.into(), + ); + let mut img = + rqrr::PreparedImage::prepare_from_greyscale(IMAGE_WIDTH, IMAGE_HEIGHT, |x, y| { + frame[y * IMAGE_WIDTH + x] + }); + let grids = img.detect_grids(); + if grids.len() == 1 { + match grids[0].decode() { + Ok((meta, content)) => { + log::info!("meta: {:?}", meta); + log::info!("************ {} ***********", content); + decode_success = true; + gfx::msg( + &mut display, + &format!("{:?}", meta), + Point::new(0, 0), + Mono::White.into(), + Mono::Black.into(), + ); + gfx::msg( + &mut display, + &format!("{:?}", content), + Point::new(0, 64), + Mono::White.into(), + Mono::Black.into(), + ); + } + Err(e) => { + log::info!("{:?}", e); + gfx::msg( + &mut display, + &format!("{:?}", e), + Point::new(0, 0), + Mono::White.into(), + Mono::Black.into(), + ); } } - for (d, d_f32) in dst.iter().zip(dst_f.iter_mut()) { - if let Some(p) = d { - *d_f32 = p.to_f32(); - } else { - all_found = false; - } - } - - if all_found { - if let Some(h) = homography::find_homography(src_f, dst_f) { - if let Some(h_inv) = h.try_inverse() { - log::info!("{:?}", h_inv); - let h_inv_fp = homography::matrix3_to_fixp(h_inv); - log::info!("{:?}", h_inv_fp); - - aligned = vec![0u8; qr_corners.qr_pixels() * qr_corners.qr_pixels()]; - // iterate through pixels and apply homography - for y in 0..qr_corners.qr_pixels() { - for x in 0..qr_corners.qr_pixels() { - let (x_src, y_src) = homography::apply_fixp_homography( - &h_inv_fp, - (x as i32, y as i32), - ); - if (x_src as i32 >= 0) - && ((x_src as i32) < dims.x as i32) - && (y_src as i32 >= 0) - && ((y_src as i32) < dims.y as i32) - { - // println!("{},{} -> {},{}", x_src as i32, y_src as i32, - // x, - // y); - aligned - [qr_corners.qr_pixels() * y as usize + x as usize] = - frame[IMAGE_WIDTH * y_src as usize + x_src as usize]; - } else { - aligned - [qr_corners.qr_pixels() * y as usize + x as usize] = - 255; - } - } - } - - // we can also know the location of the finders by transforming them - let h_fp = homography::matrix3_to_fixp(h); - for (i, &c) in candidates_orig.iter().enumerate() { - let (x, y) = homography::apply_fixp_homography( - &h_fp, - (c.x as i32, c.y as i32), - ); - x_candidates[i] = Point::new(x as isize, y as isize); - } - qr_pixels = Some(qr_corners.qr_pixels()); - } - } - } - } - - if let Some(qr_width) = qr_pixels { - // show the transformed/aligned frame - frame.fill(255); - for (dst_line, src_line) in - frame.chunks_mut(IMAGE_WIDTH).zip(aligned.chunks(qr_width)) - { - // "center up" the QR in the middle by the estimated margin - // this is just a perceptual trick to prevent users from shifting the position - // of the camera - for (dst, &src) in dst_line[qr::HOMOGRAPHY_MARGIN.abs() as usize..] - .iter_mut() - .zip(src_line.iter()) - { - *dst = src; - } - } - blit_to_display(&mut display, &frame, true, &mut bw_thresh); - - // we now have a QR code in "canonical" orientation, with a - // known width in pixels - for &x in x_candidates.iter() { - log::info!("transformed finder location {:?}", x); - } - - // Confirm that the finders coordinates are valid - let mut checked_candidates = Vec::::new(); - let x_finder_width = - qr::find_finders(&mut checked_candidates, &aligned, bw_thresh, qr_width as _) - as isize; - log::info!("x_finder width: {}", x_finder_width); - - // check that the new coordinates are within delta pixels of the original - const XFORM_DELTA: isize = 2; - let mut deltas = Vec::::new(); - for c in checked_candidates { - log::info!("x_point: {:?}", c); - for &xformed in x_candidates.iter() { - let delta = xformed - c; - log::info!("delta: {:?}", delta); - if delta.x.abs() <= XFORM_DELTA && delta.y.abs() <= XFORM_DELTA { - deltas.push(delta); - } - } - } - if deltas.len() == 3 { - let (version, modules) = qr::guess_code_version( - x_finder_width as usize, - (qr_width as isize + qr::HOMOGRAPHY_MARGIN * 2) as usize, - ); - - log::info!("image dims: {}", qr_width); - log::info!("guessed version: {}, modules: {}", version, modules); - log::info!( - "QR symbol width in pixels: {}", - qr_width - 2 * (qr::HOMOGRAPHY_MARGIN.abs() as usize) - ); - - let qr = qr::ImageRoi::new( - &mut aligned, - Point::new(qr_width as _, qr_width as _), - bw_thresh, - ); - let grid = modules::stream_to_grid( - &qr, - qr_width, - modules, - qr::HOMOGRAPHY_MARGIN.abs() as usize, - bw_thresh, - ); - - println!("grid len {}", grid.len()); - for y in 0..modules { - for x in 0..modules { - if grid[y * modules + x] { - print!("X"); - } else { - print!(" "); - } - } - println!(" {:2}", y); - } - #[cfg(feature = "b64-export")] - { - println!("begin orig base 64"); - for block in original.chunks(16384) { - let encoded = encode_base64(block); - println!("{}", encoded); - } - println!("end orig base 64"); - println!("begin base 64"); - for block in frame.chunks(16384) { - let encoded = encode_base64(block); - println!("{}", encoded); - } - println!("end base 64"); - } - - let simple = rqrr::SimpleGrid::from_func(modules, |x, y| { - grid[(modules - 1) - x + y * modules] - }); - let grid = rqrr::Grid::new(simple); - match grid.decode() { - Ok((meta, content)) => { - log::info!("meta: {:?}", meta); - log::info!("************ {} ***********", content); - decode_success = true; - gfx::msg( - &mut display, - &format!("{:?}", meta), - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); - gfx::msg( - &mut display, - &format!("{:?}", content), - Point::new(0, 64), - Mono::White.into(), - Mono::Black.into(), - ); - } - Err(e) => { - log::info!("{:?}", e); - gfx::msg( - &mut display, - &format!("{:?}", e), - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); - } - } - } else { - log::info!("Transformed image did not survive sanity check!"); - #[cfg(feature = "b64-export")] - { - println!("begin orig base 64"); - for block in original.chunks(16384) { - let encoded = encode_base64(block); - println!("{}", encoded); - } - println!("end orig base 64"); - println!("begin base 64"); - for block in frame.chunks(16384) { - let encoded = encode_base64(block); - println!("{}", encoded); - } - println!("end base 64"); - } - - gfx::msg( - &mut display, - "Hold device steady...", - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); - } - } else { - blit_to_display(&mut display, &frame, true, &mut bw_thresh); - for c in candidates_orig.iter() { - log::debug!("****** candidate: {}, {} ******", c.x, c.y); - // remap image to screen coordinates (it's 2:1) - let c_screen = *c / 2; - // flip coordinates to match the camera data - // c_screen = Point::new(c_screen.x, display.dimensions().y - 1 - c_screen.y); - qr::draw_crosshair(&mut display, c_screen); - } - gfx::msg( - &mut display, - "Align the QR code...", - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); } } else { - // blit raw camera fb to display - blit_to_display(&mut display, &frame, true, &mut bw_thresh); gfx::msg( &mut display, "Scan QR code...", @@ -632,7 +414,7 @@ pub fn wrapped_main(main_thread_token: MainThreadToken) -> ! { display.draw(); if decode_success { - tt.sleep_ms(2000).ok(); + tt.sleep_ms(1500).ok(); } // clear the front buffer diff --git a/services/bao-video/src/modules.rs b/services/bao-video/src/modules.rs deleted file mode 100644 index bbfea3c2b..000000000 --- a/services/bao-video/src/modules.rs +++ /dev/null @@ -1,154 +0,0 @@ -use ux_api::minigfx::Point; - -use crate::qr::*; - -/// This function takes a fraction described as `m.n`, where n is a fraction out of 1024, as well -/// as an `end` value, and returns numbers going from `0..end` such that with each -/// call of `next()` you produce a usize index in the range of `0..end` that is the -/// nearest algebraic integer that would result from adding the value `m.n` to the -/// current iteration count. -/// -/// Thus, a value of 1.5 would be passed as m=1, n=512 (as 512/1024 = 0.5). -/// -/// This "iterator" is not guaranteed to return every element in the range of `0..end`, -/// and in fact some numbers may not be returned at all, including `0` or `end-1`. However, -/// it does guarantee that the output is monatomically increasing, unless an error signal -/// is injected that is big enough to cause the count to go backwards. -/// -/// Features that we might need: -/// - A report of the "error sign", expressed as +1, 0, -1, which expresses the direction -/// in which we're rounding. For example, 6.3 would be returned as 6 with an error of +1. -/// whereas 6.9 would be returned as 7 with an error of -1. -/// - The ability to inject an error correction signal into the fractional counter. This -/// if the loop internally concludes that an error signal would produce a more accurate -/// sub-pixel sampling, the error count would be applied at that iteration so that subsequent -/// indices absorb the delta into their future counts. - -pub const FIXED_POINT_SHIFT: usize = 10; -pub const FIXED_POINT: usize = 1 << FIXED_POINT_SHIFT; // 1024 - -pub struct FracIter { - stride: usize, - current: usize, - end: usize, - count: usize, - finished: bool, -} -impl FracIter { - pub fn new(m: usize, n: usize, end: usize) -> Self { - assert!(n < FIXED_POINT); - // start the iterator at half the stride of a module - Self { - stride: (m << FIXED_POINT_SHIFT | n), - current: (m << FIXED_POINT_SHIFT | n) / 2, - end, - count: 0, - finished: false, - } - } - - pub fn next(&mut self) -> Option { - // short circuit computation if we've hit the end of the iterator - if self.finished { - return None; - } - if self.count < self.end { - let rounded_m = if (self.current & (FIXED_POINT - 1)) >= (FIXED_POINT / 2) { - (self.current >> FIXED_POINT_SHIFT) + 1 - } else { - self.current >> FIXED_POINT_SHIFT - }; - self.current += self.stride; - self.count += 1; - Some(rounded_m) - } else { - self.finished = true; - None - } - } - - pub fn reset(&mut self) { - self.finished = false; - self.current = self.stride / 2; - self.count = 0; - } - - #[allow(dead_code)] - /// Untested, but reserved API for adjusting grid streaming if we need better performance on - /// higher resolution QR codes - pub fn error(&self) -> isize { - if (self.current & (FIXED_POINT - 1)) == 0 { - 0 - } else if (self.current & (FIXED_POINT - 1)) >= (FIXED_POINT / 2) { - 1 - } else { - -1 - } - } - - #[allow(dead_code)] - /// Untested, but reserved API for adjusting grid streaming if we need better performance on - /// higher resolution QR codes - /// - /// The sign on m applies to n, but n is represented as a sign-less quantity - pub fn nudge(&mut self, m: isize, n: usize) { - assert!(n < FIXED_POINT); - if m >= 0 { - self.current += ((m as usize) << FIXED_POINT_SHIFT) | n; - } else { - // nudge down case - self.current -= ((-m as usize) << FIXED_POINT_SHIFT) | n; - } - } -} - -pub fn stream_to_grid( - image: &ImageRoi, - qr_size_pixels: usize, - qr_size_modules: usize, - margin: usize, - bw_thresh: u8, -) -> Vec { - let pix_per_module = ((qr_size_pixels - margin * 2) << FIXED_POINT_SHIFT) / qr_size_modules; - let mut x_frac = FracIter::new( - pix_per_module >> FIXED_POINT_SHIFT, - pix_per_module & ((FIXED_POINT) - 1), - qr_size_modules, - ); - let mut y_frac = FracIter::new( - pix_per_module >> FIXED_POINT_SHIFT, - pix_per_module & ((FIXED_POINT) - 1), - qr_size_modules, - ); - let mut grid = Vec::::new(); - let mut i = 0; - log::info!("{}", qr_size_modules); - while let Some(y) = y_frac.next() { - const FUDGE_X: usize = 0; - const FUDGE_Y: usize = 0; - while let Some(x) = x_frac.next() { - if true { - // seems to work better with the simple threshold? - if image.data[(y + margin - FUDGE_Y) * image.width + (x + margin + FUDGE_X)] < bw_thresh { - grid.push(true); - } else { - grid.push(false); - } - } else { - if image - .neighbor_luma(Point::new(x as isize + margin as isize, y as isize + margin as isize)) - .unwrap() - < bw_thresh - { - grid.push(true); - } else { - grid.push(false); - } - } - } - x_frac.reset(); - print!("{} ", i); - i += 1; - } - grid -} diff --git a/services/bao-video/src/qr.rs b/services/bao-video/src/qr.rs index fc9307f95..a69f5954a 100644 --- a/services/bao-video/src/qr.rs +++ b/services/bao-video/src/qr.rs @@ -1,8 +1,6 @@ -use core::cell::RefCell; -use core::convert::TryFrom; use core::ops::{BitXor, Not}; -use ux_api::minigfx::{ColorNative, FrameBuffer, Point}; +use ux_api::minigfx::{FrameBuffer, Point}; use crate::gfx; @@ -18,13 +16,9 @@ const UPPER_1: usize = 2 << SEQ_FP_SHIFT; const LOWER_3: usize = 2 << SEQ_FP_SHIFT; const UPPER_3: usize = 4 << SEQ_FP_SHIFT; -/// Finder search margin, as defined by expected QR code code widths (so this scales with the effective -/// resolution of the code) -pub const FINDER_SEARCH_MARGIN: isize = 2; -/// How much we want the final QR image to be "pulled in" from the outer edge of the image buffer -pub const HOMOGRAPHY_MARGIN: isize = -4; +#[allow(dead_code)] pub const CROSSHAIR_LEN: isize = 3; - +#[allow(dead_code)] pub fn draw_crosshair(image: &mut dyn FrameBuffer, p: Point) { use ux_api::minigfx::Line; gfx::line( @@ -41,426 +35,6 @@ pub fn draw_crosshair(image: &mut dyn FrameBuffer, p: Point) { ); } -#[allow(dead_code)] -/// Used for debugging QR code algorithms -pub fn draw_line(image: &mut dyn FrameBuffer, l: &LineDerivation, color: ColorNative) { - let axis = l.independent_axis; - let (m, b) = l.equation.unwrap(); - match axis { - Axis::X => { - for x in 0..image.dimensions().x { - let y = (x as f32 * m + b) as isize; - if y >= 0 && y < image.dimensions().y as isize { - image.put_pixel(Point::new(x, y), color); - } - } - } - Axis::Y => { - for y in 0..image.dimensions().y { - let x = (y as f32 * m + b) as isize; - if x >= 0 && x < image.dimensions().x as isize { - image.put_pixel(Point::new(x, y), color); - } - } - } - } -} -struct DirRange { - start: usize, - stop_exclusive: usize, - is_up: bool, - range: DirectionalRange, -} -impl Iterator for DirRange { - type Item = usize; - - fn next(&mut self) -> Option { self.range.next() } -} -impl DirRange { - pub fn new(start: usize, stop_exclusive: usize, is_up: bool) -> Self { - Self { - start, - stop_exclusive, - is_up, - range: if is_up { - DirectionalRange::Up(start..stop_exclusive) - } else { - DirectionalRange::Down((start..stop_exclusive).rev()) - }, - } - } - - pub fn reset(&mut self) { - self.range = if self.is_up { - DirectionalRange::Up(self.start..self.stop_exclusive) - } else { - DirectionalRange::Down((self.start..self.stop_exclusive).rev()) - } - } -} - -enum DirectionalRange { - Up(core::ops::Range), - Down(core::iter::Rev>), -} - -impl Iterator for DirectionalRange { - type Item = usize; - - fn next(&mut self) -> Option { - match self { - DirectionalRange::Up(iter) => iter.next(), - DirectionalRange::Down(iter) => iter.next(), - } - } -} - -#[derive(Copy, Clone, PartialEq, Eq)] -pub enum Axis { - X, - Y, -} - -fn adjust_axis(p: Point, axis: Axis) -> (f32, f32) { - ( - match axis { - Axis::X => p.x as f32, - Axis::Y => p.y as f32, - }, - match axis { - Axis::X => p.y as f32, - Axis::Y => p.x as f32, - }, - ) -} - -fn least_squares_fit(points: &[Point], axis: Axis) -> (f32, f32) { - let n = points.len() as f32; - let mut sum_x = 0.0f32; - let mut sum_y = 0.0f32; - let mut sum_xx = 0.0f32; - let mut sum_xy = 0.0f32; - for &point in points.iter() { - // flip x/y coordinates based on the independent axis - let (x, y) = adjust_axis(point, axis); - sum_x += x; - sum_y += y; - sum_xx += x * x; - sum_xy += x * y; - } - let m = (n * sum_xy - sum_x * sum_y) / (n * sum_xx - sum_x * sum_x); - let b = (sum_y - m * sum_x) / n; - (m, b) -} - -fn point_from_hv_lines(hline: &LineDerivation, vline: &LineDerivation) -> Option { - if let Some((m1, b1)) = hline.equation { - if let Some((m2v, b2v)) = vline.equation { - log::info!("h: {}, {} | v: {}, {}", m1, b1, m2v, b2v); - if m2v != 0.0 { - let m2 = 1.0 / m2v; - let b2 = -b2v / m2v; - let x = (b2 - b1) / (m1 - m2); - let y = m1 * x + b1; - Some(Point::new(x as isize, y as isize)) - } else { - let y = m1 * b2v + b1; - Some(Point::new(b2v as isize, y as isize)) - } - } else { - None - } - } else { - None - } -} - -// Threshold to reject points if they don't fit on the best-fit line -const OUTLIER_THRESHOLD: f32 = 0.5; -const OUTLIER_ITERS: usize = 5; -#[derive(Clone)] -pub struct LineDerivation { - pub equation: Option<(f32, f32)>, - pub independent_axis: Axis, - pub data_points: Vec, -} -impl LineDerivation { - pub fn new(axis: Axis) -> Self { - LineDerivation { equation: None, independent_axis: axis, data_points: Vec::new() } - } - - pub fn push(&mut self, p: Point) { self.data_points.push(p); } - - /// This implementation heavily relies on f32, so it is slow on an embedded processor; however, - /// we need the precision and the solving should be done only rarely. - pub fn solve(&mut self) { - let mut m_guess: f32 = 0.0; - let mut b_guess: f32 = 0.0; - let mut converged_in = 0; - let mut points = self.data_points.clone(); - for guesses in 0..OUTLIER_ITERS { - let mut filtered_points = Vec::::new(); - let mut residuals = Vec::::new(); - converged_in = guesses; - // guess a best-fit line - (m_guess, b_guess) = least_squares_fit(&points, self.independent_axis); - // compute the residuals of the points to the guessed line - for &p in points.iter() { - let (x, y) = adjust_axis(p, self.independent_axis); - let predicted_y = m_guess * x + b_guess; - residuals.push((y - predicted_y).abs()); - } - // extract the median residual - let mut sorted_residuals = residuals.clone(); - sorted_residuals - .sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(core::cmp::Ordering::Greater)); - let threshold = (sorted_residuals[points.len() / 2] * 1.5).max(OUTLIER_THRESHOLD); - - for (i, &p) in points.iter().enumerate() { - if residuals[i] <= threshold { - filtered_points.push(p); - } - } - if filtered_points.len() == points.len() { - break; - } else { - points = filtered_points; - } - } - log::info!("Solver converged in {} iterations", converged_in); - self.equation = Some((m_guess, b_guess)) - } -} - -pub struct Corner { - pub finder_ref: Option, - pub h_line: LineDerivation, - pub v_line: LineDerivation, -} -impl Default for Corner { - fn default() -> Self { - Self { finder_ref: None, h_line: LineDerivation::new(Axis::X), v_line: LineDerivation::new(Axis::Y) } - } -} - -#[derive(Default)] -pub struct QrCorners { - // Relies on the property that Direction enumerates and iterates consistently as a usize from 0-3, - // which covers each of the four corner directions exactly and uniquely. - corners: [Corner; 4], - width: isize, - height: isize, - derived_corner: Direction, - finder_width: usize, - // destination QR image in pixels per side (as a perfect square) - dst_pixels: usize, -} -impl QrCorners { - pub fn from_finders(points: &[Point; 3], dimensions: Point, finder_width: usize) -> Option { - let x_half = dimensions.x / 2; - let y_half = dimensions.y / 2; - - let mut qrc = QrCorners::default(); - qrc.width = dimensions.x; - qrc.height = dimensions.y; - qrc.finder_width = finder_width; - qrc.dst_pixels = dimensions.x.min(dimensions.y) as usize; - - for &p in points { - if p.x < x_half && p.y < y_half { - qrc.corners[Direction::SouthWest as usize].finder_ref = Some(p); - } else if p.x < x_half && p.y >= y_half { - qrc.corners[Direction::NorthWest as usize].finder_ref = Some(p); - } else if p.x >= x_half && p.y < y_half { - qrc.corners[Direction::SouthEast as usize].finder_ref = Some(p); - } else if p.x >= x_half && p.y >= y_half { - qrc.corners[Direction::NorthEast as usize].finder_ref = Some(p); - } - } - - // check that at least three corners are filled - if qrc.corners.iter().map(|c| if c.finder_ref.is_some() { 1 } else { 0 }).sum::() == 3 { - for (dir_index, corner) in qrc.corners.iter().enumerate() { - if corner.finder_ref.is_none() { - qrc.derived_corner = Direction::try_from(dir_index).unwrap(); - } - } - Some(qrc) - } else { - None - } - } - - #[allow(dead_code)] - pub fn derived_corner(&self) -> Direction { self.derived_corner } - - #[allow(dead_code)] - pub fn center_point(&self, dir: Direction) -> Option { self.corners[dir as usize].finder_ref } - - fn outline_search(&mut self, ir: &mut ImageRoi) { - for (d, corner) in self.corners.iter_mut().enumerate() { - let direction = Direction::try_from(d).unwrap(); - // this test ensures we automatically skip the "missing" corner - if let Some(p) = corner.finder_ref { - ir.set_roi( - Point::new( - (p.x - self.finder_width as isize / 2).max(0), - (p.y + self.finder_width as isize / 2).min(ir.height as isize), - ), - Point::new( - (p.x + self.finder_width as isize / 2).min(ir.width as isize), - (p.y - self.finder_width as isize / 2).max(0), - ), - ); - let signs: Point = direction.into(); - - let mut y_range = if signs.y < 0 { - DirRange::new(0, ir.roi_height(), true) - } else { - DirRange::new(0, ir.roi_height(), false) - }; - let mut x_range = if signs.x < 0 { - DirRange::new(0, ir.roi_width(), true) - } else { - DirRange::new(0, ir.roi_width(), false) - }; - - loop { - if let Some(y) = y_range.next() { - x_range.reset(); - loop { - if let Some(x) = x_range.next() { - if Color::Black - == ir.get_roi_binary_pixel(Point::new(x as isize, y as isize)).unwrap() - { - corner.v_line.push( - ir.roi_to_absolute(Point::new(x as isize, y as isize)).unwrap(), - ); - break; - } - } else { - break; - } - } - } else { - break; - } - } - corner.v_line.solve(); - - y_range.reset(); - x_range.reset(); - - loop { - if let Some(x) = x_range.next() { - y_range.reset(); - loop { - if let Some(y) = y_range.next() { - if Color::Black - == ir.get_roi_binary_pixel(Point::new(x as isize, y as isize)).unwrap() - { - corner.h_line.push( - ir.roi_to_absolute(Point::new(x as isize, y as isize)).unwrap(), - ); - break; - } - } else { - break; - } - } - } else { - break; - } - } - corner.h_line.solve(); - } - } - } - - /// Returns a (src, dst) tuple of point mappings for homomorphic transformation. - /// The destination is the four corners of image shape as specified when the structure - /// was made, with a margin added. - /// - /// The margin should be negative for the corners to go in toward the center. - /// - /// Canonical orientation for QR code should be like this: - /// +------+ - /// |O .| - /// | | - /// ^ |O O| - /// | +______+ - /// (0,0) -> - /// - /// Thus we want to orient so the discovered corner is in the NorthEast direction. - pub fn mapping(&mut self, ir: &mut ImageRoi, margin: isize) -> ([Option; 4], [Option; 4]) { - // adjust the final target image size by the specified margin - self.dst_pixels = (self.dst_pixels as isize + 2 * margin) as usize; - - // first, search for the lines that define the outline of the QR code - self.outline_search(ir); - - let mut src = [None; 4]; - let mut dst = [None; 4]; - let mut uk_direction: Option = None; - - for (i, corner) in self.corners.iter().enumerate() { - if let Some(_p) = corner.finder_ref { - // This is a known finder corner: - // derive the corner from the extracted h and v lines along the finder pattern - src[i] = point_from_hv_lines(&corner.h_line, &corner.v_line); - } else { - uk_direction = Direction::try_from(i).ok(); - // This is the unknown corner: - // derive the corner from the h and v lines from the nearest finders' lines - let h_line = match Direction::try_from(i) { - Ok(Direction::NorthWest) => &self.corners[Direction::NorthEast as usize].h_line, - Ok(Direction::NorthEast) => &self.corners[Direction::NorthWest as usize].h_line, - Ok(Direction::SouthWest) => &self.corners[Direction::SouthEast as usize].h_line, - Ok(Direction::SouthEast) => &self.corners[Direction::SouthWest as usize].h_line, - _ => panic!("Bad index"), - }; - let v_line = match Direction::try_from(i) { - Ok(Direction::NorthWest) => &self.corners[Direction::SouthWest as usize].v_line, - Ok(Direction::NorthEast) => &self.corners[Direction::SouthEast as usize].v_line, - Ok(Direction::SouthWest) => &self.corners[Direction::NorthWest as usize].v_line, - Ok(Direction::SouthEast) => &self.corners[Direction::NorthEast as usize].v_line, - _ => panic!("Bad index"), - }; - // draw_line(debug, &h_line, [255, 255, 0]); - // draw_line(debug, &v_line, [255, 255, 0]); - src[i] = point_from_hv_lines(h_line, v_line); - } - let min = -margin; - let max = self.dst_pixels as isize + margin; - dst[i] = match Direction::try_from(i) { - Ok(Direction::NorthWest) => Some(Point::new(min, max)), - Ok(Direction::NorthEast) => Some(Point::new(max, max)), - Ok(Direction::SouthWest) => Some(Point::new(min, min)), - Ok(Direction::SouthEast) => Some(Point::new(max, min)), - _ => None, - }; - } - - // now shuffle the dst points such that the unknown corner is in the - // NorthEast direction - let mut shuffle: [Option; 4] = [None; 4]; - shuffle.copy_from_slice(&dst); - if let Some(uk_corner) = uk_direction { - let corner_index: usize = uk_corner.into(); - if corner_index != 0 { - for _ in 0..corner_index { - shuffle[1..].copy_from_slice(&dst[..3]); - shuffle[0] = dst[3]; - dst.copy_from_slice(&shuffle); - } - } - } - (src, shuffle) - } - - pub fn qr_pixels(&self) -> usize { self.dst_pixels } -} - #[derive(Copy, Clone, Default, Debug)] struct FinderSeq { /// run length of the pixels leading up to the current position @@ -520,218 +94,6 @@ impl BitXor for Color { fn bitxor(self, rhs: Self) -> Self::Output { if self == rhs { Color::Black } else { Color::White } } } -#[derive(Copy, Clone, Debug)] -/// We use Direction as both a way to encode a meaning and a unique array index. -#[repr(usize)] -pub enum Direction { - NorthWest = 0, - NorthEast = 1, - SouthWest = 2, - SouthEast = 3, - North = 4, - West = 5, - East = 6, - South = 7, -} -impl Default for Direction { - fn default() -> Self { Self::NorthWest } -} -impl Into for Direction { - fn into(self) -> Point { - use Direction::*; - match self { - North => Point::new(0, 1), - West => Point::new(-1, 0), - East => Point::new(1, 0), - South => Point::new(0, -1), - NorthWest => Point::new(-1, 1), - NorthEast => Point::new(1, 1), - SouthWest => Point::new(-1, -1), - SouthEast => Point::new(1, -1), - } - } -} -impl Into for Direction { - fn into(self) -> usize { - use Direction::*; - match self { - NorthWest => 0, - NorthEast => 1, - SouthWest => 2, - SouthEast => 3, - North => 4, - West => 5, - East => 6, - South => 7, - } - } -} -impl TryFrom for Direction { - type Error = &'static str; - - fn try_from(value: usize) -> Result { - use Direction::*; - match value { - 0 => Ok(NorthWest), - 1 => Ok(NorthEast), - 2 => Ok(SouthWest), - 3 => Ok(SouthEast), - 4 => Ok(North), - 5 => Ok(West), - 6 => Ok(East), - 7 => Ok(South), - _ => Err("Invalid direction coding"), - } - } -} -impl Direction { - pub fn eight_way_iter() -> EightConnectedIter { EightConnectedIter { index: 0 } } -} -pub struct EightConnectedIter { - index: usize, -} - -impl Iterator for EightConnectedIter { - type Item = Direction; - - fn next(&mut self) -> Option { - let direction = match self.index { - 0 => Some(Direction::NorthWest), - 1 => Some(Direction::NorthEast), - 2 => Some(Direction::SouthWest), - 3 => Some(Direction::SouthEast), - 4 => Some(Direction::North), - 5 => Some(Direction::West), - 6 => Some(Direction::East), - 7 => Some(Direction::South), - _ => None, - }; - self.index += 1; - direction - } -} - -/// (0, 0) is at the lower left corner -pub struct ImageRoi<'a> { - pub data: &'a mut [u8], - pub width: usize, - pub height: usize, - thresh: u8, - // coordinates of a subimage, if set. The ROI includes these points. - x0: usize, - x1: usize, - y0: usize, - y1: usize, - iter_row: RefCell, - col_iter_start: usize, - col_iter_row_index: RefCell, - iter_col: RefCell, -} -impl<'a> ImageRoi<'a> { - pub fn new(data: &'a mut [u8], dimensions: Point, thresh: u8) -> Self { - // ROI is default the entire area - Self { - data, - width: dimensions.x as usize, - height: dimensions.y as usize, - thresh, - x0: 0, - x1: dimensions.x as usize, - y0: 0, - y1: dimensions.y as usize, - iter_row: RefCell::new(0), - col_iter_row_index: RefCell::new(0), - col_iter_start: 0, - iter_col: RefCell::new(0), - } - } - - pub fn binarize(&self, luma: u8) -> Color { if luma > self.thresh { Color::White } else { Color::Black } } - - pub fn get_pixel(&self, x: usize, y: usize) -> u8 { self.data[x + y * self.width] } - - pub fn get_roi_binary_pixel(&self, roi_point: Point) -> Option { - if let Some(abs_p) = self.roi_to_absolute(roi_point) { - let p = self.get_pixel(abs_p.x as usize, abs_p.y as usize); - Some(self.binarize(p)) - } else { - None - } - } - - pub fn set_roi(&mut self, tl: Point, br: Point) { - assert!(tl.x >= 0); - assert!(tl.y >= 0); - assert!(br.x >= 0); - assert!(br.y >= 0); - assert!(br.x >= tl.x); - assert!(tl.y >= br.y); - self.x0 = tl.x as usize; - self.x1 = br.x as usize; - self.y0 = br.y as usize; - self.y1 = tl.y as usize; - *self.iter_row.borrow_mut() = 0; - *self.iter_col.borrow_mut() = 0; - *self.col_iter_row_index.borrow_mut() = 0; - self.col_iter_start = 0; - } - - pub fn roi_width(&self) -> usize { self.x1 - self.x0 } - - pub fn roi_height(&self) -> usize { self.y1 - self.y0 } - - pub fn roi_to_absolute(&self, point: Point) -> Option { - let x = point.x + self.x0 as isize; - let y = point.y + self.y0 as isize; - if x >= 0 && x < self.width as isize && y >= 0 && y < self.height as isize { - Some(Point::new(x, y)) - } else { - None - } - } - - #[allow(dead_code)] - pub fn absolute_to_roi(&self, point: Point) -> Option { - let x = point.x - self.x0 as isize; - let y = point.y - self.y0 as isize; - if x >= 0 && x < self.x1 as isize && y >= 0 && y < self.y1 as isize { - Some(Point::new(x, y)) - } else { - None - } - } - - pub fn neighbor_luma(&self, point: Point) -> Option { - let x = point.x; - let y = point.y; - let mut accumulated_luma = 0; - let mut total_counted = 0; - if x >= 0 && x < self.width as isize && y >= 0 && y < self.height as isize { - for direction in Direction::eight_way_iter() { - total_counted += 1; - let d: Point = direction.into(); - let p: Point = d + point; - accumulated_luma += self.data[(p.y * (self.width as isize) + p.x).max(0) as usize] as usize; - } - } - if total_counted > 0 { Some((accumulated_luma / total_counted) as u8) } else { None } - } -} - -impl<'a> Iterator for ImageRoi<'_> { - type Item = u8; - - fn next(&mut self) -> Option { - let cur_row = *self.col_iter_row_index.borrow(); - if cur_row < self.y1 - self.y0 { - *self.col_iter_row_index.borrow_mut() = cur_row + 1; - Some(self.data[(self.y0 + cur_row) * self.width + self.x0 + self.col_iter_start]) - } else { - None - } - } -} - struct SeqBuffer { buffer: [Option; 5], start: usize, @@ -880,33 +242,3 @@ pub fn find_finders(candidates: &mut Vec, image: &[u8], thresh: u8, strid } if candidates.len() != 0 { candidate_width / candidates.len() } else { 0 } } - -const MODULES_BY_VERSION: [usize; 40] = [ - 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, - 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, -]; -const FP_SHIFT: usize = 1 << 16; -/// Returns tuple of (version, expected_modules) -pub fn guess_code_version(finder_width_pix: usize, qr_width_pix: usize) -> (usize, usize) { - let incoming_ratio = ((qr_width_pix * FP_SHIFT) / finder_width_pix) as isize; - let mut candidate_ratio = usize::MAX; - let mut candidate = 0; - let mut returned_modules = 0; - for (i, &modules) in MODULES_BY_VERSION.iter().enumerate() { - let version_ratio = ((modules * FP_SHIFT) / 7) as isize; - log::debug!( - "{}: version_ratio {} incoming_ratio {} diff {}", - i + 1, - version_ratio, - incoming_ratio, - version_ratio - incoming_ratio - ); - let prev_candidate = candidate_ratio; - candidate_ratio = candidate_ratio.min((version_ratio - incoming_ratio).abs() as usize); - if prev_candidate != candidate_ratio { - candidate = i + 1; - returned_modules = modules; - } - } - (candidate, returned_modules) -}