From 5ffacf63f32178a7f7c3ccde97635c2774fbe767 Mon Sep 17 00:00:00 2001 From: bunnie Date: Thu, 20 Feb 2025 15:06:15 +0800 Subject: [PATCH] cleanup bao-video to use the new API also condense main into a single routine again, now that we have gone through the trouble of creating interoperable APIs. --- services/bao-video/Cargo.toml | 10 +- services/bao-video/src/api.rs | 2 + services/bao-video/src/hosted.rs | 27 -- services/bao-video/src/hw.rs | 455 ---------------------------- services/bao-video/src/main.rs | 495 ++++++++++++++++++++++++++++++- 5 files changed, 495 insertions(+), 494 deletions(-) delete mode 100644 services/bao-video/src/hosted.rs delete mode 100644 services/bao-video/src/hw.rs diff --git a/services/bao-video/Cargo.toml b/services/bao-video/Cargo.toml index 1f7984b0d..044c629b5 100644 --- a/services/bao-video/Cargo.toml +++ b/services/bao-video/Cargo.toml @@ -17,7 +17,8 @@ cramium-hal = { path = "../../libs/cramium-hal", features = [ ], optional = true } cram-hal-service = { path = "../cram-hal-service", optional = true } ux-api = { path = "../../libs/ux-api" } -cramium-emu = { path = "../../libs/cramium-emu", optional = true } +cramium-emu = { path = "../cramium-emu", optional = true } +cramium-api = { path = "../../libs/cramium-api" } # QR decoding libm = { version = "0.2.8" } @@ -30,9 +31,12 @@ locales = { path = "../../locales" } # This crate is baosec specific; these flags switch between hosted mode emulation # (for UI development) and actual hardware targets. In UI development all camera data # is "faked" -board-baosec = ["cram-hal-service"] +board-baosec = ["cram-hal-service", "cramium-hal"] hosted-baosec = ["cramium-emu"] cramium-soc = ["utralib/cramium-soc"] decongest-udma = [] -default = ["decongest-udma"] +default = [ + "decongest-udma", + # "hosted-baosec", +] # hosted-baosec should be removed for commit, this is only here to make vscode happy diff --git a/services/bao-video/src/api.rs b/services/bao-video/src/api.rs index 4a040ec2b..9cabff441 100644 --- a/services/bao-video/src/api.rs +++ b/services/bao-video/src/api.rs @@ -3,3 +3,5 @@ pub enum Opcode { CamIrq, InvalidCall, } + +pub const SERVER_NAME_GFX: &str = "_Graphics_"; diff --git a/services/bao-video/src/hosted.rs b/services/bao-video/src/hosted.rs deleted file mode 100644 index accddf1b4..000000000 --- a/services/bao-video/src/hosted.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::Opcode; - -pub fn wrapped_main() -> ! { - log_server::init_wait().unwrap(); - log::set_max_level(log::LevelFilter::Info); - - let xns = xous_names::XousNames::new().unwrap(); - let sid = xns.register_name("bao video subsystem", None).expect("can't register server"); - - let tt = ticktimer::Ticktimer::new().unwrap(); - - let mut msg_opt = None; - loop { - xous::reply_and_receive_next(sid, &mut msg_opt).unwrap(); - let msg = msg_opt.as_mut().unwrap(); - let opcode = num_traits::FromPrimitive::from_usize(msg.body.id()).unwrap_or(Opcode::InvalidCall); - log::debug!("{:?}", opcode); - match opcode { - Opcode::CamIrq => { - unimplemented!() - } - Opcode::InvalidCall => { - log::error!("Invalid call to bao video server: {:?}", msg); - } - } - } -} diff --git a/services/bao-video/src/hw.rs b/services/bao-video/src/hw.rs deleted file mode 100644 index 8b0734fea..000000000 --- a/services/bao-video/src/hw.rs +++ /dev/null @@ -1,455 +0,0 @@ -use cramium_hal::iox::{IoxPort, IoxValue}; -use cramium_hal::sh1107::{Mono, Oled128x128}; -use cramium_hal::udma::PeriphId; -use num_traits::ToPrimitive; -use utralib::utra; - -pub fn blit_to_display(sh1107: &mut Oled128x128, frame: &[u8], display_cleared: bool) { - for (y, row) in frame.chunks(IMAGE_WIDTH).enumerate() { - if y & 1 == 0 { - for (x, &pixval) in row.iter().enumerate() { - if x & 1 == 0 { - if x < sh1107.dimensions().x as usize * 2 - && y < sh1107.dimensions().y as usize * 2 - (gfx::CHAR_HEIGHT as usize + 1) * 2 - { - let luminance = pixval & 0xff; - if luminance > BW_THRESH { - sh1107.put_pixel(Point::new(x as isize / 2, y as isize / 2), Mono::White.into()); - } else { - // optimization to avoid some computation if we're blitting to an already-black - // buffer - if !display_cleared { - sh1107.put_pixel( - Point::new(x as isize / 2, y as isize / 2), - Mono::Black.into(), - ); - } - } - } else { - break; - } - } - } - } - } -} - -#[repr(align(32))] -struct CamIrq { - csr: utralib::CSR, - cid: u32, -} - -fn handle_irq(_irq_no: usize, arg: *mut usize) { - let cam_irq: &mut CamIrq = unsafe { &mut *(arg as *mut CamIrq) }; - // clear the pending interrupt - assume it's just the camera for now - let pending = cam_irq.csr.r(utra::irqarray8::EV_PENDING); - cam_irq.csr.wo(utra::irqarray8::EV_PENDING, pending); - - // activate the handler - xous::try_send_message( - cam_irq.cid, - xous::Message::new_scalar(Opcode::CamIrq.to_usize().unwrap(), pending as usize, 0, 0, 0), - ) - .ok(); -} - -pub fn wrapped_main() -> ! { - log_server::init_wait().unwrap(); - log::set_max_level(log::LevelFilter::Info); - - let xns = xous_names::XousNames::new().unwrap(); - let sid = xns.register_name("bao video subsystem", None).expect("can't register server"); - - let tt = ticktimer::Ticktimer::new().unwrap(); - - let iox = cram_hal_service::IoxHal::new(); - let udma_global = cram_hal_service::UdmaGlobal::new(); - let mut i2c = cram_hal_service::I2c::new(); - - let mut sh1107 = cramium_hal::sh1107::Oled128x128::new(cram_hal_service::PERCLK, &iox, &udma_global); - sh1107.init(); - sh1107.buffer_swap(); - sh1107.draw(); - - // setup camera pins - let (cam_pdwn_bnk, cam_pdwn_pin) = cramium_hal::board::setup_ov2640_pins(&iox); - // disable camera powerdown - iox.set_gpio_pin_value(cam_pdwn_bnk, cam_pdwn_pin, cramium_hal::iox::IoxValue::Low); - udma_global.udma_clock_config(PeriphId::Cam, true); - // this is safe because we turned on the clocks before calling it - let mut cam = unsafe { cramium_hal::ov2640::Ov2640::new().expect("couldn't allocate camera") }; - - tt.sleep_ms(100).ok(); - - let (pid, mid) = cam.read_id(&mut i2c); - log::info!("Camera pid {:x}, mid {:x}", pid, mid); - cam.init(&mut i2c, cramium_hal::ov2640::Resolution::Res320x240); - cam.poke(&mut i2c, 0xFF, 0x00); - cam.poke(&mut i2c, 0xDA, 0x01); // YUV LE - tt.sleep_ms(1).ok(); - - let (cols, _rows) = cam.resolution(); - let border = (cols - IMAGE_WIDTH) / 2; - cam.set_slicing((border, 0), (cols - border, IMAGE_HEIGHT)); - log::info!("320x240 resolution setup with 256x240 slicing"); - - #[cfg(feature = "decongest-udma")] - log::info!("Decongest udma option enabled."); - - // register interrupt handler - let irq = xous::syscall::map_memory( - xous::MemoryAddress::new(utra::irqarray8::HW_IRQARRAY8_BASE), - None, - 4096, - xous::MemoryFlags::R | xous::MemoryFlags::W, - ) - .expect("couldn't map IRQ CSR range"); - let mut irq_csr = utralib::CSR::new(irq.as_mut_ptr() as *mut u32); - irq_csr.wo(utra::irqarray8::EV_PENDING, 0xFFFF); // clear any pending interrupts - - let cid = xous::connect(sid).unwrap(); // self-connection always succeeds - let cam_irq = CamIrq { csr: utralib::CSR::new(irq.as_mut_ptr() as *mut u32), cid }; - let irq_arg = &cam_irq as *const CamIrq as *mut usize; - log::debug!("irq_arg: {:x}", irq_arg as usize); - xous::claim_interrupt(utra::irqarray8::IRQARRAY8_IRQ, handle_irq, irq_arg).expect("couldn't claim IRQ8"); - // enable camera Rx IRQ - irq_csr.wfo(utra::irqarray8::EV_ENABLE_CAM_RX, 1); - - // kick off the first acquisition synchronous to the frame sync. With NTO/DAR-704 fix we - // could turn on the sync_sof field and skip this step. - while iox.get_gpio_pin_value(IoxPort::PB, 9) == IoxValue::High {} - cam.capture_async(); - - let mut frames = 0; - let mut frame = [0u8; IMAGE_WIDTH * IMAGE_HEIGHT]; - let mut decode_success; - let mut msg_opt = None; - loop { - xous::reply_and_receive_next(sid, &mut msg_opt).unwrap(); - let msg = msg_opt.as_mut().unwrap(); - let opcode = num_traits::FromPrimitive::from_usize(msg.body.id()).unwrap_or(Opcode::InvalidCall); - log::debug!("{:?}", opcode); - match opcode { - Opcode::CamIrq => { - #[cfg(not(feature = "decongest-udma"))] - cam.capture_async(); - - // copy the camera data to our FB - let fb: &[u32] = cam.rx_buf(); - // fb is an array of IMAGE_WIDTH x IMAGE_HEIGHT x u16 - // frame is an array of IMAGE_WIDTH x IMAGE_HEIGHT x u8 - // Take only the "Y" channel out of the fb array and write it to frame, but do it - // such that we are fetching a u32 each read from fb as this matches the native - // width of the bus (because fb is non-cacheable reading u16 ends up fetching the - // same word twice, then masking it at the CPU side in hardware). Also, the fb - // is slow to access relative to main memory. - // - // Also, commit the data to `frame` in inverse line order, e.g. flip the image - // vertically. - for (y_src, line) in fb.chunks(IMAGE_WIDTH / 2).enumerate() { - for (x_src, &u32src) in line.iter().enumerate() { - frame[(IMAGE_HEIGHT - y_src - 1) * IMAGE_WIDTH + 2 * x_src] = (u32src & 0xff) as u8; - frame[(IMAGE_HEIGHT - y_src - 1) * IMAGE_WIDTH + 2 * x_src + 1] = - ((u32src >> 16) & 0xff) as u8; - } - } - frames += 1; - - let mut candidates = Vec::::new(); - decode_success = false; - log::info!("------------- SEARCH {} -----------", frames); - let finder_width = qr::find_finders(&mut candidates, &frame, BW_THRESH, IMAGE_WIDTH) as isize; - if candidates.len() == 3 { - let candidates_orig = candidates.clone(); - let mut x_candidates: [Point; 3] = [Point::new(0, 0); 3]; - // apply homography to generate a new buffer for processing - let mut aligned = Vec::new(); - let mut qr_pixels: Option = None; - if let Some(mut qr_corners) = qr::QrCorners::from_finders( - &candidates.try_into().unwrap(), - Point::new(IMAGE_WIDTH as isize, IMAGE_HEIGHT as isize), - // add a search margin on the finder width - (finder_width + (qr::FINDER_SEARCH_MARGIN * finder_width) / (1 + 1 + 3 + 1 + 1)) - as usize, - ) { - let dims = Point::new(IMAGE_WIDTH as isize, IMAGE_HEIGHT as isize); - let mut il = qr::ImageRoi::new(&mut frame, dims, BW_THRESH); - let (src, dst) = qr_corners.mapping(&mut il, qr::HOMOGRAPHY_MARGIN); - let mut src_f: [(f32, f32); 4] = [(0.0, 0.0); 4]; - let mut dst_f: [(f32, f32); 4] = [(0.0, 0.0); 4]; - let mut all_found = true; - for (s, s_f32) in src.iter().zip(src_f.iter_mut()) { - if let Some(p) = s { - *s_f32 = p.to_f32(); - } else { - all_found = false; - } - } - for (d, d_f32) in dst.iter().zip(dst_f.iter_mut()) { - if let Some(p) = d { - *d_f32 = p.to_f32(); - } else { - all_found = false; - } - } - - if all_found { - if let Some(h) = homography::find_homography(src_f, dst_f) { - if let Some(h_inv) = h.try_inverse() { - log::info!("{:?}", h_inv); - let h_inv_fp = homography::matrix3_to_fixp(h_inv); - log::info!("{:?}", h_inv_fp); - - aligned = vec![0u8; qr_corners.qr_pixels() * qr_corners.qr_pixels()]; - // iterate through pixels and apply homography - for y in 0..qr_corners.qr_pixels() { - for x in 0..qr_corners.qr_pixels() { - let (x_src, y_src) = homography::apply_fixp_homography( - &h_inv_fp, - (x as i32, y as i32), - ); - if (x_src as i32 >= 0) - && ((x_src as i32) < dims.x as i32) - && (y_src as i32 >= 0) - && ((y_src as i32) < dims.y as i32) - { - // println!("{},{} -> {},{}", x_src as i32, y_src as i32, x, - // y); - aligned[qr_corners.qr_pixels() * y as usize + x as usize] = - frame[IMAGE_WIDTH * y_src as usize + x_src as usize]; - } else { - aligned[qr_corners.qr_pixels() * y as usize + x as usize] = - 255; - } - } - } - - // we can also know the location of the finders by transforming them - let h_fp = homography::matrix3_to_fixp(h); - for (i, &c) in candidates_orig.iter().enumerate() { - let (x, y) = homography::apply_fixp_homography( - &h_fp, - (c.x as i32, c.y as i32), - ); - x_candidates[i] = Point::new(x as isize, y as isize); - } - qr_pixels = Some(qr_corners.qr_pixels()); - } - } - } - } - - if let Some(qr_width) = qr_pixels { - // show the transformed/aligned frame - frame.fill(255); - for (dst_line, src_line) in - frame.chunks_mut(IMAGE_WIDTH).zip(aligned.chunks(qr_width)) - { - // "center up" the QR in the middle by the estimated margin - // this is just a perceptual trick to prevent users from shifting the position of - // the camera - for (dst, &src) in dst_line[qr::HOMOGRAPHY_MARGIN.abs() as usize..] - .iter_mut() - .zip(src_line.iter()) - { - *dst = src; - } - } - blit_to_display(&mut sh1107, &frame, true); - - // we now have a QR code in "canonical" orientation, with a - // known width in pixels - for &x in x_candidates.iter() { - log::info!("transformed finder location {:?}", x); - } - - // Confirm that the finders coordinates are valid - let mut checked_candidates = Vec::::new(); - let x_finder_width = - qr::find_finders(&mut checked_candidates, &aligned, BW_THRESH, qr_width as _) - as isize; - log::info!("x_finder width: {}", x_finder_width); - - // check that the new coordinates are within delta pixels of the original - const XFORM_DELTA: isize = 2; - let mut deltas = Vec::::new(); - for c in checked_candidates { - log::info!("x_point: {:?}", c); - for &xformed in x_candidates.iter() { - let delta = xformed - c; - log::info!("delta: {:?}", delta); - if delta.x.abs() <= XFORM_DELTA && delta.y.abs() <= XFORM_DELTA { - deltas.push(delta); - } - } - } - if deltas.len() == 3 { - let (version, modules) = qr::guess_code_version( - x_finder_width as usize, - (qr_width as isize + qr::HOMOGRAPHY_MARGIN * 2) as usize, - ); - - log::info!("image dims: {}", qr_width); - log::info!("guessed version: {}, modules: {}", version, modules); - log::info!( - "QR symbol width in pixels: {}", - qr_width - 2 * (qr::HOMOGRAPHY_MARGIN.abs() as usize) - ); - - let qr = qr::ImageRoi::new( - &mut aligned, - Point::new(qr_width as _, qr_width as _), - BW_THRESH, - ); - let grid = modules::stream_to_grid( - &qr, - qr_width, - modules, - qr::HOMOGRAPHY_MARGIN.abs() as usize, - ); - - println!("grid len {}", grid.len()); - for y in 0..modules { - for x in 0..modules { - if grid[y * modules + x] { - print!("X"); - } else { - print!(" "); - } - } - println!(" {:2}", y); - } - let simple = rqrr::SimpleGrid::from_func(modules, |x, y| { - grid[(modules - 1) - x + y * modules] - }); - let grid = rqrr::Grid::new(simple); - match grid.decode() { - Ok((meta, content)) => { - log::info!("meta: {:?}", meta); - log::info!("************ {} ***********", content); - decode_success = true; - gfx::msg( - &mut sh1107, - &format!("{:?}", meta), - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); - gfx::msg( - &mut sh1107, - &format!("{:?}", content), - Point::new(0, 64), - Mono::White.into(), - Mono::Black.into(), - ); - } - Err(e) => { - log::info!("{:?}", e); - gfx::msg( - &mut sh1107, - &format!("{:?}", e), - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); - } - } - } else { - log::info!("Transformed image did not survive sanity check!"); - gfx::msg( - &mut sh1107, - "Hold device steady...", - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); - } - } else { - blit_to_display(&mut sh1107, &frame, true); - for c in candidates_orig.iter() { - log::debug!("****** candidate: {}, {} ******", c.x, c.y); - // remap image to screen coordinates (it's 2:1) - let c_screen = *c / 2; - // flip coordinates to match the camera data - // c_screen = Point::new(c_screen.x, sh1107.dimensions().y - 1 - c_screen.y); - qr::draw_crosshair(&mut sh1107, c_screen); - } - gfx::msg( - &mut sh1107, - "Align the QR code...", - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); - } - } else { - // blit raw camera fb to sh1107 - blit_to_display(&mut sh1107, &frame, true); - gfx::msg( - &mut sh1107, - "Scan QR code...", - Point::new(0, 0), - Mono::White.into(), - Mono::Black.into(), - ); - } - - // swap the double buffer and update to the display - sh1107.buffer_swap(); - sh1107.draw(); - if decode_success { - tt.sleep_ms(2000).ok(); - } - - // clear the front buffer - sh1107.clear(); - - // re-initiate the capture. This is done at the bottom of the loop because UDMA - // congestion leads to system instability. When this problem is solved, we would - // actually want to re-initiate the capture immediately (or leave it on continuous mode) - // to allow capture to process concurrently with the code. However, there is a bug - // in the SPIM block that prevents proper usage with high bus contention that should - // be fixed in NTO. - const TIMEOUT_MS: u64 = 100; - #[cfg(feature = "decongest-udma")] - { - let start = tt.elapsed_ms(); - let mut now = tt.elapsed_ms(); - // this is required because if we initiate the capture in the middle - // of a frame, we get an offset result. This should be fixed by DAR-704 - // on NTO if the pull request is accepted; in which case, we can just rely - // on setting bit 30 of the CFG_GLOBAL register which will cause any - // RX start request to align to the beginning of a frame automatically. - while iox.get_gpio_pin_value(IoxPort::PB, 9) == IoxValue::High - && ((now - start) < TIMEOUT_MS) - { - now = tt.elapsed_ms(); - } - if now - start >= TIMEOUT_MS { - log::info!("Timeout before capture_async()!"); - } - cam.capture_async(); - } - - // this is no longer the case because we're relying on interrupts to wake us up. - /* - // wait for the transfer to finish - let start = tt.elapsed_ms(); - let mut now = tt.elapsed_ms(); - use cramium_hal::udma::Udma; - while cam.udma_busy(cramium_hal::udma::Bank::Rx) && ((now - start) < TIMEOUT_MS) { - now = tt.elapsed_ms(); - // busy-wait to get better time resolution on when the frame ends - } - if now - start >= TIMEOUT_MS { - log::info!("Timeout before rx_buf()!"); - } - */ - } - Opcode::InvalidCall => { - log::error!("Invalid call to bao video server: {:?}", msg); - } - } - } -} diff --git a/services/bao-video/src/main.rs b/services/bao-video/src/main.rs index 7a50369ce..afbe2d944 100644 --- a/services/bao-video/src/main.rs +++ b/services/bao-video/src/main.rs @@ -7,16 +7,25 @@ mod modules; mod qr; use api::*; #[cfg(feature = "board-baosec")] -mod hw; +use cram_hal_service::{I2c, UdmaGlobal}; +use cramium_api::*; +#[cfg(feature = "hosted-baosec")] +use cramium_emu::{ + camera::Ov2640, + display::{MainThreadToken, Mono, Oled128x128, claim_main_thread}, + i2c::I2c, + udma::UdmaGlobal, +}; #[cfg(feature = "board-baosec")] -use hw::wrapped_main; +use cramium_hal::{ + ov2640::Ov2640, + sh1107::{MainThreadToken, Mono, Oled128x128, claim_main_thread}, +}; +use num_traits::ToPrimitive; +#[cfg(not(feature = "hosted-baosec"))] +use utralib::utra; -#[cfg(feature = "hosted-baosec")] -mod hosted; -#[cfg(feature = "hosted-baosec")] -use hosted::wrapped_main; - -// Scope of this crate: +// Scope of this crate: *No calls to modals* this can create dependency lockups. // // bao-video contains the platform-specific drivers for the baosec platform that pertain // to video: both the capture of video, as well as any operations involving drawing to @@ -46,7 +55,475 @@ pub const BW_THRESH: u8 = 128; // luck of the draw if the interpolation hits exactly right, or if we're roughly a module // off from ideal, which causes the data around that point to be interpreted incorrectly. +pub fn blit_to_display(sh1107: &mut Oled128x128, frame: &[u8], display_cleared: bool) { + for (y, row) in frame.chunks(IMAGE_WIDTH).enumerate() { + if y & 1 == 0 { + for (x, &pixval) in row.iter().enumerate() { + if x & 1 == 0 { + if x < sh1107.dimensions().x as usize * 2 + && y < sh1107.dimensions().y as usize * 2 - (gfx::CHAR_HEIGHT as usize + 1) * 2 + { + let luminance = pixval & 0xff; + if luminance > BW_THRESH { + sh1107.put_pixel(Point::new(x as isize / 2, y as isize / 2), Mono::White.into()); + } else { + // optimization to avoid some computation if we're blitting to an already-black + // buffer + if !display_cleared { + sh1107.put_pixel( + Point::new(x as isize / 2, y as isize / 2), + Mono::Black.into(), + ); + } + } + } else { + break; + } + } + } + } + } +} + +#[repr(align(32))] +#[cfg(not(feature = "hosted-baosec"))] +struct CamIrq { + csr: utralib::CSR, + cid: u32, +} + +#[cfg(not(feature = "hosted-baosec"))] +fn handle_irq(_irq_no: usize, arg: *mut usize) { + let cam_irq: &mut CamIrq = unsafe { &mut *(arg as *mut CamIrq) }; + // clear the pending interrupt - assume it's just the camera for now + let pending = cam_irq.csr.r(utra::irqarray8::EV_PENDING); + cam_irq.csr.wo(utra::irqarray8::EV_PENDING, pending); + + // activate the handler + xous::try_send_message( + cam_irq.cid, + xous::Message::new_scalar(Opcode::CamIrq.to_usize().unwrap(), pending as usize, 0, 0, 0), + ) + .ok(); +} +#[cfg(feature = "hosted-baosec")] +fn handle_irq(_irq_no: usize, arg: *mut usize) {} + fn main() -> ! { let stack_size = 1 * 1024 * 1024; - std::thread::Builder::new().stack_size(stack_size).spawn(wrapped_main).unwrap().join().unwrap() + claim_main_thread(move |main_thread_token| { + std::thread::Builder::new() + .stack_size(stack_size) + .spawn(move || wrapped_main(main_thread_token)) + .unwrap() + .join() + .unwrap() + }) +} + +pub fn wrapped_main(main_thread_token: MainThreadToken) -> ! { + log_server::init_wait().unwrap(); + log::set_max_level(log::LevelFilter::Info); + + let xns = xous_names::XousNames::new().unwrap(); + let sid = xns.register_name(crate::api::SERVER_NAME_GFX, None).expect("can't register server"); + + let tt = ticktimer::Ticktimer::new().unwrap(); + + let iox = IoxHal::new(); + let udma_global = UdmaGlobal::new(); + let mut i2c = I2c::new(); + + let mut sh1107 = Oled128x128::new(main_thread_token, cramium_api::PERCLK, &iox, &udma_global); + sh1107.init(); + sh1107.buffer_swap(); + sh1107.draw(); + + #[cfg(not(feature = "hosted-baosec"))] + { + // setup camera pins + let (cam_pdwn_bnk, cam_pdwn_pin) = cramium_hal::board::setup_ov2640_pins(&iox); + // disable camera powerdown + iox.set_gpio_pin_value(cam_pdwn_bnk, cam_pdwn_pin, IoxValue::Low); + } + udma_global.udma_clock_config(PeriphId::Cam, true); + // this is safe because we turned on the clocks before calling it + let mut cam = unsafe { Ov2640::new().expect("couldn't allocate camera") }; + + tt.sleep_ms(100).ok(); + + let (pid, mid) = cam.read_id(&mut i2c); + log::info!("Camera pid {:x}, mid {:x}", pid, mid); + cam.init(&mut i2c, cramium_api::camera::Resolution::Res320x240); + cam.poke(&mut i2c, 0xFF, 0x00); + cam.poke(&mut i2c, 0xDA, 0x01); // YUV LE + tt.sleep_ms(1).ok(); + + let (cols, _rows) = cam.resolution(); + let border = (cols - IMAGE_WIDTH) / 2; + cam.set_slicing((border, 0), (cols - border, IMAGE_HEIGHT)); + log::info!("320x240 resolution setup with 256x240 slicing"); + + #[cfg(feature = "decongest-udma")] + log::info!("Decongest udma option enabled."); + + let cid = xous::connect(sid).unwrap(); // self-connection always succeeds + // register interrupt handler + #[cfg(not(feature = "hosted-baosec"))] + { + let irq = xous::syscall::map_memory( + xous::MemoryAddress::new(utra::irqarray8::HW_IRQARRAY8_BASE), + None, + 4096, + xous::MemoryFlags::R | xous::MemoryFlags::W, + ) + .expect("couldn't map IRQ CSR range"); + let mut irq_csr = utralib::CSR::new(irq.as_mut_ptr() as *mut u32); + irq_csr.wo(utra::irqarray8::EV_PENDING, 0xFFFF); // clear any pending interrupts + + let cam_irq = CamIrq { csr: utralib::CSR::new(irq.as_mut_ptr() as *mut u32), cid }; + let irq_arg = &cam_irq as *const CamIrq as *mut usize; + log::debug!("irq_arg: {:x}", irq_arg as usize); + xous::claim_interrupt(utra::irqarray8::IRQARRAY8_IRQ, handle_irq, irq_arg) + .expect("couldn't claim IRQ8"); + // enable camera Rx IRQ + irq_csr.wfo(utra::irqarray8::EV_ENABLE_CAM_RX, 1); + } + + // kick off the first acquisition synchronous to the frame sync. With NTO/DAR-704 fix we + // could turn on the sync_sof field and skip this step. + while iox.get_gpio_pin_value(IoxPort::PB, 9) == IoxValue::High {} + cam.capture_async(); + + let mut frames = 0; + let mut frame = [0u8; IMAGE_WIDTH * IMAGE_HEIGHT]; + let mut decode_success; + let mut msg_opt = None; + loop { + xous::reply_and_receive_next(sid, &mut msg_opt).unwrap(); + let msg = msg_opt.as_mut().unwrap(); + let opcode = num_traits::FromPrimitive::from_usize(msg.body.id()).unwrap_or(Opcode::InvalidCall); + log::debug!("{:?}", opcode); + match opcode { + Opcode::CamIrq => { + #[cfg(not(feature = "decongest-udma"))] + cam.capture_async(); + + // copy the camera data to our FB + let fb: &[u32] = cam.rx_buf(); + // fb is an array of IMAGE_WIDTH x IMAGE_HEIGHT x u16 + // frame is an array of IMAGE_WIDTH x IMAGE_HEIGHT x u8 + // Take only the "Y" channel out of the fb array and write it to frame, but do it + // such that we are fetching a u32 each read from fb as this matches the native + // width of the bus (because fb is non-cacheable reading u16 ends up fetching the + // same word twice, then masking it at the CPU side in hardware). Also, the fb + // is slow to access relative to main memory. + // + // Also, commit the data to `frame` in inverse line order, e.g. flip the image + // vertically. + for (y_src, line) in fb.chunks(IMAGE_WIDTH / 2).enumerate() { + for (x_src, &u32src) in line.iter().enumerate() { + frame[(IMAGE_HEIGHT - y_src - 1) * IMAGE_WIDTH + 2 * x_src] = (u32src & 0xff) as u8; + frame[(IMAGE_HEIGHT - y_src - 1) * IMAGE_WIDTH + 2 * x_src + 1] = + ((u32src >> 16) & 0xff) as u8; + } + } + frames += 1; + + let mut candidates = Vec::::new(); + decode_success = false; + log::info!("------------- SEARCH {} -----------", frames); + let finder_width = qr::find_finders(&mut candidates, &frame, BW_THRESH, IMAGE_WIDTH) as isize; + if candidates.len() == 3 { + let candidates_orig = candidates.clone(); + let mut x_candidates: [Point; 3] = [Point::new(0, 0); 3]; + // apply homography to generate a new buffer for processing + let mut aligned = Vec::new(); + let mut qr_pixels: Option = None; + if let Some(mut qr_corners) = qr::QrCorners::from_finders( + &candidates.try_into().unwrap(), + Point::new(IMAGE_WIDTH as isize, IMAGE_HEIGHT as isize), + // add a search margin on the finder width + (finder_width + (qr::FINDER_SEARCH_MARGIN * finder_width) / (1 + 1 + 3 + 1 + 1)) + as usize, + ) { + let dims = Point::new(IMAGE_WIDTH as isize, IMAGE_HEIGHT as isize); + let mut il = qr::ImageRoi::new(&mut frame, dims, BW_THRESH); + let (src, dst) = qr_corners.mapping(&mut il, qr::HOMOGRAPHY_MARGIN); + let mut src_f: [(f32, f32); 4] = [(0.0, 0.0); 4]; + let mut dst_f: [(f32, f32); 4] = [(0.0, 0.0); 4]; + let mut all_found = true; + for (s, s_f32) in src.iter().zip(src_f.iter_mut()) { + if let Some(p) = s { + *s_f32 = p.to_f32(); + } else { + all_found = false; + } + } + for (d, d_f32) in dst.iter().zip(dst_f.iter_mut()) { + if let Some(p) = d { + *d_f32 = p.to_f32(); + } else { + all_found = false; + } + } + + if all_found { + if let Some(h) = homography::find_homography(src_f, dst_f) { + if let Some(h_inv) = h.try_inverse() { + log::info!("{:?}", h_inv); + let h_inv_fp = homography::matrix3_to_fixp(h_inv); + log::info!("{:?}", h_inv_fp); + + aligned = vec![0u8; qr_corners.qr_pixels() * qr_corners.qr_pixels()]; + // iterate through pixels and apply homography + for y in 0..qr_corners.qr_pixels() { + for x in 0..qr_corners.qr_pixels() { + let (x_src, y_src) = homography::apply_fixp_homography( + &h_inv_fp, + (x as i32, y as i32), + ); + if (x_src as i32 >= 0) + && ((x_src as i32) < dims.x as i32) + && (y_src as i32 >= 0) + && ((y_src as i32) < dims.y as i32) + { + // println!("{},{} -> {},{}", x_src as i32, y_src as i32, x, + // y); + aligned[qr_corners.qr_pixels() * y as usize + x as usize] = + frame[IMAGE_WIDTH * y_src as usize + x_src as usize]; + } else { + aligned[qr_corners.qr_pixels() * y as usize + x as usize] = + 255; + } + } + } + + // we can also know the location of the finders by transforming them + let h_fp = homography::matrix3_to_fixp(h); + for (i, &c) in candidates_orig.iter().enumerate() { + let (x, y) = homography::apply_fixp_homography( + &h_fp, + (c.x as i32, c.y as i32), + ); + x_candidates[i] = Point::new(x as isize, y as isize); + } + qr_pixels = Some(qr_corners.qr_pixels()); + } + } + } + } + + if let Some(qr_width) = qr_pixels { + // show the transformed/aligned frame + frame.fill(255); + for (dst_line, src_line) in + frame.chunks_mut(IMAGE_WIDTH).zip(aligned.chunks(qr_width)) + { + // "center up" the QR in the middle by the estimated margin + // this is just a perceptual trick to prevent users from shifting the position of + // the camera + for (dst, &src) in dst_line[qr::HOMOGRAPHY_MARGIN.abs() as usize..] + .iter_mut() + .zip(src_line.iter()) + { + *dst = src; + } + } + blit_to_display(&mut sh1107, &frame, true); + + // we now have a QR code in "canonical" orientation, with a + // known width in pixels + for &x in x_candidates.iter() { + log::info!("transformed finder location {:?}", x); + } + + // Confirm that the finders coordinates are valid + let mut checked_candidates = Vec::::new(); + let x_finder_width = + qr::find_finders(&mut checked_candidates, &aligned, BW_THRESH, qr_width as _) + as isize; + log::info!("x_finder width: {}", x_finder_width); + + // check that the new coordinates are within delta pixels of the original + const XFORM_DELTA: isize = 2; + let mut deltas = Vec::::new(); + for c in checked_candidates { + log::info!("x_point: {:?}", c); + for &xformed in x_candidates.iter() { + let delta = xformed - c; + log::info!("delta: {:?}", delta); + if delta.x.abs() <= XFORM_DELTA && delta.y.abs() <= XFORM_DELTA { + deltas.push(delta); + } + } + } + if deltas.len() == 3 { + let (version, modules) = qr::guess_code_version( + x_finder_width as usize, + (qr_width as isize + qr::HOMOGRAPHY_MARGIN * 2) as usize, + ); + + log::info!("image dims: {}", qr_width); + log::info!("guessed version: {}, modules: {}", version, modules); + log::info!( + "QR symbol width in pixels: {}", + qr_width - 2 * (qr::HOMOGRAPHY_MARGIN.abs() as usize) + ); + + let qr = qr::ImageRoi::new( + &mut aligned, + Point::new(qr_width as _, qr_width as _), + BW_THRESH, + ); + let grid = modules::stream_to_grid( + &qr, + qr_width, + modules, + qr::HOMOGRAPHY_MARGIN.abs() as usize, + ); + + println!("grid len {}", grid.len()); + for y in 0..modules { + for x in 0..modules { + if grid[y * modules + x] { + print!("X"); + } else { + print!(" "); + } + } + println!(" {:2}", y); + } + let simple = rqrr::SimpleGrid::from_func(modules, |x, y| { + grid[(modules - 1) - x + y * modules] + }); + let grid = rqrr::Grid::new(simple); + match grid.decode() { + Ok((meta, content)) => { + log::info!("meta: {:?}", meta); + log::info!("************ {} ***********", content); + decode_success = true; + gfx::msg( + &mut sh1107, + &format!("{:?}", meta), + Point::new(0, 0), + Mono::White.into(), + Mono::Black.into(), + ); + gfx::msg( + &mut sh1107, + &format!("{:?}", content), + Point::new(0, 64), + Mono::White.into(), + Mono::Black.into(), + ); + } + Err(e) => { + log::info!("{:?}", e); + gfx::msg( + &mut sh1107, + &format!("{:?}", e), + Point::new(0, 0), + Mono::White.into(), + Mono::Black.into(), + ); + } + } + } else { + log::info!("Transformed image did not survive sanity check!"); + gfx::msg( + &mut sh1107, + "Hold device steady...", + Point::new(0, 0), + Mono::White.into(), + Mono::Black.into(), + ); + } + } else { + blit_to_display(&mut sh1107, &frame, true); + for c in candidates_orig.iter() { + log::debug!("****** candidate: {}, {} ******", c.x, c.y); + // remap image to screen coordinates (it's 2:1) + let c_screen = *c / 2; + // flip coordinates to match the camera data + // c_screen = Point::new(c_screen.x, sh1107.dimensions().y - 1 - c_screen.y); + qr::draw_crosshair(&mut sh1107, c_screen); + } + gfx::msg( + &mut sh1107, + "Align the QR code...", + Point::new(0, 0), + Mono::White.into(), + Mono::Black.into(), + ); + } + } else { + // blit raw camera fb to sh1107 + blit_to_display(&mut sh1107, &frame, true); + gfx::msg( + &mut sh1107, + "Scan QR code...", + Point::new(0, 0), + Mono::White.into(), + Mono::Black.into(), + ); + } + + // swap the double buffer and update to the display + sh1107.buffer_swap(); + sh1107.draw(); + if decode_success { + tt.sleep_ms(2000).ok(); + } + + // clear the front buffer + sh1107.clear(); + + // re-initiate the capture. This is done at the bottom of the loop because UDMA + // congestion leads to system instability. When this problem is solved, we would + // actually want to re-initiate the capture immediately (or leave it on continuous mode) + // to allow capture to process concurrently with the code. However, there is a bug + // in the SPIM block that prevents proper usage with high bus contention that should + // be fixed in NTO. + const TIMEOUT_MS: u64 = 100; + #[cfg(feature = "decongest-udma")] + { + let start = tt.elapsed_ms(); + let mut now = tt.elapsed_ms(); + // this is required because if we initiate the capture in the middle + // of a frame, we get an offset result. This should be fixed by DAR-704 + // on NTO if the pull request is accepted; in which case, we can just rely + // on setting bit 30 of the CFG_GLOBAL register which will cause any + // RX start request to align to the beginning of a frame automatically. + while iox.get_gpio_pin_value(IoxPort::PB, 9) == IoxValue::High + && ((now - start) < TIMEOUT_MS) + { + now = tt.elapsed_ms(); + } + if now - start >= TIMEOUT_MS { + log::info!("Timeout before capture_async()!"); + } + cam.capture_async(); + } + + // this is no longer the case because we're relying on interrupts to wake us up. + /* + // wait for the transfer to finish + let start = tt.elapsed_ms(); + let mut now = tt.elapsed_ms(); + use cramium_hal::udma::Udma; + while cam.udma_busy(cramium_hal::udma::Bank::Rx) && ((now - start) < TIMEOUT_MS) { + now = tt.elapsed_ms(); + // busy-wait to get better time resolution on when the frame ends + } + if now - start >= TIMEOUT_MS { + log::info!("Timeout before rx_buf()!"); + } + */ + } + Opcode::InvalidCall => { + log::error!("Invalid call to bao video server: {:?}", msg); + } + } + } }