From abb245543c3709865d256336f4f5cfa9dcb1d87d Mon Sep 17 00:00:00 2001 From: bunnie Date: Tue, 17 Sep 2024 18:29:24 +0800 Subject: [PATCH] create a planning directory, and add the rkyv test The purpose of the planning directory is a place where we can stash code for reference in various github issues and project plans without having to integrate it into the code base. This is for large changes to the code base that require extensive planning and discussion prior to integration. --- Cargo.toml | 2 + planning/rkyv-migration1/Cargo.lock | 194 ++++++++++++++++++++++++++ planning/rkyv-migration1/Cargo.toml | 8 ++ planning/rkyv-migration1/src/main.rs | 197 +++++++++++++++++++++++++++ 4 files changed, 401 insertions(+) create mode 100644 planning/rkyv-migration1/Cargo.lock create mode 100644 planning/rkyv-migration1/Cargo.toml create mode 100644 planning/rkyv-migration1/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 31759f369..0c8ccda13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,6 +125,8 @@ exclude = [ "api/xous-api-ticktimer", # The "packing" crate is forked and is a separate workspace "libs/mass-storage/packing", + # This directory is dedicated to out-of-tree tests and examples + "planning/", ] [profile.release] diff --git a/planning/rkyv-migration1/Cargo.lock b/planning/rkyv-migration1/Cargo.lock new file mode 100644 index 000000000..e2f9349d2 --- /dev/null +++ b/planning/rkyv-migration1/Cargo.lock @@ -0,0 +1,194 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bytes" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "indexmap" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "munge" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64142d38c84badf60abf06ff9bd80ad2174306a5b11bd4706535090a30a419df" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bb5c1d8184f13f7d0ccbeeca0def2f9a181bce2624302793005f5ca8aa62e5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptr_meta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rancor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947" +dependencies = [ + "ptr_meta", +] + +[[package]] +name = "rend" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c1f1959e4db12c985c0283656be0925f1539549db1e47c4bd0b8b599e1ef7" + +[[package]] +name = "rkyv" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c923eb995dc793cd26858a18f4bcce2df4b4a14a1ec767ea21a83b17b65f9aa" +dependencies = [ + "bytes", + "hashbrown", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv-test1" +version = "0.1.0" +dependencies = [ + "rkyv", + "xous", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fe1bd2d15adf4f037438696f8efe7ad1eae0b10d53bb0946d6a93faaf96051" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" + +[[package]] +name = "xous" +version = "0.9.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca3a121c4c2f9751469bbf19e777a9aaf15ad46a35d153a5e26d4b382e5171" +dependencies = [ + "lazy_static", +] diff --git a/planning/rkyv-migration1/Cargo.toml b/planning/rkyv-migration1/Cargo.toml new file mode 100644 index 000000000..39cd853b8 --- /dev/null +++ b/planning/rkyv-migration1/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "rkyv-test1" +version = "0.1.0" +edition = "2021" + +[dependencies] +rkyv = { version = "0.8.5", default-features = false, features = ["std", "alloc"] } # , "bytecheck" +xous = "0.9.63" diff --git a/planning/rkyv-migration1/src/main.rs b/planning/rkyv-migration1/src/main.rs new file mode 100644 index 000000000..45f7245bf --- /dev/null +++ b/planning/rkyv-migration1/src/main.rs @@ -0,0 +1,197 @@ +use core::mem::MaybeUninit; + +use rkyv::{ + api::low::LowSerializer, deserialize, rancor::{Error, Panic, Strategy}, ser::{allocator::SubAllocator, + writer::Buffer as RkyvBuffer, + Positional, + }, with::{ArchiveWith, Identity, SerializeWith}, Archive, Deserialize, Place, Portable, Serialize +}; + +use std::marker::PhantomData; + +use xous::{MemoryMessage, MemoryRange}; + +const PAGE_SIZE: usize = 4096; +const PAGE_POOL_SIZE: usize = 8; +// Create a "fake" memory pool of pages, just for testing +#[repr(C, align(4096))] +pub struct Pool { + pub bytes: [u8; PAGE_SIZE * PAGE_POOL_SIZE] +} + +// A test structure with rkyv derives +#[derive(Archive, Deserialize, Serialize, Debug, PartialEq)] +#[rkyv( + // This will generate a PartialEq impl between our unarchived + // and archived types + compare(PartialEq), + // Derives can be passed through to the generated type: + derive(Debug), +)] +struct Test { + int: u8, + string: String, + option: Option>, + string2: String, +} + + +// The new Xous-IPC Buffer format +#[derive(Debug)] +pub struct Buffer<'buf> { + pages: MemoryRange, + alloc_at: usize, + pos: usize, + slice: &'buf mut [u8], + should_drop: bool, + memory_message: Option<&'buf mut MemoryMessage>, +} +// what is a safe number for this? empirically, 20 bytes seems to be the overhead we've measured +const RKYV_OVERHEAD: usize = 32; + +// fake some allocateable memory, simply bump-allocated +static mut PAGE_POOL: Pool = Pool {bytes: [0u8; PAGE_SIZE * PAGE_POOL_SIZE]}; +static mut PAGE_PTR: usize = 0; + +pub fn map_memory(len: usize) -> MemoryRange { + assert!(len % 4096 == 0); + let address = unsafe{& PAGE_POOL.bytes[PAGE_PTR..PAGE_PTR + len]}; + + let mr = unsafe{MemoryRange::new(address.as_ptr() as usize, len).unwrap()}; + unsafe{ PAGE_PTR += len }; + mr +} + +type Serializer<'a, 'b> = + LowSerializer<'a, RkyvBuffer<'b>, SubAllocator<'a>, Panic>; + +impl<'buf> Buffer<'buf> { + pub fn new(len: usize) -> Self { + let len_to_page = (len + (PAGE_SIZE -1)) & !(PAGE_SIZE - 1); + // let alloc_start = len + RKYV_OVERHEAD; + let alloc_start = len_to_page - RKYV_OVERHEAD; + + // Allocate enough memory to hold the requested data + let new_mem = map_memory(len_to_page); + + Buffer { + pages: new_mem, + slice: unsafe { core::slice::from_raw_parts_mut(new_mem.as_mut_ptr(), len_to_page) }, + pos: 0, + alloc_at: alloc_start, + should_drop: true, + memory_message: None, + } + } + + pub fn into_buf(src: &T) -> Self + where + F: for<'a, 'b> SerializeWith>, + { + struct Wrap<'a, F, T>(&'a T, PhantomData); + + impl Archive for Wrap<'_, F, T> + where + F: ArchiveWith, + { + type Archived = >::Archived; + type Resolver = >::Resolver; + + fn resolve( + &self, + resolver: Self::Resolver, + out: Place, + ) { + F::resolve_with(self.0, resolver, out) + } + } + + impl<'a, 'b, F, T> Serialize> for Wrap<'_, F, T> + where + F: SerializeWith>, + { + fn serialize( + &self, + serializer: &mut Serializer<'a, 'b>, + ) -> Result { + F::serialize_with(self.0, serializer) + } + } + let mut xous_buf = Self::new(core::mem::size_of::()); + println!("alloc at: {}", xous_buf.alloc_at); + let (buf, scratch) = xous_buf.slice.split_at_mut(xous_buf.alloc_at); + + let wrap = Wrap(src, PhantomData::); + let writer = RkyvBuffer::from(buf); + let maybe_uninit_slice: &mut [MaybeUninit] = unsafe { + std::slice::from_raw_parts_mut(scratch.as_ptr() as *mut MaybeUninit, scratch.len()) + }; + let alloc = SubAllocator::new(maybe_uninit_slice); + + let serbuf = rkyv::api::low::to_bytes_in_with_alloc::<_, _, Panic>(&wrap, writer, alloc).unwrap(); + xous_buf.pos = serbuf.pos(); + println!("pos: {}", xous_buf.pos); + println!("scratch: {:x?}", &scratch[..32]); + xous_buf + } + + pub fn to_original(&self) -> core::result::Result + where + T: rkyv::Archive, + U: Portable, + E: std::fmt::Debug, ::Archived: Deserialize> + { + let r = unsafe{rkyv::access_unchecked::(&self.slice[..self.pos])}; + Ok(rkyv::deserialize::(r).unwrap()) + } + + pub fn as_flat(&self) -> core::result::Result<&U, ()> + where + T: rkyv::Archive, + U: Portable + { + let r = unsafe{rkyv::access_unchecked::(&self.slice[..self.pos])}; + Ok(r) + } +} + + +fn main() { + println!("Size of Test: {}", core::mem::size_of::()); + + // with an alloc + let value = Test { + int: 42, + string: "The quick brown fox jumps over the lazy dogs".to_string(), + option: Some(vec![1, 2, 3, 4]), + string2: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.".to_string(), + }; + + let buf = Buffer::into_buf::(&value); // AsBox + + let f = buf.as_flat::().unwrap(); + println!("f: {:?}", f); + + let t1 = buf.to_original::().unwrap(); + println!("t copy 1: {:?}", t1); + + println!("buf.slice: {:x?}", &buf.slice[..buf.pos+1]); + let archived = unsafe{rkyv::access_unchecked::(&buf.slice[..buf.pos])}; + println!("archived: {:?}", archived); + let t = rkyv::deserialize::(archived).unwrap(); + println!("t: {:?}", t); + assert_eq!(t.string, value.string); + + use rkyv::ser::allocator::Arena; + use rkyv::api::high::to_bytes_with_alloc; + let mut arena = Arena::new(); + + // let de = buf.to_original::().unwrap(); + let bytes = + to_bytes_with_alloc::<_, Error>(&value, arena.acquire()).unwrap(); + let archived = + unsafe { rkyv::access_unchecked::(&bytes[..]) }; + assert_eq!(archived, &value); + let deserialized = deserialize::(archived).unwrap(); + assert_eq!(deserialized.string2, value.string2); +}