create a planning directory, and add the rkyv test

The purpose of the planning directory is a place where we can stash
code for reference in various github issues and project plans without
having to integrate it into the code base. This is for large changes
to the code base that require extensive planning and discussion prior
to integration.
This commit is contained in:
bunnie 2024-09-17 18:29:24 +08:00
parent 19de3e0402
commit abb245543c
4 changed files with 401 additions and 0 deletions

View file

@ -125,6 +125,8 @@ exclude = [
"api/xous-api-ticktimer",
# The "packing" crate is forked and is a separate workspace
"libs/mass-storage/packing",
# This directory is dedicated to out-of-tree tests and examples
"planning/",
]
[profile.release]

194
planning/rkyv-migration1/Cargo.lock generated Normal file
View file

@ -0,0 +1,194 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "bytes"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "indexmap"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
dependencies = [
"equivalent",
"hashbrown",
]
[[package]]
name = "lazy_static"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "munge"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64142d38c84badf60abf06ff9bd80ad2174306a5b11bd4706535090a30a419df"
dependencies = [
"munge_macro",
]
[[package]]
name = "munge_macro"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bb5c1d8184f13f7d0ccbeeca0def2f9a181bce2624302793005f5ca8aa62e5e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
dependencies = [
"unicode-ident",
]
[[package]]
name = "ptr_meta"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90"
dependencies = [
"ptr_meta_derive",
]
[[package]]
name = "ptr_meta_derive"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rancor"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947"
dependencies = [
"ptr_meta",
]
[[package]]
name = "rend"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31c1f1959e4db12c985c0283656be0925f1539549db1e47c4bd0b8b599e1ef7"
[[package]]
name = "rkyv"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c923eb995dc793cd26858a18f4bcce2df4b4a14a1ec767ea21a83b17b65f9aa"
dependencies = [
"bytes",
"hashbrown",
"indexmap",
"munge",
"ptr_meta",
"rancor",
"rend",
"rkyv_derive",
"tinyvec",
"uuid",
]
[[package]]
name = "rkyv-test1"
version = "0.1.0"
dependencies = [
"rkyv",
"xous",
]
[[package]]
name = "rkyv_derive"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67fe1bd2d15adf4f037438696f8efe7ad1eae0b10d53bb0946d6a93faaf96051"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "syn"
version = "2.0.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tinyvec"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "unicode-ident"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
[[package]]
name = "uuid"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
[[package]]
name = "xous"
version = "0.9.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ca3a121c4c2f9751469bbf19e777a9aaf15ad46a35d153a5e26d4b382e5171"
dependencies = [
"lazy_static",
]

View file

@ -0,0 +1,8 @@
[package]
name = "rkyv-test1"
version = "0.1.0"
edition = "2021"
[dependencies]
rkyv = { version = "0.8.5", default-features = false, features = ["std", "alloc"] } # , "bytecheck"
xous = "0.9.63"

View file

@ -0,0 +1,197 @@
use core::mem::MaybeUninit;
use rkyv::{
api::low::LowSerializer, deserialize, rancor::{Error, Panic, Strategy}, ser::{allocator::SubAllocator,
writer::Buffer as RkyvBuffer,
Positional,
}, with::{ArchiveWith, Identity, SerializeWith}, Archive, Deserialize, Place, Portable, Serialize
};
use std::marker::PhantomData;
use xous::{MemoryMessage, MemoryRange};
const PAGE_SIZE: usize = 4096;
const PAGE_POOL_SIZE: usize = 8;
// Create a "fake" memory pool of pages, just for testing
#[repr(C, align(4096))]
pub struct Pool {
pub bytes: [u8; PAGE_SIZE * PAGE_POOL_SIZE]
}
// A test structure with rkyv derives
#[derive(Archive, Deserialize, Serialize, Debug, PartialEq)]
#[rkyv(
// This will generate a PartialEq impl between our unarchived
// and archived types
compare(PartialEq),
// Derives can be passed through to the generated type:
derive(Debug),
)]
struct Test {
int: u8,
string: String,
option: Option<Vec<i32>>,
string2: String,
}
// The new Xous-IPC Buffer format
#[derive(Debug)]
pub struct Buffer<'buf> {
pages: MemoryRange,
alloc_at: usize,
pos: usize,
slice: &'buf mut [u8],
should_drop: bool,
memory_message: Option<&'buf mut MemoryMessage>,
}
// what is a safe number for this? empirically, 20 bytes seems to be the overhead we've measured
const RKYV_OVERHEAD: usize = 32;
// fake some allocateable memory, simply bump-allocated
static mut PAGE_POOL: Pool = Pool {bytes: [0u8; PAGE_SIZE * PAGE_POOL_SIZE]};
static mut PAGE_PTR: usize = 0;
pub fn map_memory(len: usize) -> MemoryRange {
assert!(len % 4096 == 0);
let address = unsafe{& PAGE_POOL.bytes[PAGE_PTR..PAGE_PTR + len]};
let mr = unsafe{MemoryRange::new(address.as_ptr() as usize, len).unwrap()};
unsafe{ PAGE_PTR += len };
mr
}
type Serializer<'a, 'b> =
LowSerializer<'a, RkyvBuffer<'b>, SubAllocator<'a>, Panic>;
impl<'buf> Buffer<'buf> {
pub fn new(len: usize) -> Self {
let len_to_page = (len + (PAGE_SIZE -1)) & !(PAGE_SIZE - 1);
// let alloc_start = len + RKYV_OVERHEAD;
let alloc_start = len_to_page - RKYV_OVERHEAD;
// Allocate enough memory to hold the requested data
let new_mem = map_memory(len_to_page);
Buffer {
pages: new_mem,
slice: unsafe { core::slice::from_raw_parts_mut(new_mem.as_mut_ptr(), len_to_page) },
pos: 0,
alloc_at: alloc_start,
should_drop: true,
memory_message: None,
}
}
pub fn into_buf<F, T>(src: &T) -> Self
where
F: for<'a, 'b> SerializeWith<T, Serializer<'a, 'b>>,
{
struct Wrap<'a, F, T>(&'a T, PhantomData<F>);
impl<F, T> Archive for Wrap<'_, F, T>
where
F: ArchiveWith<T>,
{
type Archived = <F as ArchiveWith<T>>::Archived;
type Resolver = <F as ArchiveWith<T>>::Resolver;
fn resolve(
&self,
resolver: Self::Resolver,
out: Place<Self::Archived>,
) {
F::resolve_with(self.0, resolver, out)
}
}
impl<'a, 'b, F, T> Serialize<Serializer<'a, 'b>> for Wrap<'_, F, T>
where
F: SerializeWith<T, Serializer<'a, 'b>>,
{
fn serialize(
&self,
serializer: &mut Serializer<'a, 'b>,
) -> Result<Self::Resolver, Panic> {
F::serialize_with(self.0, serializer)
}
}
let mut xous_buf = Self::new(core::mem::size_of::<T>());
println!("alloc at: {}", xous_buf.alloc_at);
let (buf, scratch) = xous_buf.slice.split_at_mut(xous_buf.alloc_at);
let wrap = Wrap(src, PhantomData::<F>);
let writer = RkyvBuffer::from(buf);
let maybe_uninit_slice: &mut [MaybeUninit<u8>] = unsafe {
std::slice::from_raw_parts_mut(scratch.as_ptr() as *mut MaybeUninit<u8>, scratch.len())
};
let alloc = SubAllocator::new(maybe_uninit_slice);
let serbuf = rkyv::api::low::to_bytes_in_with_alloc::<_, _, Panic>(&wrap, writer, alloc).unwrap();
xous_buf.pos = serbuf.pos();
println!("pos: {}", xous_buf.pos);
println!("scratch: {:x?}", &scratch[..32]);
xous_buf
}
pub fn to_original<T, U, E>(&self) -> core::result::Result<T, E>
where
T: rkyv::Archive<Archived = U>,
U: Portable,
E: std::fmt::Debug, <T as Archive>::Archived: Deserialize<T, Strategy<rkyv::de::Pool, E>>
{
let r = unsafe{rkyv::access_unchecked::<U>(&self.slice[..self.pos])};
Ok(rkyv::deserialize::<T, E>(r).unwrap())
}
pub fn as_flat<T, U>(&self) -> core::result::Result<&U, ()>
where
T: rkyv::Archive<Archived = U>,
U: Portable
{
let r = unsafe{rkyv::access_unchecked::<U>(&self.slice[..self.pos])};
Ok(r)
}
}
fn main() {
println!("Size of Test: {}", core::mem::size_of::<Test>());
// with an alloc
let value = Test {
int: 42,
string: "The quick brown fox jumps over the lazy dogs".to_string(),
option: Some(vec![1, 2, 3, 4]),
string2: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.".to_string(),
};
let buf = Buffer::into_buf::<Identity, Test>(&value); // AsBox
let f = buf.as_flat::<Test, _>().unwrap();
println!("f: {:?}", f);
let t1 = buf.to_original::<Test, _, Error>().unwrap();
println!("t copy 1: {:?}", t1);
println!("buf.slice: {:x?}", &buf.slice[..buf.pos+1]);
let archived = unsafe{rkyv::access_unchecked::<ArchivedTest>(&buf.slice[..buf.pos])};
println!("archived: {:?}", archived);
let t = rkyv::deserialize::<Test, Error>(archived).unwrap();
println!("t: {:?}", t);
assert_eq!(t.string, value.string);
use rkyv::ser::allocator::Arena;
use rkyv::api::high::to_bytes_with_alloc;
let mut arena = Arena::new();
// let de = buf.to_original::<Test, rkyv::rancor::Error>().unwrap();
let bytes =
to_bytes_with_alloc::<_, Error>(&value, arena.acquire()).unwrap();
let archived =
unsafe { rkyv::access_unchecked::<ArchivedTest>(&bytes[..]) };
assert_eq!(archived, &value);
let deserialized = deserialize::<Test, Error>(archived).unwrap();
assert_eq!(deserialized.string2, value.string2);
}