mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-16 08:38:26 +00:00
fix the bufserializer approach
this is much more efficient than the alloc serializer
This commit is contained in:
parent
6272a94604
commit
6992b31104
4 changed files with 32 additions and 38 deletions
4
Cargo.lock
generated
4
Cargo.lock
generated
|
|
@ -3087,6 +3087,8 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "rkyv"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70de01b38fe7baba4ecdd33b777096d2b326993d8ea99bc5b6ede691883d3010"
|
||||
dependencies = [
|
||||
"memoffset",
|
||||
"ptr_meta",
|
||||
|
|
@ -3097,6 +3099,8 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "rkyv_derive"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95a169f6bc5a81033e86ed39d0f4150e2608160b73d2b93c6e8e6a3efa873f14"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
|
|||
|
|
@ -148,9 +148,6 @@ git="https://github.com/betrusted-io/ring-xous"
|
|||
rev="09207048c393d524a34f63cf65c85a973012a21d" # use the commitref because we're still updating the branch
|
||||
#branch="0.16.20-cleanup"
|
||||
|
||||
[patch.crates-io.rkyv]
|
||||
path = "../rkyv/rkyv"
|
||||
|
||||
######################################################
|
||||
### PATCHES TO DEV/TEST XOUS KERNEL CODE
|
||||
###
|
||||
|
|
|
|||
|
|
@ -898,7 +898,7 @@ impl Pddb {
|
|||
log::error!("local key index did not match remote key index: {}, {}", check_key_index, header.starting_key_index);
|
||||
}
|
||||
let mut key_count = 0;
|
||||
log::info!("header: {:?}", header);
|
||||
log::debug!("header: {:?}", header);
|
||||
while key_count < header.len {
|
||||
if index + size_of::<u32>() * 2 > msg_mem.len() {
|
||||
// quit if we don't have enough space to decode at least another two indices
|
||||
|
|
@ -908,18 +908,30 @@ impl Pddb {
|
|||
index += size_of::<u32>();
|
||||
let pos = u32::from_le_bytes(msg_mem.as_slice()[index..index + size_of::<u32>()].try_into().unwrap());
|
||||
index += size_of::<u32>();
|
||||
log::info!("unpacking message at {}({})", size, pos);
|
||||
log::debug!("unpacking message at {}({})", size, pos);
|
||||
if size != 0 && pos != 0 {
|
||||
//log::info!("extract archive: {}, {}, {}, {}", index, size, pos, msg_mem.len());
|
||||
//log::info!("{:x?}", &msg_mem.as_slice::<u8>()[index..index + (size as usize)]);
|
||||
let archived = unsafe {
|
||||
archived_value::<PddbKeyRecord>(&msg_mem.as_slice()[index..index + (size as usize)], pos as usize)
|
||||
};
|
||||
//log::info!("increment index");
|
||||
index += size as usize;
|
||||
let key = archived.deserialize(&mut AllocDeserializer).unwrap();
|
||||
//log::info!("new index: {}", index);
|
||||
let key = match archived.deserialize(&mut AllocDeserializer) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
log::error!("deserialization error: {:?}", e);
|
||||
panic!("deserializer error");
|
||||
},
|
||||
};
|
||||
//log::info!("pushing result");
|
||||
ret.push(key);
|
||||
key_count += 1;
|
||||
check_key_index += 1;
|
||||
} else {
|
||||
// we encountered a nil field, stop decoding
|
||||
log::info!("nil field");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -400,8 +400,7 @@ use locales::t;
|
|||
|
||||
use rkyv::{
|
||||
de::deserializers::AllocDeserializer,
|
||||
ser::{Serializer, serializers::WriteSerializer, serializers::BufferSerializer},
|
||||
AlignedVec,
|
||||
ser::{Serializer, serializers::BufferSerializer},
|
||||
Deserialize,
|
||||
};
|
||||
use core::mem::size_of;
|
||||
|
|
@ -1405,16 +1404,6 @@ fn wrapped_main() -> ! {
|
|||
}
|
||||
|
||||
// Optimized bulk data read handler. See lib.rs for documentation.
|
||||
// Note: it would be tempting to use a `BufferSerializer` instead of a WriteSeriaizer.
|
||||
// This would prevent the extra allocation and copy into the buffer. The problem is that
|
||||
// the BufferSerializer can return you the `pos` of the archive, but it won't tell you
|
||||
// the length of the data that's been archived. Thus, when you call `into_inner()` you get
|
||||
// back the original buffer (which is great -- we could split that and pass it on to future
|
||||
// iterations) but we don't know how much data was put in there. Maybe later versions of rkyv
|
||||
// have a call for that, but the current version we are using doesn't seem to be written to
|
||||
// allow for multiple rkyv objects to be packed into a single packet. I think the assumption
|
||||
// is rather that you're writing rkyv objects into a single file and the files are then packed
|
||||
// into disk by the filesystem.
|
||||
Opcode::DictBulkRead => {
|
||||
const BULKREAD_TIMEOUT_MS: u64 = 5000;
|
||||
let range = msg.body.memory_message_mut().unwrap();
|
||||
|
|
@ -1508,17 +1497,7 @@ fn wrapped_main() -> ! {
|
|||
&key_name,
|
||||
if state.is_basis_specified{Some(&state.basis)} else {None}
|
||||
).expect("Key went missing during bulk read"); // could be a concurrent process mutating. We don't handle this; flag with a panic.
|
||||
if attr.len < state.read_limit - state.read_total && // will it fit in the remaining read limit?
|
||||
// will it fit even in a single buffer?
|
||||
// computed as: 2x u32 for pos/len overhead
|
||||
// size of the base key record, archived
|
||||
// maximum length strings for basis name and key name
|
||||
// plus header size
|
||||
// 32 bytes "slop" to make sure we never fail to properly compute this limit
|
||||
attr.len <
|
||||
buf.len()
|
||||
- (size_of::<u32>() * 2 + size_of::<ArchivedPddbKeyRecord>() + BASIS_NAME_LEN + KEY_NAME_LEN + size_of::<BulkReadHeader>() + 32)
|
||||
{
|
||||
if attr.len < state.read_limit - state.read_total {
|
||||
let mut d = vec![0u8; attr.len];
|
||||
match basis_cache.key_read(
|
||||
&mut pddb_os,
|
||||
|
|
@ -1540,10 +1519,11 @@ fn wrapped_main() -> ! {
|
|||
data: Some(d)
|
||||
};
|
||||
state.read_total += attr.len; // commit the read length early
|
||||
let (prebuf, buf) = buf.split_at_mut(size_of::<u32>()*2);
|
||||
let mut serializer = BufferSerializer::new(buf);
|
||||
match serializer.serialize_value_len(&rec) {
|
||||
Ok((pos, len)) => SerializeResult::Success(pos, len, serializer.into_inner(), key_name, prebuf),
|
||||
let (prebuf, sbuf) = buf.split_at_mut(size_of::<u32>()*2);
|
||||
let mut serializer = BufferSerializer::new(sbuf);
|
||||
let len = size_of::<ArchivedPddbKeyRecord>();
|
||||
match serializer.serialize_value(&rec) {
|
||||
Ok(pos) => SerializeResult::Success(pos, len, serializer.into_inner(), key_name, prebuf),
|
||||
Err(_) => SerializeResult::Failure(key_name)
|
||||
}
|
||||
}
|
||||
|
|
@ -1565,8 +1545,9 @@ fn wrapped_main() -> ! {
|
|||
};
|
||||
let (prebuf, buf) = buf.split_at_mut(size_of::<u32>()*2);
|
||||
let mut serializer = BufferSerializer::new(buf);
|
||||
match serializer.serialize_value_len(&rec) {
|
||||
Ok((pos, len)) => SerializeResult::Success(pos, len, serializer.into_inner(), key_name, prebuf),
|
||||
let len = size_of::<ArchivedPddbKeyRecord>();
|
||||
match serializer.serialize_value(&rec) {
|
||||
Ok(pos) => SerializeResult::Success(pos, len, serializer.into_inner(), key_name, prebuf),
|
||||
Err(_) => SerializeResult::Failure(key_name)
|
||||
}
|
||||
}
|
||||
|
|
@ -1582,19 +1563,19 @@ fn wrapped_main() -> ! {
|
|||
// method but then we'd need to...make a serializer for the structure. which is agains the whole point of it...
|
||||
match ser_result {
|
||||
SerializeResult::Success(pos, len, sbuf, _key_name, pre_buf) => {
|
||||
log::info!("packing message of {}({})", len, pos);
|
||||
log::debug!("packing message of {}({})", len + pos, pos);
|
||||
// data can fit, copy it into the buffer.
|
||||
state.buf_starting_key_index += 1;
|
||||
header.len += 1;
|
||||
// read length increment was handled when the data was copied into the serialization buffer.
|
||||
pre_buf[..4].copy_from_slice(
|
||||
//&(sbuf.len() as u32).to_le_bytes()
|
||||
&(len as u32).to_le_bytes()
|
||||
&((len + pos) as u32).to_le_bytes()
|
||||
);
|
||||
pre_buf[4..8].copy_from_slice(
|
||||
&(pos as u32).to_le_bytes()
|
||||
);
|
||||
(_, buf) = sbuf.split_at_mut(pos);
|
||||
(_, buf) = sbuf.split_at_mut(len + pos);
|
||||
}
|
||||
SerializeResult::Failure(key_name) => {
|
||||
log::info!("ran out of space filling buffer, pushing {} back into the queue", key_name);
|
||||
|
|
|
|||
Loading…
Reference in a new issue