mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-15 08:30:02 +00:00
checkpoint with memory placement and disk storage
this probably isn't new to most other people but this has been a very visceral exercise of how tricky it is to use Repr(c) for data layouts in anything but the most trivial data structures. Unfortunately the Basis root data structure needs to be fairly rich, so we're playing around with using rkyv to help with the serialization to disk but I'm also not convinced this is a good idea yet. This checkpoints commits a "working" rkyv implementation, but I'm not entirely happy with the notion that rkyv can shuffle around data bits based on its particular version or decisions in the serializer crate. I'd like deterministic, repeatable serialization that will be solid through lots of revs...
This commit is contained in:
parent
ba9b136f87
commit
ce51205155
6 changed files with 39 additions and 18 deletions
8
Cargo.lock
generated
8
Cargo.lock
generated
|
|
@ -1900,18 +1900,18 @@ checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012"
|
|||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.129"
|
||||
version = "1.0.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1f72836d2aa753853178eda473a3b9d8e4eefdaf20523b919677e6de489f8f1"
|
||||
checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.129"
|
||||
version = "1.0.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e57ae87ad533d9a56427558b516d0adac283614e347abf85b0dc0cbbf0a249f3"
|
||||
checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
use rkyv::{Archive, Deserialize, Serialize};
|
||||
|
||||
pub(crate) const SERVER_NAME_PDDB: &str = "_Plausibly Deniable Database_";
|
||||
#[allow(dead_code)]
|
||||
pub(crate) const PDDB_MAX_BASIS_NAME_LEN: usize = 64;
|
||||
|
|
@ -51,7 +49,7 @@ pub(crate) enum Opcode {
|
|||
}
|
||||
|
||||
/// A structure for requesting a token to access a particular key/value pair
|
||||
#[derive(Archive, Serialize, Deserialize)]
|
||||
#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
|
||||
pub(crate) struct PddbKeyRequest {
|
||||
pub(crate) dict: xous_ipc::String::</*PDDB_MAX_DICT_NAME_LEN*/ 64>, // pending https://github.com/rust-lang/rust/issues/90195
|
||||
pub(crate) key: xous_ipc::String::</*PDDB_MAX_KEY_NAME_LEN*/ 128>, // pending https://github.com/rust-lang/rust/issues/90195
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ use core::mem::size_of;
|
|||
use std::convert::TryInto;
|
||||
use aes_gcm_siv::{Aes256GcmSiv, Nonce};
|
||||
use aes_gcm_siv::aead::{Aead, Payload};
|
||||
use rkyv::Aligned;
|
||||
use rkyv::ser::serializers::BufferSerializer;
|
||||
use std::iter::IntoIterator;
|
||||
|
||||
|
||||
|
|
@ -81,11 +83,14 @@ impl<'a> Iterator for BasisEncryptorIter<'a> {
|
|||
let block_iter = block.iter_mut();
|
||||
|
||||
let journal_bytes = self.basis_data.journal_rev.to_le_bytes();
|
||||
let (ser_pos, ser_basis) = self.basis_data.root.ser();
|
||||
let ser_bytes = ser_pos.to_le_bytes();
|
||||
let slice_iter =
|
||||
journal_bytes.iter() // journal rev
|
||||
.chain(self.basis_data.root.deref().iter() // basis
|
||||
.chain(ser_bytes.iter()
|
||||
.chain(ser_basis.as_ref().iter() // basis
|
||||
// .chain(self.dicts.as_slice() // dictionary
|
||||
).skip(self.vaddr);
|
||||
)).skip(self.vaddr);
|
||||
|
||||
// note that in the case that we've already serialized the journal, basis, and dictionary, this will produce nothing
|
||||
let mut written = 0;
|
||||
|
|
@ -128,7 +133,7 @@ impl<'a> Iterator for BasisEncryptorIter<'a> {
|
|||
/// As a directory structure, the basis root is designed to be read into RAM in a contiguous block.
|
||||
/// it'll typically be less than a page in length, but a pathological number of dictionaries can make it
|
||||
/// much longer.
|
||||
#[repr(C)]
|
||||
#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, PartialEq, Debug)]
|
||||
pub(crate) struct BasisRoot {
|
||||
// everything below here is encrypted using AES-GCM-SIV
|
||||
pub(crate) magic: [u8; 4],
|
||||
|
|
@ -174,6 +179,17 @@ impl BasisRoot {
|
|||
)
|
||||
}
|
||||
}*/
|
||||
use rkyv::ser::Serializer;
|
||||
impl BasisRoot { // note: example wraps u8 in "Aligned"...
|
||||
pub(crate) fn ser(&self) -> (u32, Aligned<[u8; size_of::<rkyv::ser::serializers::BufferSerializer<BasisRoot>>()]>) {
|
||||
let mut ser = BufferSerializer::new(Aligned([0u8; size_of::<rkyv::ser::serializers::BufferSerializer<BasisRoot>>()]));
|
||||
let pos = match ser.serialize_value(self) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {log::error!("serializer error: {:?}", e); panic!("serializer error");}
|
||||
};
|
||||
(pos as u32, ser.into_inner())
|
||||
}
|
||||
}
|
||||
impl Deref for BasisRoot {
|
||||
type Target = [u8];
|
||||
fn deref(&self) -> &[u8] {
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ impl Default for PtFlags {
|
|||
/// however, the sheer bulk of the page table demands a compact representation. Thus,
|
||||
/// any routines downstream of the Pte shall be coded to handle potentially a much larger
|
||||
/// nonce and checksum structure.
|
||||
#[repr(C, packed)]
|
||||
#[repr(packed)]
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Pte {
|
||||
/// the virtual address is 48 bits long
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ pub type VirtAddr = NonZeroU64;
|
|||
/// the constructing address. Thus it will tend to "round up" to the nearest page,
|
||||
/// unless the given address happens to be exactly one page in size.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, PartialEq, Eq, Copy, Clone, Debug)]
|
||||
pub(crate) struct PageAlignedVa(VirtAddr);
|
||||
impl PageAlignedVa {
|
||||
pub(crate) fn as_u32(&self) -> u32 {
|
||||
|
|
|
|||
|
|
@ -177,25 +177,32 @@ class Basis:
|
|||
i = 0
|
||||
self.journal = int.from_bytes(i_bytes[i:i+4], 'little')
|
||||
i += 4
|
||||
self.magic = int.from_bytes(i_bytes[i:i+4], 'little')
|
||||
self.rkyvpos = int.from_bytes(i_bytes[i:i+4], 'little')
|
||||
i += 4
|
||||
self.version = int.from_bytes(i_bytes[i:i+2], 'little')
|
||||
i += 2
|
||||
self.name = i_bytes[i:i+Basis.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
|
||||
i += Basis.MAX_NAME_LEN
|
||||
print("alloc on disk: {}".format(i_bytes[i:i+8].hex()))
|
||||
self.prealloc_open_end = int.from_bytes(i_bytes[i:i+8], 'little')
|
||||
i += 8
|
||||
self.age = int.from_bytes(i_bytes[i:i+4], 'little')
|
||||
i += 4
|
||||
self.num_dicts = int.from_bytes(i_bytes[i:i+4], 'little')
|
||||
i += 4
|
||||
self.version = int.from_bytes(i_bytes[i:i+2], 'little')
|
||||
i += 2
|
||||
self.magic = int.from_bytes(i_bytes[i:i+4], 'little')
|
||||
i += 4
|
||||
self.name = i_bytes[i:i+Basis.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
|
||||
i += Basis.MAX_NAME_LEN
|
||||
i += 10 # mysterious padding in the rkyv format!!
|
||||
self.dict_ptr = int.from_bytes(i_bytes[i:i+8], 'little')
|
||||
i += 8
|
||||
|
||||
def as_str(self):
|
||||
desc = ''
|
||||
desc += ' Journal rev: {}\n'.format(self.journal)
|
||||
desc += ' Rkyv pos: {}\n'.format(self.rkyvpos)
|
||||
desc += ' Magic: {:x}\n'.format(self.magic)
|
||||
desc += ' Version: {}\n'.format(self.version)
|
||||
desc += ' Version: {:x}\n'.format(self.version)
|
||||
desc += ' Age: {:x}\n'.format(self.age)
|
||||
desc += ' Name: {}\n'.format(self.name)
|
||||
desc += ' Alloc: {:x}\n'.format(self.prealloc_open_end)
|
||||
desc += ' NumDicts: {:x}\n'.format(self.num_dicts)
|
||||
|
|
|
|||
Loading…
Reference in a new issue