checkpoint with memory placement and disk storage

this probably isn't new to most other people but this has been
a very visceral exercise of how tricky it is to use Repr(c)
for data layouts in anything but the most trivial data structures.

Unfortunately the Basis root data structure needs to be fairly rich,
so we're playing around with using rkyv to help with the serialization
to disk but I'm also not convinced this is a good idea yet. This
checkpoints commits a "working" rkyv implementation, but I'm not
entirely happy with the notion that rkyv can shuffle around data
bits based on its particular version or decisions in the serializer
crate. I'd like deterministic, repeatable serialization that will
be solid through lots of revs...
This commit is contained in:
bunnie 2021-11-15 16:00:01 +08:00
parent ba9b136f87
commit ce51205155
6 changed files with 39 additions and 18 deletions

8
Cargo.lock generated
View file

@ -1900,18 +1900,18 @@ checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012"
[[package]]
name = "serde"
version = "1.0.129"
version = "1.0.130"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1f72836d2aa753853178eda473a3b9d8e4eefdaf20523b919677e6de489f8f1"
checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.129"
version = "1.0.130"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e57ae87ad533d9a56427558b516d0adac283614e347abf85b0dc0cbbf0a249f3"
checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b"
dependencies = [
"proc-macro2",
"quote",

View file

@ -1,5 +1,3 @@
use rkyv::{Archive, Deserialize, Serialize};
pub(crate) const SERVER_NAME_PDDB: &str = "_Plausibly Deniable Database_";
#[allow(dead_code)]
pub(crate) const PDDB_MAX_BASIS_NAME_LEN: usize = 64;
@ -51,7 +49,7 @@ pub(crate) enum Opcode {
}
/// A structure for requesting a token to access a particular key/value pair
#[derive(Archive, Serialize, Deserialize)]
#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
pub(crate) struct PddbKeyRequest {
pub(crate) dict: xous_ipc::String::</*PDDB_MAX_DICT_NAME_LEN*/ 64>, // pending https://github.com/rust-lang/rust/issues/90195
pub(crate) key: xous_ipc::String::</*PDDB_MAX_KEY_NAME_LEN*/ 128>, // pending https://github.com/rust-lang/rust/issues/90195

View file

@ -10,6 +10,8 @@ use core::mem::size_of;
use std::convert::TryInto;
use aes_gcm_siv::{Aes256GcmSiv, Nonce};
use aes_gcm_siv::aead::{Aead, Payload};
use rkyv::Aligned;
use rkyv::ser::serializers::BufferSerializer;
use std::iter::IntoIterator;
@ -81,11 +83,14 @@ impl<'a> Iterator for BasisEncryptorIter<'a> {
let block_iter = block.iter_mut();
let journal_bytes = self.basis_data.journal_rev.to_le_bytes();
let (ser_pos, ser_basis) = self.basis_data.root.ser();
let ser_bytes = ser_pos.to_le_bytes();
let slice_iter =
journal_bytes.iter() // journal rev
.chain(self.basis_data.root.deref().iter() // basis
.chain(ser_bytes.iter()
.chain(ser_basis.as_ref().iter() // basis
// .chain(self.dicts.as_slice() // dictionary
).skip(self.vaddr);
)).skip(self.vaddr);
// note that in the case that we've already serialized the journal, basis, and dictionary, this will produce nothing
let mut written = 0;
@ -128,7 +133,7 @@ impl<'a> Iterator for BasisEncryptorIter<'a> {
/// As a directory structure, the basis root is designed to be read into RAM in a contiguous block.
/// it'll typically be less than a page in length, but a pathological number of dictionaries can make it
/// much longer.
#[repr(C)]
#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, PartialEq, Debug)]
pub(crate) struct BasisRoot {
// everything below here is encrypted using AES-GCM-SIV
pub(crate) magic: [u8; 4],
@ -174,6 +179,17 @@ impl BasisRoot {
)
}
}*/
use rkyv::ser::Serializer;
impl BasisRoot { // note: example wraps u8 in "Aligned"...
pub(crate) fn ser(&self) -> (u32, Aligned<[u8; size_of::<rkyv::ser::serializers::BufferSerializer<BasisRoot>>()]>) {
let mut ser = BufferSerializer::new(Aligned([0u8; size_of::<rkyv::ser::serializers::BufferSerializer<BasisRoot>>()]));
let pos = match ser.serialize_value(self) {
Ok(p) => p,
Err(e) => {log::error!("serializer error: {:?}", e); panic!("serializer error");}
};
(pos as u32, ser.into_inner())
}
}
impl Deref for BasisRoot {
type Target = [u8];
fn deref(&self) -> &[u8] {

View file

@ -37,7 +37,7 @@ impl Default for PtFlags {
/// however, the sheer bulk of the page table demands a compact representation. Thus,
/// any routines downstream of the Pte shall be coded to handle potentially a much larger
/// nonce and checksum structure.
#[repr(C, packed)]
#[repr(packed)]
#[derive(Default)]
pub(crate) struct Pte {
/// the virtual address is 48 bits long

View file

@ -69,7 +69,7 @@ pub type VirtAddr = NonZeroU64;
/// the constructing address. Thus it will tend to "round up" to the nearest page,
/// unless the given address happens to be exactly one page in size.
#[allow(dead_code)]
#[derive(Copy, Clone, Debug)]
#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, PartialEq, Eq, Copy, Clone, Debug)]
pub(crate) struct PageAlignedVa(VirtAddr);
impl PageAlignedVa {
pub(crate) fn as_u32(&self) -> u32 {

View file

@ -177,25 +177,32 @@ class Basis:
i = 0
self.journal = int.from_bytes(i_bytes[i:i+4], 'little')
i += 4
self.magic = int.from_bytes(i_bytes[i:i+4], 'little')
self.rkyvpos = int.from_bytes(i_bytes[i:i+4], 'little')
i += 4
self.version = int.from_bytes(i_bytes[i:i+2], 'little')
i += 2
self.name = i_bytes[i:i+Basis.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
i += Basis.MAX_NAME_LEN
print("alloc on disk: {}".format(i_bytes[i:i+8].hex()))
self.prealloc_open_end = int.from_bytes(i_bytes[i:i+8], 'little')
i += 8
self.age = int.from_bytes(i_bytes[i:i+4], 'little')
i += 4
self.num_dicts = int.from_bytes(i_bytes[i:i+4], 'little')
i += 4
self.version = int.from_bytes(i_bytes[i:i+2], 'little')
i += 2
self.magic = int.from_bytes(i_bytes[i:i+4], 'little')
i += 4
self.name = i_bytes[i:i+Basis.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
i += Basis.MAX_NAME_LEN
i += 10 # mysterious padding in the rkyv format!!
self.dict_ptr = int.from_bytes(i_bytes[i:i+8], 'little')
i += 8
def as_str(self):
desc = ''
desc += ' Journal rev: {}\n'.format(self.journal)
desc += ' Rkyv pos: {}\n'.format(self.rkyvpos)
desc += ' Magic: {:x}\n'.format(self.magic)
desc += ' Version: {}\n'.format(self.version)
desc += ' Version: {:x}\n'.format(self.version)
desc += ' Age: {:x}\n'.format(self.age)
desc += ' Name: {}\n'.format(self.name)
desc += ' Alloc: {:x}\n'.format(self.prealloc_open_end)
desc += ' NumDicts: {:x}\n'.format(self.num_dicts)