diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index 75dd59d48..505ea5ab9 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -83,6 +83,7 @@ impl<'a> Iterator for BasisEncryptorIter<'a> { let block_iter = block.iter_mut(); let journal_bytes = self.basis_data.journal_rev.to_le_bytes(); + /* rkyv serializer let (ser_pos, ser_basis) = self.basis_data.root.ser(); let ser_bytes = ser_pos.to_le_bytes(); let slice_iter = @@ -91,6 +92,12 @@ impl<'a> Iterator for BasisEncryptorIter<'a> { .chain(ser_basis.as_ref().iter() // basis // .chain(self.dicts.as_slice() // dictionary )).skip(self.vaddr); + */ + let slice_iter = + journal_bytes.iter() // journal rev + .chain(self.basis_data.root.as_ref().iter() + // .chain(self.dicts.as_slice() // dictionary + ).skip(self.vaddr); // note that in the case that we've already serialized the journal, basis, and dictionary, this will produce nothing let mut written = 0; @@ -133,18 +140,58 @@ impl<'a> Iterator for BasisEncryptorIter<'a> { /// As a directory structure, the basis root is designed to be read into RAM in a contiguous block. /// it'll typically be less than a page in length, but a pathological number of dictionaries can make it /// much longer. +/// +/// We're using Repr(C) and alignment to 64-bits to create a consistent "FFI" layout; we use an unsafe cast +/// to [u8] as our method to serialize the structure, which means we could be subject to breakage if the Rust +/// compiler decides to change its Repr(C) FFI (it's not guaranteed, but I think at this point in the lifecycle +/// with simple primitive types it's hard to see it changing). This puts some requirements on the ordering of +/// fields in the struct below. Note that the serialization is all double-checked by the pddbdbg.py script. +/// +/// In coming to the choice to use Repr(C), I experimented with rkyv and bincode. bincode relies on the serde +/// crate, which, as of Nov 2021, has troubles taking in const generics, and thus barfs on our fixed-sized +/// string allocations that are longer than 32 bytes. Version 2.0 of bincode /might/ do this better, but as +/// of the design of this crate, it's in "alpha" with no official release to crates.io, so we're avoiding it; +/// but for sure 1.3.3 of bincode (latest stable as of the design) cannot do the job, and there's a few other +/// users reporting the issue so I'm pretty sure it's not "user error" on my part. +/// +/// rkyv handles const generics well, and it perhaps very reasonably shuffles around the order of structures +/// in the struct to improve the packing efficiency. However, this has the property that rkyv ser will never break +/// rkyv deser, but unfortunately you can't interoperate with anything that isn't rkyv (e.g., describing the data +/// layout to someone who wants to do a C implementation). There's also a risk that if we are forced to +/// upgrade rkyv later on we might break compatibility with what's stored on disk, although I'm pretty sure the +/// maintainer of rkyv tries to avoid that as much as possible. +/// +/// Repr(C), while also not guaranteed to be stable, has pressure from the CFFI users at least to keep +/// things as stable as possible, and it is by definition inter-operable with C. Repr(C) is native to Rust, +/// with no additional dependencies to pull in, which helps reduce the code base size overall. +/// So, we're using a repr(C) with an align(8), and then carefully checking our structure organization and +/// elements to keep things "in spec" with what C can natively understand, in an effort to create a disk +/// storage structure that can persist through future versions of Rust and also other implementations in other +/// languages. +/// +/// Known Repr(C) footguns: +/// - When you start laying in 64-bit types, stuff has to be 64-bit aligned, or else you'll start to get +/// uninitialized padding data inserted, which can leak stack data in the serialization process. +/// - Don't use anything that's not native to C. In particular, for primitives that we want to be "Option" +/// wrapped, we're using a NonZeroU64 format. The compiler knows how to turn that into a 64-bit C-friendly +/// structure and serialize/deserialize that into the correct Rust structure. See +/// https://doc.rust-lang.org/nomicon/other-reprs.html for a citation on that. #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, PartialEq, Debug)] +#[repr(C, align(8))] pub(crate) struct BasisRoot { // everything below here is encrypted using AES-GCM-SIV pub(crate) magic: [u8; 4], - pub(crate) version: u16, - pub(crate) name: BasisRootName, + pub(crate) version: u32, /// increments every time the BasisRoot is modified. This field must saturate, not roll over. pub(crate) age: u32, + /// number of dictionaries. + pub(crate) num_dictionaries: u32, + /* at this point, we are aligned to a 64-bit boundary. All data must stay aligned to this boundary from here out! */ + /// 64-byte name; aligns to 64-bits + pub(crate) name: BasisRootName, /// "open end" of the pre-allocated space for the Basis. All Basis data must exist in an extent that is /// less than this value. This can be grown and shrunk with allocation and compaction processes. pub(crate) prealloc_open_end: PageAlignedVa, - pub(crate) num_dictionaries: u32, /// virtual address pointer to the start of the dictionary record pub(crate) dict_ptr: Option, // dict_slice: [DictPointer; num_dictionaries], // DictPointers + num_dictionaries above can be turned into a dict_slice diff --git a/services/pddb/src/backend/hw.rs b/services/pddb/src/backend/hw.rs index 39a724228..d1c3360e7 100644 --- a/services/pddb/src/backend/hw.rs +++ b/services/pddb/src/backend/hw.rs @@ -822,12 +822,12 @@ impl PddbOs { } let basis_root = BasisRoot { magic: api::PDDB_MAGIC, - version: api::PDDB_VERSION, + version: 0x1111_1111, // api::PDDB_VERSION, name, - age: 0, + age: 0x2222_2222, prealloc_open_end: PageAlignedVa::from(INITIAL_BASIS_ALLOC * VPAGE_SIZE), - num_dictionaries: 0, - dict_ptr: None, + num_dictionaries: 0x3333_3333, + dict_ptr: Some(VirtAddr::new(0x4444_4444_4444_4444).unwrap())//None, }; log::info!("prealloc_open_end: {:x?}", basis_root.prealloc_open_end); log::info!("prealloc_open_end as vpage: {:x?}", basis_root.prealloc_open_end.as_vpage_num()); diff --git a/tools/pddbdbg.py b/tools/pddbdbg.py index 914940736..a2243528c 100755 --- a/tools/pddbdbg.py +++ b/tools/pddbdbg.py @@ -177,6 +177,24 @@ class Basis: i = 0 self.journal = int.from_bytes(i_bytes[i:i+4], 'little') i += 4 + self.magic = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.version = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.age = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.num_dicts = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.name = i_bytes[i:i+Basis.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore') + i += Basis.MAX_NAME_LEN + self.prealloc_open_end = int.from_bytes(i_bytes[i:i+8], 'little') + i += 8 + self.dict_ptr = int.from_bytes(i_bytes[i:i+8], 'little') + i += 8 + """ rkyv unpacker + i = 0 + self.journal = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 self.rkyvpos = int.from_bytes(i_bytes[i:i+4], 'little') i += 4 print("alloc on disk: {}".format(i_bytes[i:i+8].hex())) @@ -195,11 +213,12 @@ class Basis: i += 10 # mysterious padding in the rkyv format!! self.dict_ptr = int.from_bytes(i_bytes[i:i+8], 'little') i += 8 + """ def as_str(self): desc = '' desc += ' Journal rev: {}\n'.format(self.journal) - desc += ' Rkyv pos: {}\n'.format(self.rkyvpos) + #desc += ' Rkyv pos: {}\n'.format(self.rkyvpos) desc += ' Magic: {:x}\n'.format(self.magic) desc += ' Version: {:x}\n'.format(self.version) desc += ' Age: {:x}\n'.format(self.age)