From 14f4beef98ad5fb645482eeaa04be01b9ab21264 Mon Sep 17 00:00:00 2001 From: bunnie Date: Thu, 18 Nov 2021 15:38:12 +0800 Subject: [PATCH] refactor data structures into smaller files for easier nav --- services/pddb/src/backend.rs | 4 + services/pddb/src/backend/basis.rs | 192 +----------------------- services/pddb/src/backend/dictionary.rs | 115 ++++++++++++++ services/pddb/src/backend/hw.rs | 10 ++ services/pddb/src/backend/key.rs | 91 +++++++++++ 5 files changed, 221 insertions(+), 191 deletions(-) create mode 100644 services/pddb/src/backend/dictionary.rs create mode 100644 services/pddb/src/backend/key.rs diff --git a/services/pddb/src/backend.rs b/services/pddb/src/backend.rs index b5bdbb410..12b851e9b 100644 --- a/services/pddb/src/backend.rs +++ b/services/pddb/src/backend.rs @@ -1,5 +1,9 @@ mod basis; pub use basis::*; +mod dictionary; +pub use dictionary::*; +mod key; +pub use key::*; mod pagetable; pub use pagetable::*; mod fastspace; diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index 2ac2148c1..a767d5977 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -87,7 +87,7 @@ use std::io::{Result, Error, ErrorKind}; /// Memory Pools /// /// Key data is split into three categories of sizes: small, medium, and large. The thresholds -/// are subject to tuning, but roughly speaking, small data are keys < 512 bytes; medium are ~1k; +/// are subject to tuning, but roughly speaking, small data are keys <2k bytes; medium are ~4k; /// and large are bigger than 32k. /// /// Large keys are the simplest - each key starts at a VPAGE-aligned address, and allocates @@ -615,196 +615,6 @@ impl BasisCacheEntry { } } -/// RAM based copy of the dictionary structures on disk. -pub(crate) struct DictCacheEntry { - /// Use this to compute the virtual address of the dictionary's location - /// multiply this by DICT_VSIZE to get at the virtual address. This /could/ be a - /// NonZeroU32 type as it should never be 0. Maybe that's a thing to fix later on. - pub(crate) index: u32, - /// A cache of the keys within the dictionary. If the key does not exist in - /// the cache, one should consult the on-disk copy, assuming the record is clean. - pub(crate) keys: HashMap::, - /// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys` - pub(crate) key_count: u32, - /// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified. - pub(crate) clean: bool, - /// track modification count - pub(crate) age: u32, - /// copy of the flags entry on the Dict on-disk - pub(crate) flags: u32, -} -impl DictCacheEntry { - /// Update a key entry. If the key does not already exist, it will create a new one. - /// - /// `key_update` will write `data` starting at `offset`, and will grow the record if data - /// is larger than the current allocation. If `truncate` is false, the existing data past the end of - /// the `data` written is preserved; if `truncate` is true, the excess data past the end of the written - /// data is removed. - /// - /// For small records, a `key_update` call would just want to replace the entire record, so it would have - /// an `offset` of 0, `truncate` is true, and the data would be the new data. However, the `offset` and - /// `truncate` records are particularly useful for updating very large file streams, which can't be - /// held entirely in RAM. - /// - /// Note: it is up to the higher level Basis disambiguation logic to decide the cross-basis update policy: it - /// could either be to update only the dictionary in the latest open basis, update all dictionaries, or update a - /// specific dictionary in a named basis. In all of these cases, the Basis resolver will have had to find the - /// correct DictCacheEntry and issue the `key_update` to it; for multiple updates, then multiple calls to - /// multiple DictCacheEntry are required. - pub fn key_update(&mut self, name: &str, data: &[u8], offset: usize, truncate: bool) -> Result <()> { - - } - - /// If `paranoid` is true, this function calls `key_update` with 0's for the data. In either case, it - /// deletes the key record from the dictionary. - pub fn key_erase(&mut self, name: &str, paranoid: bool) { - - } -} -#[derive(Debug)] -/// On-disk representation of the dictionary header. -#[repr(C, align(8))] -pub(crate) struct Dictionary { - /// Reserved for flags on the record entry - pub(crate) flags: u32, - /// Access count to the dicitionary - pub(crate) age: u32, - /// Number of keys in the dictionary - pub(crate) num_keys: u32, - /// Name. Length should pad out the record to exactly 127 bytes. - pub(crate) name: [u8; DICT_NAME_LEN], -} -impl Deref for Dictionary { - type Target = [u8]; - fn deref(&self) -> &[u8] { - unsafe { - core::slice::from_raw_parts(self as *const Dictionary as *const u8, core::mem::size_of::()) - as &[u8] - } - } -} -impl DerefMut for Dictionary { - fn deref_mut(&mut self) -> &mut [u8] { - unsafe { - core::slice::from_raw_parts_mut(self as *mut Dictionary as *mut u8, core::mem::size_of::()) - as &mut [u8] - } - } -} - -/// This structure "enforces" the 127-byte stride of dict/key vpage entries -#[derive(Copy, Clone)] -pub(crate) struct DictKeyEntry { - pub(crate) data: [u8; DK_STRIDE], -} -impl Default for DictKeyEntry { - fn default() -> DictKeyEntry { - DictKeyEntry { - data: [0; DK_STRIDE] - } - } -} - -/// This structure helps to bookkeep which slices within a DictKey virtual page need to be updated -pub(crate) struct DictKeyVpage { - pub(crate) elements: [Option; VPAGE_SIZE / DK_STRIDE], -} -impl<'a> Default for DictKeyVpage { - fn default() -> DictKeyVpage { - DictKeyVpage { - elements: [None; VPAGE_SIZE / DK_STRIDE], - } - } -} - -/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so -/// any deserialization must essentially come with a copy step to line up the record. -#[repr(C, align(8))] -pub(crate) struct KeyDescriptor { - /// virtual address of the key's start - start: u64, - /// length of the key's stored data - len: u64, - /// amount of space reserved for the key. Must be >= len. - reserved: u64, - /// Reserved for flags on the record entry - flags: u32, - /// Access count to the key - age: u32, - /// Name. Length should pad out the record to exactly 127 bytes. - name: [u8; KEY_NAME_LEN], -} -impl Deref for KeyDescriptor { - type Target = [u8]; - fn deref(&self) -> &[u8] { - unsafe { - core::slice::from_raw_parts(self as *const KeyDescriptor as *const u8, core::mem::size_of::()) - as &[u8] - } - } -} -impl DerefMut for KeyDescriptor { - fn deref_mut(&mut self) -> &mut [u8] { - unsafe { - core::slice::from_raw_parts_mut(self as *mut KeyDescriptor as *mut u8, core::mem::size_of::()) - as &mut [u8] - } - } -} - -pub(crate) struct KeyCacheEntry { - pub(crate) start: u64, - pub(crate) len: u64, - pub(crate) reserved: u64, - pub(crate) flags: u32, - pub(crate) age: u32, - /// the current on-disk index of the KeyCacheEntry item, enumerated as "0" being the first DictKeyEntry - /// slot past the first record which is the descriptor of the dictionary. - pub(crate) descriptor_index: u32, - /// indicates if it's clean - pub(crate) clean: bool, -} -impl KeyCacheEntry { - /// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself. - pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr { - VirtAddr::new(dict_offset.get() + ((self.descriptor_index as u64 + 1) * DK_STRIDE as u64)).unwrap() - } - /// Computes the modular position of the KeyDescriptor within a vpage. - pub(crate) fn descriptor_modulus(&self) -> usize { - (self.descriptor_index as usize + 1) % (VPAGE_SIZE / DK_STRIDE) - } - /// Computes the vpage offset as measured from the start of the dictionary storage region - pub(crate) fn descriptor_vpage_num(&self) -> usize { - (self.descriptor_index as usize + 1) / (VPAGE_SIZE / DK_STRIDE) - } -} - -/// used to identify cached/chunked data from a key -/// maybe we should detach the cache_page because this is meant to be a key, and cache_page is the payload. -pub(crate) struct KeyCacheEntryEntry { - basis: String, - dict: String, - key: KeyDescriptor, - /// offset of the cached data relative to the KeyDescriptor's start - offset: u64, - /// the actual cached data - cache_page: [u8; VPAGE_SIZE], - /// true if clean and synced with the disk - clean: bool, - /// general flags, tbd - flags: u8, -} - -/// this is the structure of the Basis Key in RAM. The "key" and "iv" are actually never committed to -/// flash; only the "salt" is written to disk. The final "salt" is computed as the XOR of the salt on disk -/// and the user-provided "basis name". We never record the "basis name" on disk, so that the existence of -/// any Basis can be denied. -pub(crate) struct BasisKey { - salt: [u8; 16], - key: [u8; 32], // derived from lower 256 bits of sha512(bcrypt(salt, pw)) - iv: [u8; 16], // an IV derived from the upper 128 bits of the sha512 hash from above, XOR with the salt -} - // **** // Beginning of serializers for the data structures in this file. // **** diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs new file mode 100644 index 000000000..2cb2b59d8 --- /dev/null +++ b/services/pddb/src/backend/dictionary.rs @@ -0,0 +1,115 @@ +use crate::api::*; +use super::*; + +use core::cell::RefCell; +use std::rc::Rc; +use core::ops::{Deref, DerefMut}; +use core::mem::size_of; +use std::convert::TryInto; +use aes_gcm_siv::{Aes256GcmSiv, Nonce}; +use aes_gcm_siv::aead::{Aead, Payload}; +use std::iter::IntoIterator; +use std::collections::HashMap; +use std::io::{Result, Error, ErrorKind}; + +/// RAM based copy of the dictionary structures on disk. +pub(crate) struct DictCacheEntry { + /// Use this to compute the virtual address of the dictionary's location + /// multiply this by DICT_VSIZE to get at the virtual address. This /could/ be a + /// NonZeroU32 type as it should never be 0. Maybe that's a thing to fix later on. + pub(crate) index: u32, + /// A cache of the keys within the dictionary. If the key does not exist in + /// the cache, one should consult the on-disk copy, assuming the record is clean. + pub(crate) keys: HashMap::, + /// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys` + pub(crate) key_count: u32, + /// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified. + pub(crate) clean: bool, + /// track modification count + pub(crate) age: u32, + /// copy of the flags entry on the Dict on-disk + pub(crate) flags: u32, +} +impl DictCacheEntry { + /// Update a key entry. If the key does not already exist, it will create a new one. + /// + /// `key_update` will write `data` starting at `offset`, and will grow the record if data + /// is larger than the current allocation. If `truncate` is false, the existing data past the end of + /// the `data` written is preserved; if `truncate` is true, the excess data past the end of the written + /// data is removed. + /// + /// For small records, a `key_update` call would just want to replace the entire record, so it would have + /// an `offset` of 0, `truncate` is true, and the data would be the new data. However, the `offset` and + /// `truncate` records are particularly useful for updating very large file streams, which can't be + /// held entirely in RAM. + /// + /// Note: it is up to the higher level Basis disambiguation logic to decide the cross-basis update policy: it + /// could either be to update only the dictionary in the latest open basis, update all dictionaries, or update a + /// specific dictionary in a named basis. In all of these cases, the Basis resolver will have had to find the + /// correct DictCacheEntry and issue the `key_update` to it; for multiple updates, then multiple calls to + /// multiple DictCacheEntry are required. + pub fn key_update(&mut self, name: &str, data: &[u8], offset: usize, truncate: bool) -> Result <()> { + Ok(()) + } + + /// If `paranoid` is true, this function calls `key_update` with 0's for the data. In either case, it + /// deletes the key record from the dictionary. + pub fn key_erase(&mut self, name: &str, paranoid: bool) { + + } +} +#[derive(Debug)] +/// On-disk representation of the dictionary header. +#[repr(C, align(8))] +pub(crate) struct Dictionary { + /// Reserved for flags on the record entry + pub(crate) flags: u32, + /// Access count to the dicitionary + pub(crate) age: u32, + /// Number of keys in the dictionary + pub(crate) num_keys: u32, + /// Name. Length should pad out the record to exactly 127 bytes. + pub(crate) name: [u8; DICT_NAME_LEN], +} +impl Deref for Dictionary { + type Target = [u8]; + fn deref(&self) -> &[u8] { + unsafe { + core::slice::from_raw_parts(self as *const Dictionary as *const u8, core::mem::size_of::()) + as &[u8] + } + } +} +impl DerefMut for Dictionary { + fn deref_mut(&mut self) -> &mut [u8] { + unsafe { + core::slice::from_raw_parts_mut(self as *mut Dictionary as *mut u8, core::mem::size_of::()) + as &mut [u8] + } + } +} + +/// This structure "enforces" the 127-byte stride of dict/key vpage entries +#[derive(Copy, Clone)] +pub(crate) struct DictKeyEntry { + pub(crate) data: [u8; DK_STRIDE], +} +impl Default for DictKeyEntry { + fn default() -> DictKeyEntry { + DictKeyEntry { + data: [0; DK_STRIDE] + } + } +} + +/// This structure helps to bookkeep which slices within a DictKey virtual page need to be updated +pub(crate) struct DictKeyVpage { + pub(crate) elements: [Option; VPAGE_SIZE / DK_STRIDE], +} +impl<'a> Default for DictKeyVpage { + fn default() -> DictKeyVpage { + DictKeyVpage { + elements: [None; VPAGE_SIZE / DK_STRIDE], + } + } +} diff --git a/services/pddb/src/backend/hw.rs b/services/pddb/src/backend/hw.rs index a0962b88c..e452fd555 100644 --- a/services/pddb/src/backend/hw.rs +++ b/services/pddb/src/backend/hw.rs @@ -49,6 +49,16 @@ impl Deref for StaticCryptoData { } } +/// this is the structure of the Basis Key in RAM. The "key" and "iv" are actually never committed to +/// flash; only the "salt" is written to disk. The final "salt" is computed as the XOR of the salt on disk +/// and the user-provided "basis name". We never record the "basis name" on disk, so that the existence of +/// any Basis can be denied. +pub(crate) struct BasisKey { + salt: [u8; 16], + key: [u8; 32], // derived from lower 256 bits of sha512(bcrypt(salt, pw)) + iv: [u8; 16], // an IV derived from the upper 128 bits of the sha512 hash from above, XOR with the salt +} + // emulated #[cfg(not(any(target_os = "none", target_os = "xous")))] type EmuMemoryRange = EmuStorage; diff --git a/services/pddb/src/backend/key.rs b/services/pddb/src/backend/key.rs new file mode 100644 index 000000000..dce27f6ca --- /dev/null +++ b/services/pddb/src/backend/key.rs @@ -0,0 +1,91 @@ +use crate::api::*; +use super::*; + +use core::cell::RefCell; +use std::rc::Rc; +use core::ops::{Deref, DerefMut}; +use core::mem::size_of; +use std::convert::TryInto; +use aes_gcm_siv::{Aes256GcmSiv, Nonce}; +use aes_gcm_siv::aead::{Aead, Payload}; +use std::iter::IntoIterator; +use std::collections::HashMap; +use std::io::{Result, Error, ErrorKind}; + +/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so +/// any deserialization must essentially come with a copy step to line up the record. +#[repr(C, align(8))] +pub(crate) struct KeyDescriptor { + /// virtual address of the key's start + pub(crate) start: u64, + /// length of the key's stored data + pub(crate) len: u64, + /// amount of space reserved for the key. Must be >= len. + pub(crate) reserved: u64, + /// Reserved for flags on the record entry + pub(crate) flags: u32, + /// Access count to the key + pub(crate) age: u32, + /// Name. Length should pad out the record to exactly 127 bytes. + pub(crate) name: [u8; KEY_NAME_LEN], +} +impl Deref for KeyDescriptor { + type Target = [u8]; + fn deref(&self) -> &[u8] { + unsafe { + core::slice::from_raw_parts(self as *const KeyDescriptor as *const u8, core::mem::size_of::()) + as &[u8] + } + } +} +impl DerefMut for KeyDescriptor { + fn deref_mut(&mut self) -> &mut [u8] { + unsafe { + core::slice::from_raw_parts_mut(self as *mut KeyDescriptor as *mut u8, core::mem::size_of::()) + as &mut [u8] + } + } +} + +pub(crate) struct KeyCacheEntry { + pub(crate) start: u64, + pub(crate) len: u64, + pub(crate) reserved: u64, + pub(crate) flags: u32, + pub(crate) age: u32, + /// the current on-disk index of the KeyCacheEntry item, enumerated as "0" being the first DictKeyEntry + /// slot past the first record which is the descriptor of the dictionary. + pub(crate) descriptor_index: u32, + /// indicates if the cache entry is currently synchronized with what's on disk. + pub(crate) clean: bool, +} +impl KeyCacheEntry { + /// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself. + pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr { + VirtAddr::new(dict_offset.get() + ((self.descriptor_index as u64 + 1) * DK_STRIDE as u64)).unwrap() + } + /// Computes the modular position of the KeyDescriptor within a vpage. + pub(crate) fn descriptor_modulus(&self) -> usize { + (self.descriptor_index as usize + 1) % (VPAGE_SIZE / DK_STRIDE) + } + /// Computes the vpage offset as measured from the start of the dictionary storage region + pub(crate) fn descriptor_vpage_num(&self) -> usize { + (self.descriptor_index as usize + 1) / (VPAGE_SIZE / DK_STRIDE) + } +} + +/// used to identify cached/chunked data from a key +/// maybe we should detach the cache_page because this is meant to be a key, and cache_page is the payload. +pub(crate) struct KeyCacheEntryEntry { + basis: String, + dict: String, + key: KeyDescriptor, + /// offset of the cached data relative to the KeyDescriptor's start + offset: u64, + /// the actual cached data + cache_page: [u8; VPAGE_SIZE], + /// true if clean and synced with the disk + clean: bool, + /// general flags, tbd + flags: u8, +} \ No newline at end of file