refactor data structures into smaller files for easier nav

This commit is contained in:
bunnie 2021-11-18 15:38:12 +08:00
parent 8186930e94
commit 14f4beef98
5 changed files with 221 additions and 191 deletions

View file

@ -1,5 +1,9 @@
mod basis; mod basis;
pub use basis::*; pub use basis::*;
mod dictionary;
pub use dictionary::*;
mod key;
pub use key::*;
mod pagetable; mod pagetable;
pub use pagetable::*; pub use pagetable::*;
mod fastspace; mod fastspace;

View file

@ -87,7 +87,7 @@ use std::io::{Result, Error, ErrorKind};
/// Memory Pools /// Memory Pools
/// ///
/// Key data is split into three categories of sizes: small, medium, and large. The thresholds /// Key data is split into three categories of sizes: small, medium, and large. The thresholds
/// are subject to tuning, but roughly speaking, small data are keys < 512 bytes; medium are ~1k; /// are subject to tuning, but roughly speaking, small data are keys <2k bytes; medium are ~4k;
/// and large are bigger than 32k. /// and large are bigger than 32k.
/// ///
/// Large keys are the simplest - each key starts at a VPAGE-aligned address, and allocates /// Large keys are the simplest - each key starts at a VPAGE-aligned address, and allocates
@ -615,196 +615,6 @@ impl BasisCacheEntry {
} }
} }
/// RAM based copy of the dictionary structures on disk.
pub(crate) struct DictCacheEntry {
/// Use this to compute the virtual address of the dictionary's location
/// multiply this by DICT_VSIZE to get at the virtual address. This /could/ be a
/// NonZeroU32 type as it should never be 0. Maybe that's a thing to fix later on.
pub(crate) index: u32,
/// A cache of the keys within the dictionary. If the key does not exist in
/// the cache, one should consult the on-disk copy, assuming the record is clean.
pub(crate) keys: HashMap::<String, KeyCacheEntry>,
/// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys`
pub(crate) key_count: u32,
/// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified.
pub(crate) clean: bool,
/// track modification count
pub(crate) age: u32,
/// copy of the flags entry on the Dict on-disk
pub(crate) flags: u32,
}
impl DictCacheEntry {
/// Update a key entry. If the key does not already exist, it will create a new one.
///
/// `key_update` will write `data` starting at `offset`, and will grow the record if data
/// is larger than the current allocation. If `truncate` is false, the existing data past the end of
/// the `data` written is preserved; if `truncate` is true, the excess data past the end of the written
/// data is removed.
///
/// For small records, a `key_update` call would just want to replace the entire record, so it would have
/// an `offset` of 0, `truncate` is true, and the data would be the new data. However, the `offset` and
/// `truncate` records are particularly useful for updating very large file streams, which can't be
/// held entirely in RAM.
///
/// Note: it is up to the higher level Basis disambiguation logic to decide the cross-basis update policy: it
/// could either be to update only the dictionary in the latest open basis, update all dictionaries, or update a
/// specific dictionary in a named basis. In all of these cases, the Basis resolver will have had to find the
/// correct DictCacheEntry and issue the `key_update` to it; for multiple updates, then multiple calls to
/// multiple DictCacheEntry are required.
pub fn key_update(&mut self, name: &str, data: &[u8], offset: usize, truncate: bool) -> Result <()> {
}
/// If `paranoid` is true, this function calls `key_update` with 0's for the data. In either case, it
/// deletes the key record from the dictionary.
pub fn key_erase(&mut self, name: &str, paranoid: bool) {
}
}
#[derive(Debug)]
/// On-disk representation of the dictionary header.
#[repr(C, align(8))]
pub(crate) struct Dictionary {
/// Reserved for flags on the record entry
pub(crate) flags: u32,
/// Access count to the dicitionary
pub(crate) age: u32,
/// Number of keys in the dictionary
pub(crate) num_keys: u32,
/// Name. Length should pad out the record to exactly 127 bytes.
pub(crate) name: [u8; DICT_NAME_LEN],
}
impl Deref for Dictionary {
type Target = [u8];
fn deref(&self) -> &[u8] {
unsafe {
core::slice::from_raw_parts(self as *const Dictionary as *const u8, core::mem::size_of::<Dictionary>())
as &[u8]
}
}
}
impl DerefMut for Dictionary {
fn deref_mut(&mut self) -> &mut [u8] {
unsafe {
core::slice::from_raw_parts_mut(self as *mut Dictionary as *mut u8, core::mem::size_of::<Dictionary>())
as &mut [u8]
}
}
}
/// This structure "enforces" the 127-byte stride of dict/key vpage entries
#[derive(Copy, Clone)]
pub(crate) struct DictKeyEntry {
pub(crate) data: [u8; DK_STRIDE],
}
impl Default for DictKeyEntry {
fn default() -> DictKeyEntry {
DictKeyEntry {
data: [0; DK_STRIDE]
}
}
}
/// This structure helps to bookkeep which slices within a DictKey virtual page need to be updated
pub(crate) struct DictKeyVpage {
pub(crate) elements: [Option<DictKeyEntry>; VPAGE_SIZE / DK_STRIDE],
}
impl<'a> Default for DictKeyVpage {
fn default() -> DictKeyVpage {
DictKeyVpage {
elements: [None; VPAGE_SIZE / DK_STRIDE],
}
}
}
/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so
/// any deserialization must essentially come with a copy step to line up the record.
#[repr(C, align(8))]
pub(crate) struct KeyDescriptor {
/// virtual address of the key's start
start: u64,
/// length of the key's stored data
len: u64,
/// amount of space reserved for the key. Must be >= len.
reserved: u64,
/// Reserved for flags on the record entry
flags: u32,
/// Access count to the key
age: u32,
/// Name. Length should pad out the record to exactly 127 bytes.
name: [u8; KEY_NAME_LEN],
}
impl Deref for KeyDescriptor {
type Target = [u8];
fn deref(&self) -> &[u8] {
unsafe {
core::slice::from_raw_parts(self as *const KeyDescriptor as *const u8, core::mem::size_of::<KeyDescriptor>())
as &[u8]
}
}
}
impl DerefMut for KeyDescriptor {
fn deref_mut(&mut self) -> &mut [u8] {
unsafe {
core::slice::from_raw_parts_mut(self as *mut KeyDescriptor as *mut u8, core::mem::size_of::<KeyDescriptor>())
as &mut [u8]
}
}
}
pub(crate) struct KeyCacheEntry {
pub(crate) start: u64,
pub(crate) len: u64,
pub(crate) reserved: u64,
pub(crate) flags: u32,
pub(crate) age: u32,
/// the current on-disk index of the KeyCacheEntry item, enumerated as "0" being the first DictKeyEntry
/// slot past the first record which is the descriptor of the dictionary.
pub(crate) descriptor_index: u32,
/// indicates if it's clean
pub(crate) clean: bool,
}
impl KeyCacheEntry {
/// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself.
pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr {
VirtAddr::new(dict_offset.get() + ((self.descriptor_index as u64 + 1) * DK_STRIDE as u64)).unwrap()
}
/// Computes the modular position of the KeyDescriptor within a vpage.
pub(crate) fn descriptor_modulus(&self) -> usize {
(self.descriptor_index as usize + 1) % (VPAGE_SIZE / DK_STRIDE)
}
/// Computes the vpage offset as measured from the start of the dictionary storage region
pub(crate) fn descriptor_vpage_num(&self) -> usize {
(self.descriptor_index as usize + 1) / (VPAGE_SIZE / DK_STRIDE)
}
}
/// used to identify cached/chunked data from a key
/// maybe we should detach the cache_page because this is meant to be a key, and cache_page is the payload.
pub(crate) struct KeyCacheEntryEntry {
basis: String,
dict: String,
key: KeyDescriptor,
/// offset of the cached data relative to the KeyDescriptor's start
offset: u64,
/// the actual cached data
cache_page: [u8; VPAGE_SIZE],
/// true if clean and synced with the disk
clean: bool,
/// general flags, tbd
flags: u8,
}
/// this is the structure of the Basis Key in RAM. The "key" and "iv" are actually never committed to
/// flash; only the "salt" is written to disk. The final "salt" is computed as the XOR of the salt on disk
/// and the user-provided "basis name". We never record the "basis name" on disk, so that the existence of
/// any Basis can be denied.
pub(crate) struct BasisKey {
salt: [u8; 16],
key: [u8; 32], // derived from lower 256 bits of sha512(bcrypt(salt, pw))
iv: [u8; 16], // an IV derived from the upper 128 bits of the sha512 hash from above, XOR with the salt
}
// **** // ****
// Beginning of serializers for the data structures in this file. // Beginning of serializers for the data structures in this file.
// **** // ****

View file

@ -0,0 +1,115 @@
use crate::api::*;
use super::*;
use core::cell::RefCell;
use std::rc::Rc;
use core::ops::{Deref, DerefMut};
use core::mem::size_of;
use std::convert::TryInto;
use aes_gcm_siv::{Aes256GcmSiv, Nonce};
use aes_gcm_siv::aead::{Aead, Payload};
use std::iter::IntoIterator;
use std::collections::HashMap;
use std::io::{Result, Error, ErrorKind};
/// RAM based copy of the dictionary structures on disk.
pub(crate) struct DictCacheEntry {
/// Use this to compute the virtual address of the dictionary's location
/// multiply this by DICT_VSIZE to get at the virtual address. This /could/ be a
/// NonZeroU32 type as it should never be 0. Maybe that's a thing to fix later on.
pub(crate) index: u32,
/// A cache of the keys within the dictionary. If the key does not exist in
/// the cache, one should consult the on-disk copy, assuming the record is clean.
pub(crate) keys: HashMap::<String, KeyCacheEntry>,
/// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys`
pub(crate) key_count: u32,
/// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified.
pub(crate) clean: bool,
/// track modification count
pub(crate) age: u32,
/// copy of the flags entry on the Dict on-disk
pub(crate) flags: u32,
}
impl DictCacheEntry {
/// Update a key entry. If the key does not already exist, it will create a new one.
///
/// `key_update` will write `data` starting at `offset`, and will grow the record if data
/// is larger than the current allocation. If `truncate` is false, the existing data past the end of
/// the `data` written is preserved; if `truncate` is true, the excess data past the end of the written
/// data is removed.
///
/// For small records, a `key_update` call would just want to replace the entire record, so it would have
/// an `offset` of 0, `truncate` is true, and the data would be the new data. However, the `offset` and
/// `truncate` records are particularly useful for updating very large file streams, which can't be
/// held entirely in RAM.
///
/// Note: it is up to the higher level Basis disambiguation logic to decide the cross-basis update policy: it
/// could either be to update only the dictionary in the latest open basis, update all dictionaries, or update a
/// specific dictionary in a named basis. In all of these cases, the Basis resolver will have had to find the
/// correct DictCacheEntry and issue the `key_update` to it; for multiple updates, then multiple calls to
/// multiple DictCacheEntry are required.
pub fn key_update(&mut self, name: &str, data: &[u8], offset: usize, truncate: bool) -> Result <()> {
Ok(())
}
/// If `paranoid` is true, this function calls `key_update` with 0's for the data. In either case, it
/// deletes the key record from the dictionary.
pub fn key_erase(&mut self, name: &str, paranoid: bool) {
}
}
#[derive(Debug)]
/// On-disk representation of the dictionary header.
#[repr(C, align(8))]
pub(crate) struct Dictionary {
/// Reserved for flags on the record entry
pub(crate) flags: u32,
/// Access count to the dicitionary
pub(crate) age: u32,
/// Number of keys in the dictionary
pub(crate) num_keys: u32,
/// Name. Length should pad out the record to exactly 127 bytes.
pub(crate) name: [u8; DICT_NAME_LEN],
}
impl Deref for Dictionary {
type Target = [u8];
fn deref(&self) -> &[u8] {
unsafe {
core::slice::from_raw_parts(self as *const Dictionary as *const u8, core::mem::size_of::<Dictionary>())
as &[u8]
}
}
}
impl DerefMut for Dictionary {
fn deref_mut(&mut self) -> &mut [u8] {
unsafe {
core::slice::from_raw_parts_mut(self as *mut Dictionary as *mut u8, core::mem::size_of::<Dictionary>())
as &mut [u8]
}
}
}
/// This structure "enforces" the 127-byte stride of dict/key vpage entries
#[derive(Copy, Clone)]
pub(crate) struct DictKeyEntry {
pub(crate) data: [u8; DK_STRIDE],
}
impl Default for DictKeyEntry {
fn default() -> DictKeyEntry {
DictKeyEntry {
data: [0; DK_STRIDE]
}
}
}
/// This structure helps to bookkeep which slices within a DictKey virtual page need to be updated
pub(crate) struct DictKeyVpage {
pub(crate) elements: [Option<DictKeyEntry>; VPAGE_SIZE / DK_STRIDE],
}
impl<'a> Default for DictKeyVpage {
fn default() -> DictKeyVpage {
DictKeyVpage {
elements: [None; VPAGE_SIZE / DK_STRIDE],
}
}
}

View file

@ -49,6 +49,16 @@ impl Deref for StaticCryptoData {
} }
} }
/// this is the structure of the Basis Key in RAM. The "key" and "iv" are actually never committed to
/// flash; only the "salt" is written to disk. The final "salt" is computed as the XOR of the salt on disk
/// and the user-provided "basis name". We never record the "basis name" on disk, so that the existence of
/// any Basis can be denied.
pub(crate) struct BasisKey {
salt: [u8; 16],
key: [u8; 32], // derived from lower 256 bits of sha512(bcrypt(salt, pw))
iv: [u8; 16], // an IV derived from the upper 128 bits of the sha512 hash from above, XOR with the salt
}
// emulated // emulated
#[cfg(not(any(target_os = "none", target_os = "xous")))] #[cfg(not(any(target_os = "none", target_os = "xous")))]
type EmuMemoryRange = EmuStorage; type EmuMemoryRange = EmuStorage;

View file

@ -0,0 +1,91 @@
use crate::api::*;
use super::*;
use core::cell::RefCell;
use std::rc::Rc;
use core::ops::{Deref, DerefMut};
use core::mem::size_of;
use std::convert::TryInto;
use aes_gcm_siv::{Aes256GcmSiv, Nonce};
use aes_gcm_siv::aead::{Aead, Payload};
use std::iter::IntoIterator;
use std::collections::HashMap;
use std::io::{Result, Error, ErrorKind};
/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so
/// any deserialization must essentially come with a copy step to line up the record.
#[repr(C, align(8))]
pub(crate) struct KeyDescriptor {
/// virtual address of the key's start
pub(crate) start: u64,
/// length of the key's stored data
pub(crate) len: u64,
/// amount of space reserved for the key. Must be >= len.
pub(crate) reserved: u64,
/// Reserved for flags on the record entry
pub(crate) flags: u32,
/// Access count to the key
pub(crate) age: u32,
/// Name. Length should pad out the record to exactly 127 bytes.
pub(crate) name: [u8; KEY_NAME_LEN],
}
impl Deref for KeyDescriptor {
type Target = [u8];
fn deref(&self) -> &[u8] {
unsafe {
core::slice::from_raw_parts(self as *const KeyDescriptor as *const u8, core::mem::size_of::<KeyDescriptor>())
as &[u8]
}
}
}
impl DerefMut for KeyDescriptor {
fn deref_mut(&mut self) -> &mut [u8] {
unsafe {
core::slice::from_raw_parts_mut(self as *mut KeyDescriptor as *mut u8, core::mem::size_of::<KeyDescriptor>())
as &mut [u8]
}
}
}
pub(crate) struct KeyCacheEntry {
pub(crate) start: u64,
pub(crate) len: u64,
pub(crate) reserved: u64,
pub(crate) flags: u32,
pub(crate) age: u32,
/// the current on-disk index of the KeyCacheEntry item, enumerated as "0" being the first DictKeyEntry
/// slot past the first record which is the descriptor of the dictionary.
pub(crate) descriptor_index: u32,
/// indicates if the cache entry is currently synchronized with what's on disk.
pub(crate) clean: bool,
}
impl KeyCacheEntry {
/// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself.
pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr {
VirtAddr::new(dict_offset.get() + ((self.descriptor_index as u64 + 1) * DK_STRIDE as u64)).unwrap()
}
/// Computes the modular position of the KeyDescriptor within a vpage.
pub(crate) fn descriptor_modulus(&self) -> usize {
(self.descriptor_index as usize + 1) % (VPAGE_SIZE / DK_STRIDE)
}
/// Computes the vpage offset as measured from the start of the dictionary storage region
pub(crate) fn descriptor_vpage_num(&self) -> usize {
(self.descriptor_index as usize + 1) / (VPAGE_SIZE / DK_STRIDE)
}
}
/// used to identify cached/chunked data from a key
/// maybe we should detach the cache_page because this is meant to be a key, and cache_page is the payload.
pub(crate) struct KeyCacheEntryEntry {
basis: String,
dict: String,
key: KeyDescriptor,
/// offset of the cached data relative to the KeyDescriptor's start
offset: u64,
/// the actual cached data
cache_page: [u8; VPAGE_SIZE],
/// true if clean and synced with the disk
clean: bool,
/// general flags, tbd
flags: u8,
}