mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-18 08:52:45 +00:00
refactor data structures into smaller files for easier nav
This commit is contained in:
parent
8186930e94
commit
14f4beef98
5 changed files with 221 additions and 191 deletions
|
|
@ -1,5 +1,9 @@
|
|||
mod basis;
|
||||
pub use basis::*;
|
||||
mod dictionary;
|
||||
pub use dictionary::*;
|
||||
mod key;
|
||||
pub use key::*;
|
||||
mod pagetable;
|
||||
pub use pagetable::*;
|
||||
mod fastspace;
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ use std::io::{Result, Error, ErrorKind};
|
|||
/// Memory Pools
|
||||
///
|
||||
/// Key data is split into three categories of sizes: small, medium, and large. The thresholds
|
||||
/// are subject to tuning, but roughly speaking, small data are keys < 512 bytes; medium are ~1k;
|
||||
/// are subject to tuning, but roughly speaking, small data are keys <2k bytes; medium are ~4k;
|
||||
/// and large are bigger than 32k.
|
||||
///
|
||||
/// Large keys are the simplest - each key starts at a VPAGE-aligned address, and allocates
|
||||
|
|
@ -615,196 +615,6 @@ impl BasisCacheEntry {
|
|||
}
|
||||
}
|
||||
|
||||
/// RAM based copy of the dictionary structures on disk.
|
||||
pub(crate) struct DictCacheEntry {
|
||||
/// Use this to compute the virtual address of the dictionary's location
|
||||
/// multiply this by DICT_VSIZE to get at the virtual address. This /could/ be a
|
||||
/// NonZeroU32 type as it should never be 0. Maybe that's a thing to fix later on.
|
||||
pub(crate) index: u32,
|
||||
/// A cache of the keys within the dictionary. If the key does not exist in
|
||||
/// the cache, one should consult the on-disk copy, assuming the record is clean.
|
||||
pub(crate) keys: HashMap::<String, KeyCacheEntry>,
|
||||
/// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys`
|
||||
pub(crate) key_count: u32,
|
||||
/// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified.
|
||||
pub(crate) clean: bool,
|
||||
/// track modification count
|
||||
pub(crate) age: u32,
|
||||
/// copy of the flags entry on the Dict on-disk
|
||||
pub(crate) flags: u32,
|
||||
}
|
||||
impl DictCacheEntry {
|
||||
/// Update a key entry. If the key does not already exist, it will create a new one.
|
||||
///
|
||||
/// `key_update` will write `data` starting at `offset`, and will grow the record if data
|
||||
/// is larger than the current allocation. If `truncate` is false, the existing data past the end of
|
||||
/// the `data` written is preserved; if `truncate` is true, the excess data past the end of the written
|
||||
/// data is removed.
|
||||
///
|
||||
/// For small records, a `key_update` call would just want to replace the entire record, so it would have
|
||||
/// an `offset` of 0, `truncate` is true, and the data would be the new data. However, the `offset` and
|
||||
/// `truncate` records are particularly useful for updating very large file streams, which can't be
|
||||
/// held entirely in RAM.
|
||||
///
|
||||
/// Note: it is up to the higher level Basis disambiguation logic to decide the cross-basis update policy: it
|
||||
/// could either be to update only the dictionary in the latest open basis, update all dictionaries, or update a
|
||||
/// specific dictionary in a named basis. In all of these cases, the Basis resolver will have had to find the
|
||||
/// correct DictCacheEntry and issue the `key_update` to it; for multiple updates, then multiple calls to
|
||||
/// multiple DictCacheEntry are required.
|
||||
pub fn key_update(&mut self, name: &str, data: &[u8], offset: usize, truncate: bool) -> Result <()> {
|
||||
|
||||
}
|
||||
|
||||
/// If `paranoid` is true, this function calls `key_update` with 0's for the data. In either case, it
|
||||
/// deletes the key record from the dictionary.
|
||||
pub fn key_erase(&mut self, name: &str, paranoid: bool) {
|
||||
|
||||
}
|
||||
}
|
||||
#[derive(Debug)]
|
||||
/// On-disk representation of the dictionary header.
|
||||
#[repr(C, align(8))]
|
||||
pub(crate) struct Dictionary {
|
||||
/// Reserved for flags on the record entry
|
||||
pub(crate) flags: u32,
|
||||
/// Access count to the dicitionary
|
||||
pub(crate) age: u32,
|
||||
/// Number of keys in the dictionary
|
||||
pub(crate) num_keys: u32,
|
||||
/// Name. Length should pad out the record to exactly 127 bytes.
|
||||
pub(crate) name: [u8; DICT_NAME_LEN],
|
||||
}
|
||||
impl Deref for Dictionary {
|
||||
type Target = [u8];
|
||||
fn deref(&self) -> &[u8] {
|
||||
unsafe {
|
||||
core::slice::from_raw_parts(self as *const Dictionary as *const u8, core::mem::size_of::<Dictionary>())
|
||||
as &[u8]
|
||||
}
|
||||
}
|
||||
}
|
||||
impl DerefMut for Dictionary {
|
||||
fn deref_mut(&mut self) -> &mut [u8] {
|
||||
unsafe {
|
||||
core::slice::from_raw_parts_mut(self as *mut Dictionary as *mut u8, core::mem::size_of::<Dictionary>())
|
||||
as &mut [u8]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This structure "enforces" the 127-byte stride of dict/key vpage entries
|
||||
#[derive(Copy, Clone)]
|
||||
pub(crate) struct DictKeyEntry {
|
||||
pub(crate) data: [u8; DK_STRIDE],
|
||||
}
|
||||
impl Default for DictKeyEntry {
|
||||
fn default() -> DictKeyEntry {
|
||||
DictKeyEntry {
|
||||
data: [0; DK_STRIDE]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This structure helps to bookkeep which slices within a DictKey virtual page need to be updated
|
||||
pub(crate) struct DictKeyVpage {
|
||||
pub(crate) elements: [Option<DictKeyEntry>; VPAGE_SIZE / DK_STRIDE],
|
||||
}
|
||||
impl<'a> Default for DictKeyVpage {
|
||||
fn default() -> DictKeyVpage {
|
||||
DictKeyVpage {
|
||||
elements: [None; VPAGE_SIZE / DK_STRIDE],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so
|
||||
/// any deserialization must essentially come with a copy step to line up the record.
|
||||
#[repr(C, align(8))]
|
||||
pub(crate) struct KeyDescriptor {
|
||||
/// virtual address of the key's start
|
||||
start: u64,
|
||||
/// length of the key's stored data
|
||||
len: u64,
|
||||
/// amount of space reserved for the key. Must be >= len.
|
||||
reserved: u64,
|
||||
/// Reserved for flags on the record entry
|
||||
flags: u32,
|
||||
/// Access count to the key
|
||||
age: u32,
|
||||
/// Name. Length should pad out the record to exactly 127 bytes.
|
||||
name: [u8; KEY_NAME_LEN],
|
||||
}
|
||||
impl Deref for KeyDescriptor {
|
||||
type Target = [u8];
|
||||
fn deref(&self) -> &[u8] {
|
||||
unsafe {
|
||||
core::slice::from_raw_parts(self as *const KeyDescriptor as *const u8, core::mem::size_of::<KeyDescriptor>())
|
||||
as &[u8]
|
||||
}
|
||||
}
|
||||
}
|
||||
impl DerefMut for KeyDescriptor {
|
||||
fn deref_mut(&mut self) -> &mut [u8] {
|
||||
unsafe {
|
||||
core::slice::from_raw_parts_mut(self as *mut KeyDescriptor as *mut u8, core::mem::size_of::<KeyDescriptor>())
|
||||
as &mut [u8]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct KeyCacheEntry {
|
||||
pub(crate) start: u64,
|
||||
pub(crate) len: u64,
|
||||
pub(crate) reserved: u64,
|
||||
pub(crate) flags: u32,
|
||||
pub(crate) age: u32,
|
||||
/// the current on-disk index of the KeyCacheEntry item, enumerated as "0" being the first DictKeyEntry
|
||||
/// slot past the first record which is the descriptor of the dictionary.
|
||||
pub(crate) descriptor_index: u32,
|
||||
/// indicates if it's clean
|
||||
pub(crate) clean: bool,
|
||||
}
|
||||
impl KeyCacheEntry {
|
||||
/// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself.
|
||||
pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr {
|
||||
VirtAddr::new(dict_offset.get() + ((self.descriptor_index as u64 + 1) * DK_STRIDE as u64)).unwrap()
|
||||
}
|
||||
/// Computes the modular position of the KeyDescriptor within a vpage.
|
||||
pub(crate) fn descriptor_modulus(&self) -> usize {
|
||||
(self.descriptor_index as usize + 1) % (VPAGE_SIZE / DK_STRIDE)
|
||||
}
|
||||
/// Computes the vpage offset as measured from the start of the dictionary storage region
|
||||
pub(crate) fn descriptor_vpage_num(&self) -> usize {
|
||||
(self.descriptor_index as usize + 1) / (VPAGE_SIZE / DK_STRIDE)
|
||||
}
|
||||
}
|
||||
|
||||
/// used to identify cached/chunked data from a key
|
||||
/// maybe we should detach the cache_page because this is meant to be a key, and cache_page is the payload.
|
||||
pub(crate) struct KeyCacheEntryEntry {
|
||||
basis: String,
|
||||
dict: String,
|
||||
key: KeyDescriptor,
|
||||
/// offset of the cached data relative to the KeyDescriptor's start
|
||||
offset: u64,
|
||||
/// the actual cached data
|
||||
cache_page: [u8; VPAGE_SIZE],
|
||||
/// true if clean and synced with the disk
|
||||
clean: bool,
|
||||
/// general flags, tbd
|
||||
flags: u8,
|
||||
}
|
||||
|
||||
/// this is the structure of the Basis Key in RAM. The "key" and "iv" are actually never committed to
|
||||
/// flash; only the "salt" is written to disk. The final "salt" is computed as the XOR of the salt on disk
|
||||
/// and the user-provided "basis name". We never record the "basis name" on disk, so that the existence of
|
||||
/// any Basis can be denied.
|
||||
pub(crate) struct BasisKey {
|
||||
salt: [u8; 16],
|
||||
key: [u8; 32], // derived from lower 256 bits of sha512(bcrypt(salt, pw))
|
||||
iv: [u8; 16], // an IV derived from the upper 128 bits of the sha512 hash from above, XOR with the salt
|
||||
}
|
||||
|
||||
// ****
|
||||
// Beginning of serializers for the data structures in this file.
|
||||
// ****
|
||||
|
|
|
|||
115
services/pddb/src/backend/dictionary.rs
Normal file
115
services/pddb/src/backend/dictionary.rs
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
use crate::api::*;
|
||||
use super::*;
|
||||
|
||||
use core::cell::RefCell;
|
||||
use std::rc::Rc;
|
||||
use core::ops::{Deref, DerefMut};
|
||||
use core::mem::size_of;
|
||||
use std::convert::TryInto;
|
||||
use aes_gcm_siv::{Aes256GcmSiv, Nonce};
|
||||
use aes_gcm_siv::aead::{Aead, Payload};
|
||||
use std::iter::IntoIterator;
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Result, Error, ErrorKind};
|
||||
|
||||
/// RAM based copy of the dictionary structures on disk.
|
||||
pub(crate) struct DictCacheEntry {
|
||||
/// Use this to compute the virtual address of the dictionary's location
|
||||
/// multiply this by DICT_VSIZE to get at the virtual address. This /could/ be a
|
||||
/// NonZeroU32 type as it should never be 0. Maybe that's a thing to fix later on.
|
||||
pub(crate) index: u32,
|
||||
/// A cache of the keys within the dictionary. If the key does not exist in
|
||||
/// the cache, one should consult the on-disk copy, assuming the record is clean.
|
||||
pub(crate) keys: HashMap::<String, KeyCacheEntry>,
|
||||
/// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys`
|
||||
pub(crate) key_count: u32,
|
||||
/// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified.
|
||||
pub(crate) clean: bool,
|
||||
/// track modification count
|
||||
pub(crate) age: u32,
|
||||
/// copy of the flags entry on the Dict on-disk
|
||||
pub(crate) flags: u32,
|
||||
}
|
||||
impl DictCacheEntry {
|
||||
/// Update a key entry. If the key does not already exist, it will create a new one.
|
||||
///
|
||||
/// `key_update` will write `data` starting at `offset`, and will grow the record if data
|
||||
/// is larger than the current allocation. If `truncate` is false, the existing data past the end of
|
||||
/// the `data` written is preserved; if `truncate` is true, the excess data past the end of the written
|
||||
/// data is removed.
|
||||
///
|
||||
/// For small records, a `key_update` call would just want to replace the entire record, so it would have
|
||||
/// an `offset` of 0, `truncate` is true, and the data would be the new data. However, the `offset` and
|
||||
/// `truncate` records are particularly useful for updating very large file streams, which can't be
|
||||
/// held entirely in RAM.
|
||||
///
|
||||
/// Note: it is up to the higher level Basis disambiguation logic to decide the cross-basis update policy: it
|
||||
/// could either be to update only the dictionary in the latest open basis, update all dictionaries, or update a
|
||||
/// specific dictionary in a named basis. In all of these cases, the Basis resolver will have had to find the
|
||||
/// correct DictCacheEntry and issue the `key_update` to it; for multiple updates, then multiple calls to
|
||||
/// multiple DictCacheEntry are required.
|
||||
pub fn key_update(&mut self, name: &str, data: &[u8], offset: usize, truncate: bool) -> Result <()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// If `paranoid` is true, this function calls `key_update` with 0's for the data. In either case, it
|
||||
/// deletes the key record from the dictionary.
|
||||
pub fn key_erase(&mut self, name: &str, paranoid: bool) {
|
||||
|
||||
}
|
||||
}
|
||||
#[derive(Debug)]
|
||||
/// On-disk representation of the dictionary header.
|
||||
#[repr(C, align(8))]
|
||||
pub(crate) struct Dictionary {
|
||||
/// Reserved for flags on the record entry
|
||||
pub(crate) flags: u32,
|
||||
/// Access count to the dicitionary
|
||||
pub(crate) age: u32,
|
||||
/// Number of keys in the dictionary
|
||||
pub(crate) num_keys: u32,
|
||||
/// Name. Length should pad out the record to exactly 127 bytes.
|
||||
pub(crate) name: [u8; DICT_NAME_LEN],
|
||||
}
|
||||
impl Deref for Dictionary {
|
||||
type Target = [u8];
|
||||
fn deref(&self) -> &[u8] {
|
||||
unsafe {
|
||||
core::slice::from_raw_parts(self as *const Dictionary as *const u8, core::mem::size_of::<Dictionary>())
|
||||
as &[u8]
|
||||
}
|
||||
}
|
||||
}
|
||||
impl DerefMut for Dictionary {
|
||||
fn deref_mut(&mut self) -> &mut [u8] {
|
||||
unsafe {
|
||||
core::slice::from_raw_parts_mut(self as *mut Dictionary as *mut u8, core::mem::size_of::<Dictionary>())
|
||||
as &mut [u8]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This structure "enforces" the 127-byte stride of dict/key vpage entries
|
||||
#[derive(Copy, Clone)]
|
||||
pub(crate) struct DictKeyEntry {
|
||||
pub(crate) data: [u8; DK_STRIDE],
|
||||
}
|
||||
impl Default for DictKeyEntry {
|
||||
fn default() -> DictKeyEntry {
|
||||
DictKeyEntry {
|
||||
data: [0; DK_STRIDE]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This structure helps to bookkeep which slices within a DictKey virtual page need to be updated
|
||||
pub(crate) struct DictKeyVpage {
|
||||
pub(crate) elements: [Option<DictKeyEntry>; VPAGE_SIZE / DK_STRIDE],
|
||||
}
|
||||
impl<'a> Default for DictKeyVpage {
|
||||
fn default() -> DictKeyVpage {
|
||||
DictKeyVpage {
|
||||
elements: [None; VPAGE_SIZE / DK_STRIDE],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -49,6 +49,16 @@ impl Deref for StaticCryptoData {
|
|||
}
|
||||
}
|
||||
|
||||
/// this is the structure of the Basis Key in RAM. The "key" and "iv" are actually never committed to
|
||||
/// flash; only the "salt" is written to disk. The final "salt" is computed as the XOR of the salt on disk
|
||||
/// and the user-provided "basis name". We never record the "basis name" on disk, so that the existence of
|
||||
/// any Basis can be denied.
|
||||
pub(crate) struct BasisKey {
|
||||
salt: [u8; 16],
|
||||
key: [u8; 32], // derived from lower 256 bits of sha512(bcrypt(salt, pw))
|
||||
iv: [u8; 16], // an IV derived from the upper 128 bits of the sha512 hash from above, XOR with the salt
|
||||
}
|
||||
|
||||
// emulated
|
||||
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
||||
type EmuMemoryRange = EmuStorage;
|
||||
|
|
|
|||
91
services/pddb/src/backend/key.rs
Normal file
91
services/pddb/src/backend/key.rs
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
use crate::api::*;
|
||||
use super::*;
|
||||
|
||||
use core::cell::RefCell;
|
||||
use std::rc::Rc;
|
||||
use core::ops::{Deref, DerefMut};
|
||||
use core::mem::size_of;
|
||||
use std::convert::TryInto;
|
||||
use aes_gcm_siv::{Aes256GcmSiv, Nonce};
|
||||
use aes_gcm_siv::aead::{Aead, Payload};
|
||||
use std::iter::IntoIterator;
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Result, Error, ErrorKind};
|
||||
|
||||
/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so
|
||||
/// any deserialization must essentially come with a copy step to line up the record.
|
||||
#[repr(C, align(8))]
|
||||
pub(crate) struct KeyDescriptor {
|
||||
/// virtual address of the key's start
|
||||
pub(crate) start: u64,
|
||||
/// length of the key's stored data
|
||||
pub(crate) len: u64,
|
||||
/// amount of space reserved for the key. Must be >= len.
|
||||
pub(crate) reserved: u64,
|
||||
/// Reserved for flags on the record entry
|
||||
pub(crate) flags: u32,
|
||||
/// Access count to the key
|
||||
pub(crate) age: u32,
|
||||
/// Name. Length should pad out the record to exactly 127 bytes.
|
||||
pub(crate) name: [u8; KEY_NAME_LEN],
|
||||
}
|
||||
impl Deref for KeyDescriptor {
|
||||
type Target = [u8];
|
||||
fn deref(&self) -> &[u8] {
|
||||
unsafe {
|
||||
core::slice::from_raw_parts(self as *const KeyDescriptor as *const u8, core::mem::size_of::<KeyDescriptor>())
|
||||
as &[u8]
|
||||
}
|
||||
}
|
||||
}
|
||||
impl DerefMut for KeyDescriptor {
|
||||
fn deref_mut(&mut self) -> &mut [u8] {
|
||||
unsafe {
|
||||
core::slice::from_raw_parts_mut(self as *mut KeyDescriptor as *mut u8, core::mem::size_of::<KeyDescriptor>())
|
||||
as &mut [u8]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct KeyCacheEntry {
|
||||
pub(crate) start: u64,
|
||||
pub(crate) len: u64,
|
||||
pub(crate) reserved: u64,
|
||||
pub(crate) flags: u32,
|
||||
pub(crate) age: u32,
|
||||
/// the current on-disk index of the KeyCacheEntry item, enumerated as "0" being the first DictKeyEntry
|
||||
/// slot past the first record which is the descriptor of the dictionary.
|
||||
pub(crate) descriptor_index: u32,
|
||||
/// indicates if the cache entry is currently synchronized with what's on disk.
|
||||
pub(crate) clean: bool,
|
||||
}
|
||||
impl KeyCacheEntry {
|
||||
/// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself.
|
||||
pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr {
|
||||
VirtAddr::new(dict_offset.get() + ((self.descriptor_index as u64 + 1) * DK_STRIDE as u64)).unwrap()
|
||||
}
|
||||
/// Computes the modular position of the KeyDescriptor within a vpage.
|
||||
pub(crate) fn descriptor_modulus(&self) -> usize {
|
||||
(self.descriptor_index as usize + 1) % (VPAGE_SIZE / DK_STRIDE)
|
||||
}
|
||||
/// Computes the vpage offset as measured from the start of the dictionary storage region
|
||||
pub(crate) fn descriptor_vpage_num(&self) -> usize {
|
||||
(self.descriptor_index as usize + 1) / (VPAGE_SIZE / DK_STRIDE)
|
||||
}
|
||||
}
|
||||
|
||||
/// used to identify cached/chunked data from a key
|
||||
/// maybe we should detach the cache_page because this is meant to be a key, and cache_page is the payload.
|
||||
pub(crate) struct KeyCacheEntryEntry {
|
||||
basis: String,
|
||||
dict: String,
|
||||
key: KeyDescriptor,
|
||||
/// offset of the cached data relative to the KeyDescriptor's start
|
||||
offset: u64,
|
||||
/// the actual cached data
|
||||
cache_page: [u8; VPAGE_SIZE],
|
||||
/// true if clean and synced with the disk
|
||||
clean: bool,
|
||||
/// general flags, tbd
|
||||
flags: u8,
|
||||
}
|
||||
Loading…
Reference in a new issue