moving toward refactoring the basis sync function

This commit is contained in:
bunnie 2021-11-18 00:09:55 +08:00
parent 70a478795d
commit 507d99ee41
7 changed files with 88 additions and 34 deletions

View file

@ -29,10 +29,13 @@ pub(crate) const FSCB_FILL_COEFFICIENT: f32 = 0.5;
/// top-up, to try and mitigate analysis patterns of the amount of free space available based
/// on a fixed ratio reduction over time. Expressed as the extents of a random +/- offset
/// from the FILL_COEFFICIENT.
#[allow(dead_code)]
pub(crate) const FSCB_FILL_UNCERTAINTY: f32 = 0.1;
pub const PDDB_DEFAULT_SYSTEM_BASIS: &'static str = ".System";
#[allow(dead_code)]
pub(crate) const PDDB_DEFAULT_SYSTEM_BASIS: &'static str = ".System";
// this isn't an "official" basis, but it is used for the AAD for encrypting the FastSpace structure
#[allow(dead_code)]
pub(crate) const PDDB_FAST_SPACE_SYSTEM_BASIS: &'static str = ".FastSpace";
#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)]

View file

@ -3,15 +3,11 @@ use super::*;
use core::cell::RefCell;
use std::rc::Rc;
use core::num::NonZeroU64;
use core::ops::{Deref, DerefMut};
use core::{mem, slice};
use core::mem::size_of;
use std::convert::TryInto;
use aes_gcm_siv::{Aes256GcmSiv, Nonce};
use aes_gcm_siv::aead::{Aead, Payload};
use rkyv::Aligned;
use rkyv::ser::serializers::BufferSerializer;
use std::iter::IntoIterator;
use std::collections::HashMap;
use std::io::{Result, Error, ErrorKind};
@ -122,10 +118,10 @@ use std::io::{Result, Error, ErrorKind};
/// This is chosen such that 32 of these entries fit evenly into a VPAGE.
pub(crate) const DK_STRIDE: usize = 127;
/// This is the format of the Basis as stored on disk
#[derive(PartialEq, Debug, Default)]
#[repr(C, align(8))]
pub(crate) struct BasisRoot {
// everything below here is encrypted using AES-GCM-SIV
pub(crate) magic: [u8; 4],
pub(crate) version: u32,
/// increments every time the BasisRoot is modified. This field must saturate, not roll over.
@ -136,6 +132,15 @@ pub(crate) struct BasisRoot {
/// 64-byte name; aligns to 64-bits
pub(crate) name: BasisRootName,
}
impl BasisRoot {
pub(crate) fn aad(&self, dna: u64) -> Vec::<u8> {
let mut aad = Vec::<u8>::new();
aad.extend_from_slice(&self.name.0);
aad.extend_from_slice(&PDDB_VERSION.to_le_bytes());
aad.extend_from_slice(&dna.to_le_bytes());
aad
}
}
impl Deref for BasisRoot {
type Target = [u8];
fn deref(&self) -> &[u8] {
@ -154,6 +159,7 @@ impl DerefMut for BasisRoot {
}
}
/// A list of open Basis that we can use to search and operate upon. Sort of the "root" data structure of the PDDB.
pub(crate) struct BasisCache {
/// the cache entries themselves
cache: Vec::<BasisCacheEntry>,
@ -196,7 +202,7 @@ impl BasisCache {
} else {
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
// doesn't exist. Do a comprehensive search if our cache isn't complete.
if basis.root.num_dictionaries as usize != basis.dicts.len() {
if basis.num_dicts as usize != basis.dicts.len() {
if basis.dict_deep_search(hw, name).is_some() {
return Err(Error::new(ErrorKind::AlreadyExists, "Dictionary already exists"));
}
@ -206,7 +212,7 @@ impl BasisCache {
// allocate a vpage offset for the dictionary
let dict_index = basis.dict_get_free_offset(hw);
let dict_offset = VirtAddr::new(dict_index as u64 * DICT_VSIZE).unwrap();
let pp = basis.v2p_map.entry(dict_offset).or_insert_with(||
basis.v2p_map.entry(dict_offset).or_insert_with(||
hw.try_fast_space_alloc().expect("No free space to allocate dict"));
// create the cache entry
@ -222,9 +228,13 @@ impl BasisCache {
flags: 0,
key_count: 0,
};
log::info!("adding dictionary {}", name);
basis.dicts.insert(String::from(name), dict_cache);
basis.num_dicts += 1;
// encrypt and write the dict entry to disk
basis.dict_sync(hw, name)?;
// sync the root basis structure as well, while we're at it...
basis.basis_sync(hw);
Ok(())
} else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
@ -240,8 +250,9 @@ pub(crate) struct BasisCacheEntry {
pub clean: bool,
/// last sync time, in systicks, if any
pub last_sync: Option<u64>,
/// the basis root record, as read in from disk
pub root: BasisRoot,
/// Number of dictionaries. This should be greater than or equal to the number of elements in dicts.
/// If dicts has less elements than num_dicts, it means there are still dicts on-disk that haven't been cached.
pub num_dicts: u32,
/// dictionary array.
pub dicts: HashMap::<String, DictCacheEntry>,
/// A cached copy of the absolute offset of the next free dictionary slot,
@ -255,6 +266,8 @@ pub(crate) struct BasisCacheEntry {
pub age: u32,
/// virtual to physical page map. It's the reverse mapping of the physical page table on disk.
pub v2p_map: HashMap<VirtAddr, PhysPage>,
/// the last journal rev written to disk
pub journal: u32,
}
impl BasisCacheEntry {
/*
@ -333,7 +346,7 @@ impl BasisCacheEntry {
pub(crate) fn dict_deep_search(&mut self, hw: &mut PddbOs, name: &str) -> Option<(u32, Dictionary)> {
let mut try_entry = 1;
let mut dict_count = 0;
while try_entry <= DICT_MAXCOUNT && dict_count < self.root.num_dictionaries {
while try_entry <= DICT_MAXCOUNT && dict_count < self.num_dicts {
let dict_vaddr = VirtAddr::new(try_entry as u64 * DICT_VSIZE).unwrap();
if let Some(pp) = self.v2p_map.get(&dict_vaddr) {
if let Some(dict) = self.dict_decrypt(hw, &pp) {
@ -386,6 +399,7 @@ impl BasisCacheEntry {
if let Some(dict) = self.dicts.get_mut(&String::from(name)) {
let dict_offset = VirtAddr::new(dict.index as u64 * DICT_VSIZE).unwrap();
if !dict.clean {
log::info!("syncing dictionary {}", name);
let mut dict_name = [0u8; DICT_NAME_LEN];
for (src, dst) in name.bytes().into_iter().zip(dict_name.iter_mut()) {
*dst = src;
@ -461,8 +475,10 @@ impl BasisCacheEntry {
// 3. merge the vpage modifications into the disk
let mut page = if let Some(data) = hw.data_decrypt_page(&self.cipher, &self.aad, &pp) {
log::info!("merging dictionary data into existing page");
data
} else {
log::info!("existing data invalid, creating a new page");
// the existing data was invalid (this happens e.g. on the first time a dict is created). Just overwrite the whole page.
vec![0u8; VPAGE_SIZE + size_of::<JournalType>()]
};
@ -516,6 +532,38 @@ impl BasisCacheEntry {
pub(crate) fn dict_compact(&self, basis_name: Option<&str>) -> Result<()> {
unimplemented!();
}
/// Syncs this cache entry to the hardware.
pub(crate) fn basis_sync(&mut self, hw: &mut PddbOs) {
let basis_root = BasisRoot {
magic: PDDB_MAGIC,
version: PDDB_VERSION,
name: BasisRootName::try_from_str(&self.name).unwrap(),
age: self.age,
num_dictionaries: self.num_dicts,
};
let aad = basis_root.aad(hw.dna());
let pp = self.v2p_map.get(&VirtAddr::new(1 * VPAGE_SIZE as u64).unwrap())
.expect("Internal consistency error: Basis exists, but its root map was not allocated!");
self.journal = self.journal.saturating_add(1);
let journal_bytes = self.journal.to_le_bytes();
let slice_iter =
journal_bytes.iter() // journal rev
.chain(basis_root.as_ref().iter());
let mut block = [0 as u8; VPAGE_SIZE + size_of::<JournalType>()];
for (&src, dst) in slice_iter.zip(block.iter_mut()) {
*dst = src;
}
let nonce = hw.nonce_gen();
let ciphertext = self.cipher.encrypt(
&nonce,
Payload {
aad: &aad,
msg: &block,
}
).unwrap();
hw.patch_data(&ciphertext, pp.page_number() * PAGE_SIZE as u32);
}
}
/// RAM based copy of the dictionary structures on disk.
@ -687,6 +735,20 @@ pub(crate) struct BasisKey {
/// Newtype for BasisRootName so we can give it a default initializer.
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
pub struct BasisRootName(pub [u8; BASIS_NAME_LEN]);
impl BasisRootName {
pub fn try_from_str(name: &str) -> Result<BasisRootName> {
let mut alloc = [0u8; BASIS_NAME_LEN];
let bytes = name.as_bytes();
if bytes.len() > BASIS_NAME_LEN {
Err(Error::new(ErrorKind::InvalidInput, "basis name is too long")) // FileNameTooLong is still nightly :-/
} else {
for (&src, dst) in bytes.iter().zip(alloc.iter_mut()) {
*dst = src;
}
Ok(BasisRootName(alloc))
}
}
}
impl Default for BasisRootName {
fn default() -> BasisRootName {
BasisRootName([0; BASIS_NAME_LEN])

View file

@ -1,4 +1,3 @@
use core::num::NonZeroU32;
use core::ops::{Deref, DerefMut};
use core::convert::TryInto;
@ -156,7 +155,6 @@ impl DerefMut for SpaceUpdate {
mod tests {
use super::*;
#[test]
fn test_fast_space_size() {
assert!(core::mem::size_of::<FastSpaceInFlash>() & (PAGE_SIZE - 1) == 0, "FastSpaceInFlash is not exactly a multiple of one page in size");

View file

@ -19,7 +19,6 @@ use std::io::{Result, Error, ErrorKind};
pub(crate) const MBBB_PAGES: usize = 10;
pub(crate) const FSCB_PAGES: usize = 16;
pub(crate) const INITIAL_BASIS_ALLOC: usize = 16;
/// size of a physical page
pub const PAGE_SIZE: usize = spinor::SPINOR_ERASE_SIZE as usize;
@ -200,6 +199,7 @@ impl PddbOs {
let nonce_array = self.entropy.borrow_mut().get_nonce();
*Nonce::from_slice(&nonce_array)
}
pub(crate) fn dna(&self) -> u64 {self.dna}
/// patches data at an offset starting from the data physical base address, which corresponds
/// exactly to the first entry in the page table
@ -972,13 +972,14 @@ impl PddbOs {
name: basis_name.clone(),
clean: true,
last_sync: Some(self.tt.elapsed_ms()),
root: basis_root,
num_dicts: basis_root.num_dictionaries,
dicts: HashMap::<String, DictCacheEntry>::new(),
cipher,
aad,
age: 0,
age: basis_root.age,
free_dict_offset: None,
v2p_map: sysbasis_map,
journal: u32::from_le_bytes(vpage[..size_of::<JournalType>()].try_into().unwrap()),
};
log::info!("System BasisRoot record found, saving in cache");
// save the v2p reverse mapping too.
@ -1095,14 +1096,10 @@ impl PddbOs {
}
// step 6. create the system basis root structure
let mut name: [u8; BASIS_NAME_LEN] = [0; BASIS_NAME_LEN];
for (&src, dst) in PDDB_DEFAULT_SYSTEM_BASIS.as_bytes().iter().zip(name.iter_mut()) {
*dst = src;
}
let basis_root = BasisRoot {
magic: api::PDDB_MAGIC,
version: api::PDDB_VERSION,
name: BasisRootName(name),
name: BasisRootName::try_from_str(PDDB_DEFAULT_SYSTEM_BASIS).unwrap(),
age: 0,
num_dictionaries: 0,
};
@ -1171,11 +1168,6 @@ impl PddbOs {
Ok(())
}
/// syncs up the named basis
pub(crate) fn basis_sync(&self, name: &str) {
////// TODO HERE
}
/// This function will prompt the user to unlock all the Basis. If the user asserts all
/// Basis have been unlocked, the function returns `true`. The other option is the user
/// can decline to unlock all the Basis right now, cancelling out of the process, which will

View file

@ -1,5 +1,4 @@
use core::num::{NonZeroU32, NonZeroU64};
use core::convert::TryFrom;
use core::num::NonZeroU64;
use core::ops::Add;
use super::{PAGE_SIZE, VPAGE_SIZE};
use crate::SpaceState;
@ -179,7 +178,6 @@ impl Add for PageAlignedPa {
mod tests {
use super::*;
#[test]
/// PAGE_SIZE is required to be a power of two. 0x1000 -> 0x1000 - 1 = 0xFFF, which forms the bitmasks.
fn test_page_size() {
@ -192,7 +190,7 @@ mod tests {
/// obvious but not explicitly stated fact always remains true.
#[test]
fn test_bitfield_bool() {
bitfield! {
super::bitfield! {
pub struct Test(u8);
impl Debug;
pub test, set_test: 1;

View file

@ -371,9 +371,6 @@ fn xmain() -> ! {
pddb_os.pddb_format().unwrap();
log::info!("Done initializing disk");
#[cfg(not(any(target_os = "none", target_os = "xous")))]
pddb_os.dbg_dump();
// it's a vector because order is important: by default access to keys/dicts go into the latest entry first, and then recurse to the earliest
let mut basis_cache = BasisCache::new();
@ -387,8 +384,12 @@ fn xmain() -> ! {
log::info!("size of vpage: {}", VPAGE_SIZE);
// add a "system settings" dictionary to the default basis
log::info!("adding 'system settings' dictionary");
basis_cache.dict_add(&mut pddb_os, "system settings", None).expect("couldn't add system settings dictionary");
#[cfg(not(any(target_os = "none", target_os = "xous")))]
pddb_os.dbg_dump();
// register a suspend/resume listener
let sr_cid = xous::connect(pddb_sid).expect("couldn't create suspend callback connection");
let susres = susres::Susres::new(&xns, api::Opcode::SuspendResume as u32, sr_cid).expect("couldn't create suspend/resume object");

View file

@ -171,7 +171,7 @@ def main():
if errors == 0:
print([hex(x) for x in basis_data[:256]])
# print([hex(x) for x in basis_data[:256]])
basis = Basis(basis_data)
print(basis.as_str())