mirror of
https://github.com/saymrwulf/xous-core.git
synced 2026-11-18 08:52:45 +00:00
now passing dictionary add/remove test
This commit is contained in:
parent
77d3f8a646
commit
d716fea17a
6 changed files with 216 additions and 131 deletions
|
|
@ -7,7 +7,7 @@ use std::convert::TryInto;
|
||||||
use aes_gcm_siv::{Aes256GcmSiv, Key};
|
use aes_gcm_siv::{Aes256GcmSiv, Key};
|
||||||
use aes_gcm_siv::aead::NewAead;
|
use aes_gcm_siv::aead::NewAead;
|
||||||
use std::iter::IntoIterator;
|
use std::iter::IntoIterator;
|
||||||
use std::collections::{BinaryHeap, HashMap};
|
use std::collections::{BinaryHeap, HashMap, HashSet};
|
||||||
use std::io::{Result, Error, ErrorKind};
|
use std::io::{Result, Error, ErrorKind};
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -304,32 +304,37 @@ impl BasisCache {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns a list of all the known dictionaries, across all the basis. A HashSet is returned
|
||||||
|
/// because you can have the same-named dictionary in multiple basis, and what we're asking for
|
||||||
|
/// is the union of all the dictionary names, without duplicates.
|
||||||
|
pub(crate) fn dict_list(&mut self, hw: &mut PddbOs) -> HashSet::<String> {
|
||||||
|
let mut dict_set = HashSet::<String>::new();
|
||||||
|
for basis in self.cache.iter_mut() {
|
||||||
|
basis.populate_caches(hw);
|
||||||
|
for key in basis.dicts.keys() {
|
||||||
|
dict_set.insert(String::from(key));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dict_set
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This version of the call only removes one instance of a dictionary from the specified basis.
|
||||||
|
/// Perhaps there also needs to be a `dict_remove_all` call which iterates through every basis
|
||||||
|
/// makes sure the dictionary is removed from all the possible known basis. Anyways, that function
|
||||||
|
/// would be a variant of this targeted version.
|
||||||
pub(crate) fn dict_remove(&mut self,
|
pub(crate) fn dict_remove(&mut self,
|
||||||
hw: &mut PddbOs, dict: &str, basis_name: Option<&str>, paranoid: bool
|
hw: &mut PddbOs, dict: &str, basis_name: Option<&str>, paranoid: bool
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
if let Some(basis_index) = self.select_basis(basis_name) {
|
if let Some(basis_index) = self.select_basis(basis_name) {
|
||||||
|
log::info!("deleting dict {}", dict);
|
||||||
let basis = &mut self.cache[basis_index];
|
let basis = &mut self.cache[basis_index];
|
||||||
// make sure the cache is "hot" before checking it. The rule is we only operate on data in cache, then sync it to disk.
|
|
||||||
if !basis.dicts.contains_key(dict) {
|
basis.age = basis.age.saturating_add(1);
|
||||||
if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) {
|
basis.clean = false;
|
||||||
if dict_record.flags.valid() {
|
basis.dict_delete(hw, dict, paranoid)?;
|
||||||
let dict_name = String::from(cstr_to_string(&dict_record.name));
|
basis.basis_sync(hw);
|
||||||
let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad);
|
basis.pt_sync(hw);
|
||||||
basis.dicts.insert(dict_name, dcache);
|
Ok(())
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// now check the cache.
|
|
||||||
if basis.dicts.contains_key(dict) {
|
|
||||||
basis.age = basis.age.saturating_add(1);
|
|
||||||
basis.clean = false;
|
|
||||||
return basis.dict_delete(hw, dict, paranoid)
|
|
||||||
} else {
|
|
||||||
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
|
Err(Error::new(ErrorKind::NotFound, "Requested basis not found, or PDDB not mounted."))
|
||||||
}
|
}
|
||||||
|
|
@ -705,6 +710,7 @@ impl BasisCacheEntry {
|
||||||
self.large_alloc_ptr = Some(PageAlignedVa::from(maybe_end));
|
self.large_alloc_ptr = Some(PageAlignedVa::from(maybe_end));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// allocate a pointer data in the large pool, of length `amount`. "always" succeeds because...
|
/// allocate a pointer data in the large pool, of length `amount`. "always" succeeds because...
|
||||||
/// there's 16 million terabytes of large pool to allocate before you run out?
|
/// there's 16 million terabytes of large pool to allocate before you run out?
|
||||||
/*
|
/*
|
||||||
|
|
@ -717,6 +723,7 @@ impl BasisCacheEntry {
|
||||||
self.large_alloc_ptr = Some(self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)) + PageAlignedVa::from(amount));
|
self.large_alloc_ptr = Some(self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)) + PageAlignedVa::from(amount));
|
||||||
return alloc_ptr.as_u64()
|
return alloc_ptr.as_u64()
|
||||||
}*/
|
}*/
|
||||||
|
|
||||||
/// do a deep scan of all the dictionaries and keys and attempt to populate all the caches
|
/// do a deep scan of all the dictionaries and keys and attempt to populate all the caches
|
||||||
pub(crate) fn populate_caches(&mut self, hw: &mut PddbOs) {
|
pub(crate) fn populate_caches(&mut self, hw: &mut PddbOs) {
|
||||||
let mut try_entry = 1;
|
let mut try_entry = 1;
|
||||||
|
|
@ -758,6 +765,7 @@ impl BasisCacheEntry {
|
||||||
pub(crate) fn dict_delete(&mut self, hw: &mut PddbOs, name: &str, paranoid: bool) -> Result<()> {
|
pub(crate) fn dict_delete(&mut self, hw: &mut PddbOs, name: &str, paranoid: bool) -> Result<()> {
|
||||||
let mut dict_found = false;
|
let mut dict_found = false;
|
||||||
if !self.dicts.contains_key(name) {
|
if !self.dicts.contains_key(name) {
|
||||||
|
log::info!("dict delete: key not in cache {}", name);
|
||||||
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
|
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
|
||||||
// doesn't exist. Do a comprehensive search if our cache isn't complete.
|
// doesn't exist. Do a comprehensive search if our cache isn't complete.
|
||||||
if let Some((index, dict_record)) = self.dict_deep_search(hw, name) {
|
if let Some((index, dict_record)) = self.dict_deep_search(hw, name) {
|
||||||
|
|
@ -771,43 +779,52 @@ impl BasisCacheEntry {
|
||||||
}
|
}
|
||||||
if dict_found {
|
if dict_found {
|
||||||
let dcache = self.dicts.get_mut(name).expect("entry was ensured, but somehow missing");
|
let dcache = self.dicts.get_mut(name).expect("entry was ensured, but somehow missing");
|
||||||
|
//log::info!("dcache {} index {}, key_count {}", name, dcache.index, dcache.key_count);
|
||||||
// ensure all the keys are in RAM
|
// ensure all the keys are in RAM
|
||||||
dcache.fill(hw, &self.v2p_map, &self.cipher);
|
dcache.fill(hw, &self.v2p_map, &self.cipher);
|
||||||
|
//log::info!("dcache filled");
|
||||||
// allocate a copy of the key list, to avoid interior mutability problems with the next remove step
|
// allocate a copy of the key list, to avoid interior mutability problems with the next remove step
|
||||||
let mut key_list = Vec::<String>::new();
|
let mut key_list = Vec::<String>::new();
|
||||||
for key in dcache.keys.keys() {
|
for key in dcache.keys.keys() {
|
||||||
key_list.push(key.to_string());
|
key_list.push(key.to_string());
|
||||||
}
|
}
|
||||||
for key in key_list {
|
for key in key_list {
|
||||||
|
//log::info!("removing {}:{}", name, key);
|
||||||
// this will wipe any large pools if paranoid is set
|
// this will wipe any large pools if paranoid is set
|
||||||
dcache.key_remove(hw, &mut self.v2p_map, &self.cipher, &key, paranoid);
|
dcache.key_remove(hw, &mut self.v2p_map, &self.cipher, &key, paranoid);
|
||||||
}
|
}
|
||||||
// wipe & de-allocate any small pages
|
// wipe & de-allocate any small pages
|
||||||
for index in 0..dcache.small_pool.len() {
|
for index in 0..dcache.small_pool.len() {
|
||||||
let pool_vaddr = VirtAddr::new(dcache.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START + index as u64 * SMALL_CAPACITY as u64).unwrap();
|
let pool_vaddr = VirtAddr::new(dcache.index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START + index as u64 * SMALL_CAPACITY as u64).unwrap();
|
||||||
if let Some(pp) = self.v2p_map.get(&pool_vaddr) {
|
if let Some(pp) = self.v2p_map.remove(&pool_vaddr) {
|
||||||
if paranoid {
|
if paranoid {
|
||||||
let mut random = [0u8; PAGE_SIZE];
|
let mut random = [0u8; PAGE_SIZE];
|
||||||
hw.trng_slice(&mut random);
|
hw.trng_slice(&mut random);
|
||||||
hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
|
hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
|
||||||
}
|
}
|
||||||
hw.fast_space_free(*pp);
|
hw.fast_space_free(pp);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// erase the header by writing over with random data. This makes the dictionary unsearchable, but if you
|
// erase the entire dictionary + key allocation area by writing over with random data. It's important to
|
||||||
// have the key, you can of course do a hard-scan and try partially re-assemble the dictionary if you
|
// do a comprehensive erase, because if the dictionary slot is re-used, the previously allocated key entries
|
||||||
// did not do the delete in paranoid mode.
|
// decrypt correctly, are interpreted as valid keys, and thus cause consistency errors.
|
||||||
if let Some(pp) = self.v2p_map.get(&VirtAddr::new(dcache.index as u64 * DICT_VSIZE as u64).unwrap()) {
|
let key_pages = 1 + (dcache.last_disk_key_index + 1) as usize / DK_PER_VPAGE;
|
||||||
let mut random = [0u8; PAGE_SIZE];
|
for page in 0..key_pages {
|
||||||
hw.trng_slice(&mut random);
|
let dk_vaddr = VirtAddr::new(dcache.index as u64 * DICT_VSIZE as u64 + page as u64 * VPAGE_SIZE as u64).unwrap();
|
||||||
hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
|
if let Some(pp) = self.v2p_map.remove(&dk_vaddr) {
|
||||||
hw.fast_space_free(*pp);
|
log::info!("erasing dk page 0x{:x}/0x{:x}", dk_vaddr, pp.page_number() as usize * PAGE_SIZE);
|
||||||
} else {
|
let mut random = [0u8; PAGE_SIZE];
|
||||||
log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table.");
|
hw.trng_slice(&mut random);
|
||||||
|
hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
|
||||||
|
hw.fast_space_free(pp);
|
||||||
|
} else {
|
||||||
|
log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// mark data for re-use
|
// mark data for re-use
|
||||||
self.free_dict_offset = Some(dcache.index);
|
self.free_dict_offset = Some(dcache.index);
|
||||||
|
self.num_dicts -= 1;
|
||||||
// remove the cache entry
|
// remove the cache entry
|
||||||
self.dicts.remove(name);
|
self.dicts.remove(name);
|
||||||
|
|
||||||
|
|
@ -910,7 +927,6 @@ impl BasisCacheEntry {
|
||||||
if let Some(dict) = self.dicts.get_mut(&String::from(name)) {
|
if let Some(dict) = self.dicts.get_mut(&String::from(name)) {
|
||||||
let dict_offset = VirtAddr::new(dict.index as u64 * DICT_VSIZE).unwrap();
|
let dict_offset = VirtAddr::new(dict.index as u64 * DICT_VSIZE).unwrap();
|
||||||
if !dict.clean {
|
if !dict.clean {
|
||||||
log::info!("syncing dictionary {}", name);
|
|
||||||
let mut dict_name = [0u8; DICT_NAME_LEN];
|
let mut dict_name = [0u8; DICT_NAME_LEN];
|
||||||
for (src, dst) in name.bytes().into_iter().zip(dict_name.iter_mut()) {
|
for (src, dst) in name.bytes().into_iter().zip(dict_name.iter_mut()) {
|
||||||
*dst = src;
|
*dst = src;
|
||||||
|
|
@ -922,7 +938,7 @@ impl BasisCacheEntry {
|
||||||
free_key_index: dict.last_disk_key_index,
|
free_key_index: dict.last_disk_key_index,
|
||||||
name: dict_name,
|
name: dict_name,
|
||||||
};
|
};
|
||||||
log::info!("syncing dict: {:?}", dict_disk);
|
log::info!("syncing dict {}: {:?}", name, dict_disk);
|
||||||
// log::info!("raw: {:x?}", dict_disk.deref());
|
// log::info!("raw: {:x?}", dict_disk.deref());
|
||||||
// observation: all keys to be flushed to disk will be in the KeyCacheEntry. Some may be clean,
|
// observation: all keys to be flushed to disk will be in the KeyCacheEntry. Some may be clean,
|
||||||
// but definitely all the dirty ones are in there (if they aren't, where else would they be??)
|
// but definitely all the dirty ones are in there (if they aren't, where else would they be??)
|
||||||
|
|
|
||||||
|
|
@ -167,6 +167,7 @@ impl DictCacheEntry {
|
||||||
}
|
}
|
||||||
/// Populates cache entries, reporting the maximum extent of large alloc data seen so far.
|
/// Populates cache entries, reporting the maximum extent of large alloc data seen so far.
|
||||||
/// Requires a descriptor for the hardware, and our virtual to physical page mapping.
|
/// Requires a descriptor for the hardware, and our virtual to physical page mapping.
|
||||||
|
/// Does not overwrite existing cache entries, if they already exist -- only loads in ones that are missing.
|
||||||
/// Todo: condense routines in common with ensure_key_entry() to make it easier to maintain.
|
/// Todo: condense routines in common with ensure_key_entry() to make it easier to maintain.
|
||||||
pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv) -> VirtAddr {
|
pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv) -> VirtAddr {
|
||||||
let mut try_entry = 1;
|
let mut try_entry = 1;
|
||||||
|
|
@ -209,15 +210,19 @@ impl DictCacheEntry {
|
||||||
data: None,
|
data: None,
|
||||||
};
|
};
|
||||||
let kname = cstr_to_string(&keydesc.name);
|
let kname = cstr_to_string(&keydesc.name);
|
||||||
if keydesc.start + keydesc.reserved > alloc_top.get() {
|
if !self.keys.contains_key(&kname) {
|
||||||
// if the key is within the large pool space, note its allocation for the basis overall
|
if keydesc.start + keydesc.reserved > alloc_top.get() {
|
||||||
alloc_top = VirtAddr::new(keydesc.start + keydesc.reserved).unwrap();
|
// if the key is within the large pool space, note its allocation for the basis overall
|
||||||
// nothing else needs to be done -- we don't pre-cache large key data.
|
alloc_top = VirtAddr::new(keydesc.start + keydesc.reserved).unwrap();
|
||||||
|
// nothing else needs to be done -- we don't pre-cache large key data.
|
||||||
|
} else {
|
||||||
|
// try to fill the small key cache entry details
|
||||||
|
self.try_fill_small_key(hw, v2p_map, cipher, &mut data_cache, &mut kcache, &kname);
|
||||||
|
}
|
||||||
|
self.keys.insert(kname, kcache);
|
||||||
} else {
|
} else {
|
||||||
// try to fill the small key cache entry details
|
log::info!("fill: entry already present {}", kname);
|
||||||
self.try_fill_small_key(hw, v2p_map, cipher, &mut data_cache, &mut kcache, &kname);
|
|
||||||
}
|
}
|
||||||
self.keys.insert(kname, kcache);
|
|
||||||
key_count += 1;
|
key_count += 1;
|
||||||
}
|
}
|
||||||
try_entry += 1;
|
try_entry += 1;
|
||||||
|
|
|
||||||
|
|
@ -189,6 +189,7 @@ impl PddbOs {
|
||||||
pub fn dbg_dump(&self, _name: Option<String>) {
|
pub fn dbg_dump(&self, _name: Option<String>) {
|
||||||
// placeholder
|
// placeholder
|
||||||
}
|
}
|
||||||
|
#[allow(dead_code)]
|
||||||
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
||||||
/// used to reset the hardware structure for repeated runs of testing within a single invocation
|
/// used to reset the hardware structure for repeated runs of testing within a single invocation
|
||||||
pub fn test_reset(&mut self) {
|
pub fn test_reset(&mut self) {
|
||||||
|
|
@ -859,10 +860,10 @@ impl PddbOs {
|
||||||
let cipher = self.cipher_ecb.as_ref().expect("Inconsistent internal state - syskey_ensure() failed");
|
let cipher = self.cipher_ecb.as_ref().expect("Inconsistent internal state - syskey_ensure() failed");
|
||||||
let mut update = SpaceUpdate::new(self.entropy.borrow_mut().get_u64(), ppc);
|
let mut update = SpaceUpdate::new(self.entropy.borrow_mut().get_u64(), ppc);
|
||||||
let mut block = Block::from_mut_slice(update.deref_mut());
|
let mut block = Block::from_mut_slice(update.deref_mut());
|
||||||
log::info!("block: {:x?}", block);
|
log::trace!("block: {:x?}", block);
|
||||||
cipher.encrypt_block(&mut block);
|
cipher.encrypt_block(&mut block);
|
||||||
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
||||||
log::info!("patch: {:x?}", block);
|
log::trace!("patch: {:x?}", block);
|
||||||
self.patch_fscb(&block, log_addr);
|
self.patch_fscb(&block, log_addr);
|
||||||
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
||||||
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
||||||
|
|
@ -894,10 +895,10 @@ impl PddbOs {
|
||||||
let cipher = self.cipher_ecb.as_ref().expect("Inconsistent internal state - syskey_ensure() failed");
|
let cipher = self.cipher_ecb.as_ref().expect("Inconsistent internal state - syskey_ensure() failed");
|
||||||
let mut update = SpaceUpdate::new(self.entropy.borrow_mut().get_u64(), pp);
|
let mut update = SpaceUpdate::new(self.entropy.borrow_mut().get_u64(), pp);
|
||||||
let mut block = Block::from_mut_slice(update.deref_mut());
|
let mut block = Block::from_mut_slice(update.deref_mut());
|
||||||
log::info!("block: {:x?}", block);
|
log::trace!("block: {:x?}", block);
|
||||||
cipher.encrypt_block(&mut block);
|
cipher.encrypt_block(&mut block);
|
||||||
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
let log_addr = self.fspace_log_next_addr.take().unwrap() as PhysAddr;
|
||||||
log::info!("patch: {:x?}", block);
|
log::trace!("patch: {:x?}", block);
|
||||||
self.patch_fscb(&block, log_addr);
|
self.patch_fscb(&block, log_addr);
|
||||||
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
let next_addr = log_addr + aes::BLOCK_SIZE as PhysAddr;
|
||||||
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
if (next_addr & (PAGE_SIZE as PhysAddr - 1)) != 0 {
|
||||||
|
|
|
||||||
|
|
@ -357,7 +357,7 @@ const PDDB_BACKING_SIZE: usize = 0x4000_0000;
|
||||||
#[xous::xous_main]
|
#[xous::xous_main]
|
||||||
fn xmain() -> ! {
|
fn xmain() -> ! {
|
||||||
log_server::init_wait().unwrap();
|
log_server::init_wait().unwrap();
|
||||||
log::set_max_level(log::LevelFilter::Trace);
|
log::set_max_level(log::LevelFilter::Debug);
|
||||||
log::info!("my PID is {}", xous::process::id());
|
log::info!("my PID is {}", xous::process::id());
|
||||||
|
|
||||||
let xns = xous_names::XousNames::new().unwrap();
|
let xns = xous_names::XousNames::new().unwrap();
|
||||||
|
|
@ -375,9 +375,18 @@ fn xmain() -> ! {
|
||||||
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
||||||
{
|
{
|
||||||
log::info!("Creating `basecase1`");
|
log::info!("Creating `basecase1`");
|
||||||
create_testcase(&mut pddb_os, None, None, None);
|
let mut basis_cache = BasisCache::new();
|
||||||
|
create_basis_testcase(&mut pddb_os, &mut basis_cache, None, None, None);
|
||||||
|
log::info!("Saving `basecase1` to local host");
|
||||||
pddb_os.dbg_dump(Some("basecase1".to_string()));
|
pddb_os.dbg_dump(Some("basecase1".to_string()));
|
||||||
|
|
||||||
|
log::info!("Doing delete/add consistency");
|
||||||
|
delete_add_dict_consistency(&mut pddb_os, &mut basis_cache, None, None, None);
|
||||||
|
log::info!("Saving `dacheck` to local host");
|
||||||
|
pddb_os.dbg_dump(Some("dacheck".to_string()));
|
||||||
}
|
}
|
||||||
|
log::info!("CI done");
|
||||||
|
/*
|
||||||
{ // a simple case that could be run directly on the hardware
|
{ // a simple case that could be run directly on the hardware
|
||||||
log::info!("Running `manual` test case");
|
log::info!("Running `manual` test case");
|
||||||
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
||||||
|
|
@ -408,6 +417,7 @@ fn xmain() -> ! {
|
||||||
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
#[cfg(not(any(target_os = "none", target_os = "xous")))]
|
||||||
pddb_os.dbg_dump(Some("manual".to_string()));
|
pddb_os.dbg_dump(Some("manual".to_string()));
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
/* list of test cases:
|
/* list of test cases:
|
||||||
genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
|
genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
|
||||||
delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
|
delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,40 @@
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use crate::*;
|
use crate::*;
|
||||||
|
|
||||||
pub(crate) fn create_testcase(hw: &mut PddbOs,
|
fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bound: usize) -> (String, Vec::<u8>) {
|
||||||
maybe_num_dicts: Option<usize>, maybe_num_keys: Option<usize>, maybe_key_sizes: Option<(usize, usize)>) {
|
|
||||||
let mut rng = rand::thread_rng();
|
let mut rng = rand::thread_rng();
|
||||||
|
// we want roughly half our keys to be in the small bin, and half in the large bin
|
||||||
|
let keylen = if rng.gen_bool(0.5) {
|
||||||
|
rng.gen_range(lower_size_bound..VPAGE_SIZE)
|
||||||
|
} else {
|
||||||
|
rng.gen_range(VPAGE_SIZE..upper_size_bound)
|
||||||
|
};
|
||||||
|
// record the owning dictionary name & length with the key. This isn't mandatory for a key name,
|
||||||
|
// but it helps the test checking program check things.
|
||||||
|
let keyname = format!("sanitycheck|{}|key{}|len{}", dictname, keynum, keylen);
|
||||||
|
let mut keydata = Vec::<u8>::new();
|
||||||
|
for i in 0..keylen-4 {
|
||||||
|
// the data starts with a number equal to the key number, and increments from there
|
||||||
|
keydata.push((keynum + i) as u8);
|
||||||
|
}
|
||||||
|
// a checksum is appended to each run of data
|
||||||
|
// copy the stored data, and pad it out to a multiple of word lengths with 0's so we can compute the checksum
|
||||||
|
let mut checkdata = Vec::<u8>::new();
|
||||||
|
for &b in &keydata {
|
||||||
|
checkdata.push(b);
|
||||||
|
}
|
||||||
|
while checkdata.len() % 4 != 0 {
|
||||||
|
checkdata.push(0);
|
||||||
|
}
|
||||||
|
let checksum = murmur3_32(&checkdata, 0);
|
||||||
|
keydata.append(&mut checksum.to_le_bytes().to_vec());
|
||||||
|
(keyname, keydata)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn create_basis_testcase(hw: &mut PddbOs, basis_cache: &mut BasisCache,
|
||||||
|
maybe_num_dicts: Option<usize>, maybe_num_keys: Option<usize>, maybe_key_sizes: Option<(usize, usize)>) {
|
||||||
|
|
||||||
hw.pddb_format().unwrap();
|
hw.pddb_format().unwrap();
|
||||||
let mut basis_cache = BasisCache::new();
|
|
||||||
let sys_basis = hw.pddb_mount().expect("couldn't mount system basis");
|
let sys_basis = hw.pddb_mount().expect("couldn't mount system basis");
|
||||||
basis_cache.basis_add(sys_basis);
|
basis_cache.basis_add(sys_basis);
|
||||||
|
|
||||||
|
|
@ -25,37 +53,44 @@ pub(crate) fn create_testcase(hw: &mut PddbOs,
|
||||||
for keynum in 1..=num_keys {
|
for keynum in 1..=num_keys {
|
||||||
for dictnum in 1..=num_dicts {
|
for dictnum in 1..=num_dicts {
|
||||||
let dictname = format!("dict{}", dictnum);
|
let dictname = format!("dict{}", dictnum);
|
||||||
// we want roughly half our keys to be in the small bin, and half in the large bin
|
let (keyname, keydata) = gen_key(&dictname, keynum, key_lower_bound, key_upper_bound);
|
||||||
let keylen = if rng.gen_bool(0.5) {
|
|
||||||
rng.gen_range(key_lower_bound..VPAGE_SIZE)
|
|
||||||
} else {
|
|
||||||
rng.gen_range(VPAGE_SIZE..key_upper_bound)
|
|
||||||
};
|
|
||||||
// record the owning dictionary name & length with the key. This isn't mandatory for a key name,
|
|
||||||
// but it helps the test checking program check things.
|
|
||||||
let keyname = format!("sanitycheck|{}|key{}|len{}", dictname, keynum, keylen);
|
|
||||||
let mut keydata = Vec::<u8>::new();
|
|
||||||
for i in 0..keylen-4 {
|
|
||||||
// the data starts with a number equal to the key number, and increments from there
|
|
||||||
keydata.push((keynum + i) as u8);
|
|
||||||
}
|
|
||||||
// a checksum is appended to each run of data
|
|
||||||
// copy the stored data, and pad it out to a multiple of word lengths with 0's so we can compute the checksum
|
|
||||||
let mut checkdata = Vec::<u8>::new();
|
|
||||||
for &b in &keydata {
|
|
||||||
checkdata.push(b);
|
|
||||||
}
|
|
||||||
while checkdata.len() % 4 != 0 {
|
|
||||||
checkdata.push(0);
|
|
||||||
}
|
|
||||||
let checksum = murmur3_32(&checkdata, 0);
|
|
||||||
keydata.append(&mut checksum.to_le_bytes().to_vec());
|
|
||||||
// now we're ready to write it out
|
// now we're ready to write it out
|
||||||
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None, None, None, false).unwrap();
|
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None, None, None, false).unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Delete & add dictionary consistency check
|
||||||
|
pub(crate) fn delete_add_dict_consistency(hw: &mut PddbOs, basis_cache: &mut BasisCache,
|
||||||
|
maybe_num_dicts: Option<usize>, maybe_num_keys: Option<usize>, maybe_key_sizes: Option<(usize, usize)>) {
|
||||||
|
let evict_count = maybe_num_dicts.unwrap_or(1);
|
||||||
|
let num_keys = maybe_num_keys.unwrap_or(36);
|
||||||
|
let (key_lower_bound, key_upper_bound) = maybe_key_sizes.unwrap_or((1, 9000));
|
||||||
|
|
||||||
|
let dict_list = basis_cache.dict_list(hw);
|
||||||
|
let dict_start_index = dict_list.len();
|
||||||
|
for (evicted, evict_dict) in dict_list.iter().enumerate() {
|
||||||
|
if evicted < evict_count {
|
||||||
|
match basis_cache.dict_remove(hw, evict_dict, None, false) {
|
||||||
|
Ok(_) => {},
|
||||||
|
Err(e) => log::error!("Error evicting dictionary {}: {:?}", evict_dict, e),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for keynum in 1..=num_keys {
|
||||||
|
for dictnum in 1..=evict_count {
|
||||||
|
let dictname = format!("dict{}", dictnum + dict_start_index);
|
||||||
|
let (keyname, keydata) = gen_key(&dictname, keynum, key_lower_bound, key_upper_bound);
|
||||||
|
// now we're ready to write it out
|
||||||
|
basis_cache.key_update(hw, &dictname, &keyname, &keydata, None, None, None, false).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
pub(crate) fn manual_testcase(hw: &mut PddbOs) {
|
pub(crate) fn manual_testcase(hw: &mut PddbOs) {
|
||||||
log::info!("Initializing disk...");
|
log::info!("Initializing disk...");
|
||||||
hw.pddb_format().unwrap();
|
hw.pddb_format().unwrap();
|
||||||
|
|
|
||||||
128
tools/pddbdbg.py
128
tools/pddbdbg.py
|
|
@ -115,7 +115,6 @@ def main():
|
||||||
keyfile = './tools/pddb-images/{}.key'.format(args.name)
|
keyfile = './tools/pddb-images/{}.key'.format(args.name)
|
||||||
imagefile = './tools/pddb-images/{}.bin'.format(args.name)
|
imagefile = './tools/pddb-images/{}.bin'.format(args.name)
|
||||||
|
|
||||||
|
|
||||||
keys = {}
|
keys = {}
|
||||||
with open(keyfile, 'rb') as key_f:
|
with open(keyfile, 'rb') as key_f:
|
||||||
raw_key = key_f.read()
|
raw_key = key_f.read()
|
||||||
|
|
@ -140,6 +139,7 @@ def main():
|
||||||
KEY_PAGES = 1
|
KEY_PAGES = 1
|
||||||
global PAGE_SIZE
|
global PAGE_SIZE
|
||||||
global VPAGE_SIZE
|
global VPAGE_SIZE
|
||||||
|
MAX_DICTS = 16384
|
||||||
|
|
||||||
with open(imagefile, 'rb') as img_f:
|
with open(imagefile, 'rb') as img_f:
|
||||||
raw_img = img_f.read()
|
raw_img = img_f.read()
|
||||||
|
|
@ -178,9 +178,19 @@ def main():
|
||||||
print(basis.as_str())
|
print(basis.as_str())
|
||||||
|
|
||||||
basis_dicts = {}
|
basis_dicts = {}
|
||||||
for dict_index in range(basis.num_dicts):
|
dicts_found = 0
|
||||||
|
dict_index = 0
|
||||||
|
while dict_index < MAX_DICTS and dicts_found < basis.num_dicts:
|
||||||
bdict = BasisDicts(dict_index, v2p_table, data, key, name)
|
bdict = BasisDicts(dict_index, v2p_table, data, key, name)
|
||||||
basis_dicts[bdict.name] = bdict
|
if bdict.valid:
|
||||||
|
basis_dicts[bdict.name] = bdict
|
||||||
|
dicts_found += 1
|
||||||
|
dict_index += 1
|
||||||
|
if dicts_found != basis.num_dicts:
|
||||||
|
print("Expected {} dictionaries, only found {}; searched {}".format(basis.num_dicts, dicts_found, dict_index))
|
||||||
|
found_all_dicts = False
|
||||||
|
else:
|
||||||
|
found_all_dicts = True
|
||||||
|
|
||||||
print(" Dictionaries: ")
|
print(" Dictionaries: ")
|
||||||
for bdict in basis_dicts.values():
|
for bdict in basis_dicts.values():
|
||||||
|
|
@ -189,6 +199,10 @@ def main():
|
||||||
print("CI checks:")
|
print("CI checks:")
|
||||||
for bdict in basis_dicts.values():
|
for bdict in basis_dicts.values():
|
||||||
bdict.ci_check()
|
bdict.ci_check()
|
||||||
|
if found_all_dicts:
|
||||||
|
print("All dicts were found.")
|
||||||
|
else:
|
||||||
|
print("Missing dictionaries, something is wrong.")
|
||||||
|
|
||||||
except ValueError:
|
except ValueError:
|
||||||
print("couldn't decrypt basis root vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE]))
|
print("couldn't decrypt basis root vpage @ {:x} ppage @ {:x}".format(VPAGE_SIZE, v2p_table[VPAGE_SIZE]))
|
||||||
|
|
@ -255,7 +269,7 @@ class KeyDescriptor:
|
||||||
print('checksum: {:x}, refchecksum: {:x}\n'.format(checksum, refcheck))
|
print('checksum: {:x}, refchecksum: {:x}\n'.format(checksum, refcheck))
|
||||||
|
|
||||||
|
|
||||||
def as_str(self, indent):
|
def as_str(self, indent=''):
|
||||||
PRINT_LEN = 64
|
PRINT_LEN = 64
|
||||||
desc = ''
|
desc = ''
|
||||||
desc += indent + 'Start: 0x{:x}\n'.format(self.start)
|
desc += indent + 'Start: 0x{:x}\n'.format(self.start)
|
||||||
|
|
@ -288,59 +302,63 @@ class BasisDicts:
|
||||||
global PAGE_SIZE
|
global PAGE_SIZE
|
||||||
global VPAGE_SIZE
|
global VPAGE_SIZE
|
||||||
dict_header_vaddr = self.DICT_VSTRIDE * (index + 1)
|
dict_header_vaddr = self.DICT_VSTRIDE * (index + 1)
|
||||||
pp = v2p[dict_header_vaddr]
|
if dict_header_vaddr in v2p:
|
||||||
self.keys = {}
|
self.valid = True
|
||||||
try:
|
pp = v2p[dict_header_vaddr]
|
||||||
print('dict decrypt @ pp {:x}, nonce: {}'.format(pp, disk[pp:pp+12].hex()))
|
self.keys = {}
|
||||||
cipher = AES_GCM_SIV(key, disk[pp:pp+12])
|
try:
|
||||||
pt_data = cipher.decrypt(disk[pp+12:pp+PAGE_SIZE], basis_aad(name))
|
print('dict decrypt @ pp {:x}, nonce: {}'.format(pp, disk[pp:pp+12].hex()))
|
||||||
# print('raw pt_data: {}'.format(pt_data[:127].hex()))
|
cipher = AES_GCM_SIV(key, disk[pp:pp+12])
|
||||||
i = 0
|
pt_data = cipher.decrypt(disk[pp+12:pp+PAGE_SIZE], basis_aad(name))
|
||||||
self.journal = int.from_bytes(pt_data[i:i+4], 'little')
|
# print('raw pt_data: {}'.format(pt_data[:127].hex()))
|
||||||
i += 4
|
i = 0
|
||||||
self.flags = int.from_bytes(pt_data[i:i+4], 'little')
|
self.journal = int.from_bytes(pt_data[i:i+4], 'little')
|
||||||
i += 4
|
i += 4
|
||||||
self.age = int.from_bytes(pt_data[i:i+4], 'little')
|
self.flags = int.from_bytes(pt_data[i:i+4], 'little')
|
||||||
i += 4
|
i += 4
|
||||||
self.num_keys = int.from_bytes(pt_data[i:i+4], 'little')
|
self.age = int.from_bytes(pt_data[i:i+4], 'little')
|
||||||
i += 4
|
i += 4
|
||||||
self.free_key_index = int.from_bytes(pt_data[i:i+4], 'little')
|
self.num_keys = int.from_bytes(pt_data[i:i+4], 'little')
|
||||||
i += 4
|
i += 4
|
||||||
self.name = pt_data[i:i+BasisDicts.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
|
self.free_key_index = int.from_bytes(pt_data[i:i+4], 'little')
|
||||||
i += BasisDicts.MAX_NAME_LEN
|
i += 4
|
||||||
print("dict header len: {}".format(i-4)) # subtract 4 because of the journal
|
self.name = pt_data[i:i+BasisDicts.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore')
|
||||||
except ValueError:
|
i += BasisDicts.MAX_NAME_LEN
|
||||||
print("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr]))
|
print("dict header len: {}".format(i-4)) # subtract 4 because of the journal
|
||||||
|
except ValueError:
|
||||||
|
print("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr]))
|
||||||
|
|
||||||
if self.num_keys > 0:
|
if self.num_keys > 0:
|
||||||
keys_found = 0
|
keys_found = 0
|
||||||
keys_tried = 1
|
keys_tried = 1
|
||||||
while (keys_found < self.num_keys) and keys_tried < 131071:
|
while (keys_found < self.num_keys) and keys_tried < 131071:
|
||||||
keyindex_start_vaddr = self.DICT_VSTRIDE * (index + 1) + (keys_tried * self.KV_STRIDE)
|
keyindex_start_vaddr = self.DICT_VSTRIDE * (index + 1) + (keys_tried * self.KV_STRIDE)
|
||||||
try:
|
|
||||||
pp_start = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE]
|
|
||||||
pp_data = disk[pp_start:pp_start + PAGE_SIZE]
|
|
||||||
try:
|
try:
|
||||||
cipher = AES_GCM_SIV(key, pp_data[:12])
|
pp_start = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE]
|
||||||
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
|
pp_data = disk[pp_start:pp_start + PAGE_SIZE]
|
||||||
key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE
|
try:
|
||||||
# print(key_index)
|
cipher = AES_GCM_SIV(key, pp_data[:12])
|
||||||
maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name)
|
pt_data = cipher.decrypt(pp_data[12:], basis_aad(name))
|
||||||
if maybe_key.valid:
|
key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE
|
||||||
self.keys[maybe_key.name] = maybe_key
|
# print(key_index)
|
||||||
keys_found += 1
|
maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name)
|
||||||
|
if maybe_key.valid:
|
||||||
|
self.keys[maybe_key.name] = maybe_key
|
||||||
|
keys_found += 1
|
||||||
|
|
||||||
except ValueError:
|
except ValueError:
|
||||||
print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr), pp_start)
|
print("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr), pp_start)
|
||||||
keys_tried += 1
|
keys_tried += 1
|
||||||
except KeyError:
|
except KeyError:
|
||||||
# the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried"
|
# the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried"
|
||||||
keys_tried += VPAGE_SIZE // self.KV_STRIDE
|
keys_tried += VPAGE_SIZE // self.KV_STRIDE
|
||||||
if keys_found < self.num_keys:
|
if keys_found < self.num_keys:
|
||||||
print("Expected {} keys but only found {}".format(keys_found, self.num_keys))
|
print("Expected {} keys but only found {}".format(self.num_keys, keys_found))
|
||||||
self.found_all_keys = False
|
self.found_all_keys = False
|
||||||
else:
|
else:
|
||||||
self.found_all_keys = True
|
self.found_all_keys = True
|
||||||
|
else:
|
||||||
|
self.valid = False
|
||||||
|
|
||||||
def as_str(self):
|
def as_str(self):
|
||||||
desc = ''
|
desc = ''
|
||||||
|
|
@ -367,7 +385,7 @@ class BasisDicts:
|
||||||
print(" Key length mismatch: {}/{}", keylen, key.len)
|
print(" Key length mismatch: {}/{}", keylen, key.len)
|
||||||
if key.ci_ok == False:
|
if key.ci_ok == False:
|
||||||
print(" CI failed on key:")
|
print(" CI failed on key:")
|
||||||
print(key.as_str())
|
print(key.as_str(' '))
|
||||||
check_ok = False
|
check_ok = False
|
||||||
if self.found_all_keys == False:
|
if self.found_all_keys == False:
|
||||||
print(' Dictionary was missing keys')
|
print(' Dictionary was missing keys')
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue