some mods to the logging infra to chase down a "rare" issue

This commit is contained in:
bunnie 2021-12-22 07:09:30 +08:00
parent 5d226a002a
commit c1c57edfd6
5 changed files with 41 additions and 33 deletions

View file

@ -329,7 +329,7 @@ impl BasisCache {
small_pool_free: BinaryHeap::<KeySmallPoolOrd>::new(),
aad: my_aad,
};
log::info!("adding dictionary {}", name);
log::debug!("adding dictionary {}", name);
basis.dicts.insert(String::from(name), dict_cache);
basis.num_dicts += 1;
// encrypt and write the dict entry to disk
@ -407,7 +407,7 @@ impl BasisCache {
hw: &mut PddbOs, dict: &str, basis_name: Option<&str>, paranoid: bool
) -> Result<()> {
if let Some(basis_index) = self.select_basis(basis_name) {
log::info!("deleting dict {}", dict);
log::debug!("deleting dict {}", dict);
let basis = &mut self.cache[basis_index];
basis.age = basis.age.saturating_add(1);
@ -602,8 +602,8 @@ impl BasisCache {
// it's probably going in the small pool.
// index exists, see if the page exists
if reserved == 0 { // something went wrong here
log::info!("reserved {}", reserved);
log::info!("{}:{} - [{}]{:x}+{}->{}", dict, key, kcache.descriptor_index, kcache.start, kcache.len, kcache.reserved);
log::debug!("reserved {}", reserved);
log::debug!("{}:{} - [{}]{:x}+{}->{}", dict, key, kcache.descriptor_index, kcache.start, kcache.len, kcache.reserved);
}
if dict_entry.small_pool.len() > key_index {
log::trace!("resolved key index {}, small pool len: {}", key_index, dict_entry.small_pool.len());
@ -814,7 +814,7 @@ impl BasisCache {
log::error!("PDDB mount requested {}, but got {}; aborting.", name, basis_name);
return None;
}
log::info!("Basis {} record found, generating cache entry", name);
log::debug!("Basis {} record found, generating cache entry", name);
BasisCacheEntry::mount(hw, &basis_name, &basis_key, false, policy)
} else {
None
@ -849,7 +849,7 @@ impl BasisCache {
rpte.set_clean(true); // it's not clean _right now_ but it will be by the time this routine is done...
rpte.set_valid(true);
let va = VirtAddr::new((1 * VPAGE_SIZE) as u64).unwrap(); // page 1 is where the root goes, by definition
log::info!("adding basis {}: va {:x?} with pte {:?}", name, va, rpte);
log::debug!("adding basis {}: va {:x?} with pte {:?}", name, va, rpte);
basis_v2p_map.insert(va, rpte);
} else {
return Err(Error::new(ErrorKind::Other, "Space reservation failed"));
@ -1256,11 +1256,11 @@ impl BasisCacheEntry {
for (&virt, phys) in self.v2p_map.iter_mut() {
if !phys.valid() {
// erase the entry
log::info!("deleting pte va: {:x?} pa: {:x?}", virt, phys);
log::debug!("deleting pte va: {:x?} pa: {:x?}", virt, phys);
kill_list.push(virt);
hw.pt_erase(phys.page_number());
} else if !phys.clean() {
log::info!("syncing dirty pte va: {:x?} pa: {:x?}", virt, phys);
log::debug!("syncing dirty pte va: {:x?} pa: {:x?}", virt, phys);
hw.pt_patch_mapping(virt, phys.page_number(), &self.cipher_ecb);
phys.set_clean(true);
}
@ -1297,7 +1297,7 @@ impl BasisCacheEntry {
free_key_index: dict.last_disk_key_index,
name: dict_name,
};
log::info!("syncing dict {} with {} keys", name, dict.key_count);
log::debug!("syncing dict {} with {} keys", name, dict.key_count);
// log::info!("raw: {:x?}", dict_disk.deref());
// observation: all keys to be flushed to disk will be in the KeyCacheEntry. Some may be clean,
// but definitely all the dirty ones are in there (if they aren't, where else would they be??)
@ -1342,7 +1342,7 @@ impl BasisCacheEntry {
if !key.clean && key.flags.valid() {
if key.descriptor_vaddr(dict_offset) >= cur_vpage &&
key.descriptor_vaddr(dict_offset) < next_vpage {
log::info!("merging in key {}", key_name);
log::debug!("merging in key {}", key_name);
// key is within the current page, add it to the target list
let mut dk_entry = DictKeyEntry::default();
let mut kn = [0u8; KEY_NAME_LEN];
@ -1402,7 +1402,7 @@ impl BasisCacheEntry {
break;
}
}
log::info!("done syncing dict");
log::debug!("done syncing dict");
dict.clean = true;
}
Ok(())
@ -1455,7 +1455,7 @@ impl BasisCacheEntry {
false
};
if !dict_in_cache_and_valid {
log::info!("dict: key not in cache {}", name);
log::debug!("dict: key not in cache {}", name);
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
// doesn't exist. Do a comprehensive search if our cache isn't complete.
if let Some((index, dict_record)) = self.dict_deep_search(hw, name) {

View file

@ -208,7 +208,7 @@ impl DictCacheEntry {
name_str: &str) -> bool {
// only fill if the key isn't in the cache.
if !self.keys.contains_key(name_str) {
log::info!("searching for key {}", name_str);
log::debug!("searching for key {}", name_str);
let mut data_cache = PlaintextCache { data: None, tag: None };
let mut try_entry = 1;
let mut key_count = 0;
@ -315,7 +315,7 @@ impl DictCacheEntry {
log::error!("Key {}'s data region at pp: {:x?} va: {:x} is unreadable", key_name, data_cache.tag, kcache.start);
}
} else {
log::info!("pool is clean for key {} / index {}, skipping fill", key_name, pool_index);
log::debug!("pool is clean for key {} / index {}, skipping fill", key_name, pool_index);
}
}
} else {
@ -368,11 +368,11 @@ impl DictCacheEntry {
update_data.push(0);
}
for (&src, dst) in data.iter().zip(update_data[offset..].iter_mut()) { *dst = src };
log::info!("update/extend: removing {}", name);
log::debug!("update/extend: removing {}", name);
// now remove the old key entirely
self.key_remove(hw, v2p_map, cipher, name, false);
if update_data.len() > 4 { // just make sure that this log call doesn't fail on an index violation...
log::info!("update/extend: re-adding {} with data len {}: {:x?}...", name, update_data.len(), &update_data[..4]);
log::debug!("update/extend: re-adding {} with data len {}: {:x?}...", name, update_data.len(), &update_data[..4]);
}
// and re-add it with the extended data; if it's no longer a small key after this, it'll be handled inside this call.
return self.key_update(hw, v2p_map, cipher, name, &update_data, 0, alloc_hint, truncate, large_alloc_ptr);
@ -403,7 +403,7 @@ impl DictCacheEntry {
}
// the key exists, *and* there's sufficient space for the data
if kcache.start < SMALL_POOL_END {
log::info!("doing data update");
log::debug!("doing data update");
kcache.age = kcache.age.saturating_add(1);
kcache.clean = false;
// it's a small key; note that we didn't consult the *size* of the key to determine its pool type:
@ -534,7 +534,7 @@ impl DictCacheEntry {
// key does not exist (or was previously erased) -- create one or replace the erased one.
// try to fit the key in the small pool first
if ((data.len() + offset) < SMALL_CAPACITY) && (alloc_hint.unwrap_or(0) < SMALL_CAPACITY) {
log::info!("creating small key");
log::debug!("creating small key");
// handle the case that we're a brand new dictionary and no small keys have ever been stored before.
if self.small_pool.len() == 0 {
self.small_pool.push(KeySmallPool::new());
@ -603,7 +603,7 @@ impl DictCacheEntry {
self.keys.insert(name.to_string(), kcache);
self.key_count += 1;
} else {
log::info!("creating large key");
log::debug!("creating large key");
// it didn't fit in the small pool, stick it in the big pool.
let reservation = PageAlignedVa::from(
if alloc_hint.unwrap_or(0) > data.len() + offset {
@ -715,7 +715,7 @@ impl DictCacheEntry {
if let Some(key_to_free) = need_free_key {
self.put_free_key_index(key_to_free);
self.key_count -= 1;
log::info!("key_count: {}", self.key_count);
log::debug!("key_count: {}", self.key_count);
}
if need_rebuild {
// no stable "retain" api, so we have to clear the heap and rebuild it https://github.com/rust-lang/rust/issues/71503
@ -797,7 +797,7 @@ impl DictCacheEntry {
for key_name in &entry.contents {
let kcache = self.keys.get_mut(key_name).expect("data record without index");
if kcache.flags.valid() { // only sync valid keys, not ones that were marked for deletion but not yet synced.
log::info!("sync {}/{}:0x{:x}..{}->{}", key_name, index, kcache.start, kcache.len, kcache.reserved);
log::debug!("sync {}/{}:0x{:x}..{}->{}", key_name, index, kcache.start, kcache.len, kcache.reserved);
kcache.start = pool_vaddr.get() + pool_offset as u64;
kcache.age = kcache.age.saturating_add(1);
kcache.clean = false;

View file

@ -606,10 +606,11 @@ impl PddbOs {
noise = -noise;
}
let deniable_free_pages = (total_free_pages as f32 * (FSCB_FILL_COEFFICIENT + noise)) as usize;
// we're guarantede to have at least one free page, because we errored out if the pages was 0 above.
// we're guaranteed to have at least one free page, because we errored out if the pages was 0 above.
let deniable_free_pages = if deniable_free_pages == 0 { 1 } else { deniable_free_pages };
free_pool.truncate(deniable_free_pages);
log::info!("free_pool after PD trim: {}; max pages allowed: {}", free_pool.len(), deniable_free_pages);
log::warn!("total_free: {}; free_pool after PD trim: {}; max pages allowed: {}",
total_free_pages, free_pool.len(), deniable_free_pages);
// 5. Take the free_pool and annotate it for writing to disk
let mut page_pool = Vec::<PhysPage>::new();
@ -738,7 +739,7 @@ impl PddbOs {
}
cipher.decrypt_block(&mut block);
if let Some(pp) = SpaceUpdate::try_into_phys_page(block.as_slice()) {
log::info!("maybe replacing fspace block: {:x?}", pp);
log::debug!("maybe replacing fspace block: {:x?}", pp);
// note: pp.valid() isn't the cryptographic check, the cryptographic check of record validity is in try_into_phys_page()
if pp.valid() { // PS: it should always be valid!
if let Some(prev_pp) = self.fspace_cache.get(&pp) {
@ -749,7 +750,7 @@ impl PddbOs {
panic!("Inconsistent FSCB state");
}
} else {
log::info!("Strange...we have a journal entry for a free space page that isn't already in our cache. Guru meditation: {:?}", pp);
log::warn!("Strange...we have a journal entry for a free space page that isn't already in our cache. Guru meditation: {:?}", pp);
self.fspace_cache.insert(pp);
}
}
@ -809,6 +810,7 @@ impl PddbOs {
}
if is_blank {
self.fspace_log_next_addr = Some( (page_start + (1 + index) * aes::BLOCK_SIZE) as PhysAddr );
log::info!("Next FSCB entry: {:x?}", self.fspace_log_next_addr);
return true
}
}
@ -823,6 +825,7 @@ impl PddbOs {
let random_index = self.entropy.borrow_mut().get_u32() as usize % blank_pages.len();
// set the next log address at an offset of one AES block in from the top.
self.fspace_log_next_addr = Some((blank_pages[random_index] + aes::BLOCK_SIZE) as PhysAddr);
log::warn!("Moving log to fresh FSCB page: {:x?}", self.fspace_log_next_addr);
true
} else {
false
@ -933,7 +936,7 @@ impl PddbOs {
true
} else {
if self.fast_space_len() <= pages + BUFFER {
log::trace!("alloc forced by lack of free space");
log::warn!("FastSpace alloc forced by lack of free space");
// if we're really out of space, do an expensive full-space sweep
if let Some(used_pages) = self.pddb_generate_used_map(cache) {
let free_pool = self.fast_space_generate(used_pages);
@ -966,7 +969,7 @@ impl PddbOs {
} else {
// log regenration is faster & less intrusive than fastspace regeneration, and we would have
// to do this more often. So we have a separate path for this outcome.
log::trace!("alloc forced by lack of log space");
log::warn!("FastSpace alloc forced by lack of log space");
let mut fast_space = FastSpace {
free_pool: [PhysPage(0); FASTSPACE_FREE_POOL_LEN],
};

View file

@ -436,6 +436,8 @@ fn xmain() -> ! {
const EXTRA_BASIS: &'static str = "Basis2";
const EXTRA_BASIS_PW: &'static str = "some password blah blah";
log::set_max_level(log::LevelFilter::Info);
pddb_os.test_reset();
log::info!("Creating `basecase1e`");
let mut basis_cache = BasisCache::new();
@ -466,10 +468,12 @@ fn xmain() -> ! {
patch_test(&mut pddb_os, &mut basis_cache, None, None, true);
pddb_os.dbg_dump(Some("patche".to_string()), None);
log::set_max_level(log::LevelFilter::Debug);
log::info!("Doing delete pattern test");
delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None);
pddb_os.dbg_dump(Some("patterne".to_string()), None);
/*
log::set_max_level(log::LevelFilter::Info);
// extended tests.
// allocation space curtailed to force resource exhaustion faster.
// note to self: FSCB_PAGES revert to 16 (hw.rs), FASTSPACE_PAGES revert to 2 (fastspace.rs)
@ -498,7 +502,7 @@ fn xmain() -> ! {
Some(50), None, None, None);
log::info!("Saving `dachecke4` to local host");
pddb_os.dbg_dump(Some("dachecke4".to_string()), None);
*/
let mut pre_list = HashSet::<String>::new();
for dict in basis_cache.dict_list(&mut pddb_os, None).iter() {
for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() {
@ -577,6 +581,7 @@ fn xmain() -> ! {
list_all(&mut pddb_os, &mut basis_cache);
}
log::set_max_level(log::LevelFilter::Debug);
log::info!("CI done");
/*
{ // a simple case that could be run directly on the hardware

View file

@ -205,7 +205,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache,
}
for dict in basis_cache.dict_list(hw, None).iter() {
let da = basis_cache.dict_attributes(hw, dict, None).unwrap();
log::info!("{:?}", da);
log::debug!("{:?}", da);
}
for keynum in 1..=num_keys {
@ -226,7 +226,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache,
}
for dict in basis_cache.dict_list(hw, None).iter() {
let da = basis_cache.dict_attributes(hw, dict, None).unwrap();
log::info!("{:?}", da);
log::debug!("{:?}", da);
}
log::info!("doing all-basis sync");
@ -250,14 +250,14 @@ pub(crate) fn list_all(hw: &mut PddbOs, basis_cache: &mut BasisCache) {
continue;
}
};
log::info!("{:?}", da);
log::debug!("{:?}", da);
let mut sanity_count = 0;
for key in basis_cache.key_list(hw, dict, None).unwrap().iter() {
let attrs = match basis_cache.key_attributes(hw, dict, key, None) {
Ok(a) => a,
Err(e) => {log::debug!("key not in basis, searching another basis, {:?}", e); continue},
};
log::info!("{}:{}=>{:?}", dict, key, attrs);
log::debug!("{}:{}=>{:?}", dict, key, attrs);
sanity_count += 1;
}
log::info!("sanity check count: {}", sanity_count);