From c1c57edfd61b0ef81ec1fbe28abfc463a524fc96 Mon Sep 17 00:00:00 2001 From: bunnie Date: Wed, 22 Dec 2021 07:09:30 +0800 Subject: [PATCH] some mods to the logging infra to chase down a "rare" issue --- services/pddb/src/backend/basis.rs | 24 ++++++++++++------------ services/pddb/src/backend/dictionary.rs | 18 +++++++++--------- services/pddb/src/backend/hw.rs | 15 +++++++++------ services/pddb/src/main.rs | 9 +++++++-- services/pddb/src/tests.rs | 8 ++++---- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index ea3cb640c..45a5b38fd 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -329,7 +329,7 @@ impl BasisCache { small_pool_free: BinaryHeap::::new(), aad: my_aad, }; - log::info!("adding dictionary {}", name); + log::debug!("adding dictionary {}", name); basis.dicts.insert(String::from(name), dict_cache); basis.num_dicts += 1; // encrypt and write the dict entry to disk @@ -407,7 +407,7 @@ impl BasisCache { hw: &mut PddbOs, dict: &str, basis_name: Option<&str>, paranoid: bool ) -> Result<()> { if let Some(basis_index) = self.select_basis(basis_name) { - log::info!("deleting dict {}", dict); + log::debug!("deleting dict {}", dict); let basis = &mut self.cache[basis_index]; basis.age = basis.age.saturating_add(1); @@ -602,8 +602,8 @@ impl BasisCache { // it's probably going in the small pool. // index exists, see if the page exists if reserved == 0 { // something went wrong here - log::info!("reserved {}", reserved); - log::info!("{}:{} - [{}]{:x}+{}->{}", dict, key, kcache.descriptor_index, kcache.start, kcache.len, kcache.reserved); + log::debug!("reserved {}", reserved); + log::debug!("{}:{} - [{}]{:x}+{}->{}", dict, key, kcache.descriptor_index, kcache.start, kcache.len, kcache.reserved); } if dict_entry.small_pool.len() > key_index { log::trace!("resolved key index {}, small pool len: {}", key_index, dict_entry.small_pool.len()); @@ -814,7 +814,7 @@ impl BasisCache { log::error!("PDDB mount requested {}, but got {}; aborting.", name, basis_name); return None; } - log::info!("Basis {} record found, generating cache entry", name); + log::debug!("Basis {} record found, generating cache entry", name); BasisCacheEntry::mount(hw, &basis_name, &basis_key, false, policy) } else { None @@ -849,7 +849,7 @@ impl BasisCache { rpte.set_clean(true); // it's not clean _right now_ but it will be by the time this routine is done... rpte.set_valid(true); let va = VirtAddr::new((1 * VPAGE_SIZE) as u64).unwrap(); // page 1 is where the root goes, by definition - log::info!("adding basis {}: va {:x?} with pte {:?}", name, va, rpte); + log::debug!("adding basis {}: va {:x?} with pte {:?}", name, va, rpte); basis_v2p_map.insert(va, rpte); } else { return Err(Error::new(ErrorKind::Other, "Space reservation failed")); @@ -1256,11 +1256,11 @@ impl BasisCacheEntry { for (&virt, phys) in self.v2p_map.iter_mut() { if !phys.valid() { // erase the entry - log::info!("deleting pte va: {:x?} pa: {:x?}", virt, phys); + log::debug!("deleting pte va: {:x?} pa: {:x?}", virt, phys); kill_list.push(virt); hw.pt_erase(phys.page_number()); } else if !phys.clean() { - log::info!("syncing dirty pte va: {:x?} pa: {:x?}", virt, phys); + log::debug!("syncing dirty pte va: {:x?} pa: {:x?}", virt, phys); hw.pt_patch_mapping(virt, phys.page_number(), &self.cipher_ecb); phys.set_clean(true); } @@ -1297,7 +1297,7 @@ impl BasisCacheEntry { free_key_index: dict.last_disk_key_index, name: dict_name, }; - log::info!("syncing dict {} with {} keys", name, dict.key_count); + log::debug!("syncing dict {} with {} keys", name, dict.key_count); // log::info!("raw: {:x?}", dict_disk.deref()); // observation: all keys to be flushed to disk will be in the KeyCacheEntry. Some may be clean, // but definitely all the dirty ones are in there (if they aren't, where else would they be??) @@ -1342,7 +1342,7 @@ impl BasisCacheEntry { if !key.clean && key.flags.valid() { if key.descriptor_vaddr(dict_offset) >= cur_vpage && key.descriptor_vaddr(dict_offset) < next_vpage { - log::info!("merging in key {}", key_name); + log::debug!("merging in key {}", key_name); // key is within the current page, add it to the target list let mut dk_entry = DictKeyEntry::default(); let mut kn = [0u8; KEY_NAME_LEN]; @@ -1402,7 +1402,7 @@ impl BasisCacheEntry { break; } } - log::info!("done syncing dict"); + log::debug!("done syncing dict"); dict.clean = true; } Ok(()) @@ -1455,7 +1455,7 @@ impl BasisCacheEntry { false }; if !dict_in_cache_and_valid { - log::info!("dict: key not in cache {}", name); + log::debug!("dict: key not in cache {}", name); // if the dictionary doesn't exist in our cache it doesn't necessarily mean it // doesn't exist. Do a comprehensive search if our cache isn't complete. if let Some((index, dict_record)) = self.dict_deep_search(hw, name) { diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 132234565..69daa99af 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -208,7 +208,7 @@ impl DictCacheEntry { name_str: &str) -> bool { // only fill if the key isn't in the cache. if !self.keys.contains_key(name_str) { - log::info!("searching for key {}", name_str); + log::debug!("searching for key {}", name_str); let mut data_cache = PlaintextCache { data: None, tag: None }; let mut try_entry = 1; let mut key_count = 0; @@ -315,7 +315,7 @@ impl DictCacheEntry { log::error!("Key {}'s data region at pp: {:x?} va: {:x} is unreadable", key_name, data_cache.tag, kcache.start); } } else { - log::info!("pool is clean for key {} / index {}, skipping fill", key_name, pool_index); + log::debug!("pool is clean for key {} / index {}, skipping fill", key_name, pool_index); } } } else { @@ -368,11 +368,11 @@ impl DictCacheEntry { update_data.push(0); } for (&src, dst) in data.iter().zip(update_data[offset..].iter_mut()) { *dst = src }; - log::info!("update/extend: removing {}", name); + log::debug!("update/extend: removing {}", name); // now remove the old key entirely self.key_remove(hw, v2p_map, cipher, name, false); if update_data.len() > 4 { // just make sure that this log call doesn't fail on an index violation... - log::info!("update/extend: re-adding {} with data len {}: {:x?}...", name, update_data.len(), &update_data[..4]); + log::debug!("update/extend: re-adding {} with data len {}: {:x?}...", name, update_data.len(), &update_data[..4]); } // and re-add it with the extended data; if it's no longer a small key after this, it'll be handled inside this call. return self.key_update(hw, v2p_map, cipher, name, &update_data, 0, alloc_hint, truncate, large_alloc_ptr); @@ -403,7 +403,7 @@ impl DictCacheEntry { } // the key exists, *and* there's sufficient space for the data if kcache.start < SMALL_POOL_END { - log::info!("doing data update"); + log::debug!("doing data update"); kcache.age = kcache.age.saturating_add(1); kcache.clean = false; // it's a small key; note that we didn't consult the *size* of the key to determine its pool type: @@ -534,7 +534,7 @@ impl DictCacheEntry { // key does not exist (or was previously erased) -- create one or replace the erased one. // try to fit the key in the small pool first if ((data.len() + offset) < SMALL_CAPACITY) && (alloc_hint.unwrap_or(0) < SMALL_CAPACITY) { - log::info!("creating small key"); + log::debug!("creating small key"); // handle the case that we're a brand new dictionary and no small keys have ever been stored before. if self.small_pool.len() == 0 { self.small_pool.push(KeySmallPool::new()); @@ -603,7 +603,7 @@ impl DictCacheEntry { self.keys.insert(name.to_string(), kcache); self.key_count += 1; } else { - log::info!("creating large key"); + log::debug!("creating large key"); // it didn't fit in the small pool, stick it in the big pool. let reservation = PageAlignedVa::from( if alloc_hint.unwrap_or(0) > data.len() + offset { @@ -715,7 +715,7 @@ impl DictCacheEntry { if let Some(key_to_free) = need_free_key { self.put_free_key_index(key_to_free); self.key_count -= 1; - log::info!("key_count: {}", self.key_count); + log::debug!("key_count: {}", self.key_count); } if need_rebuild { // no stable "retain" api, so we have to clear the heap and rebuild it https://github.com/rust-lang/rust/issues/71503 @@ -797,7 +797,7 @@ impl DictCacheEntry { for key_name in &entry.contents { let kcache = self.keys.get_mut(key_name).expect("data record without index"); if kcache.flags.valid() { // only sync valid keys, not ones that were marked for deletion but not yet synced. - log::info!("sync {}/{}:0x{:x}..{}->{}", key_name, index, kcache.start, kcache.len, kcache.reserved); + log::debug!("sync {}/{}:0x{:x}..{}->{}", key_name, index, kcache.start, kcache.len, kcache.reserved); kcache.start = pool_vaddr.get() + pool_offset as u64; kcache.age = kcache.age.saturating_add(1); kcache.clean = false; diff --git a/services/pddb/src/backend/hw.rs b/services/pddb/src/backend/hw.rs index f114b0b76..d123c2767 100644 --- a/services/pddb/src/backend/hw.rs +++ b/services/pddb/src/backend/hw.rs @@ -606,10 +606,11 @@ impl PddbOs { noise = -noise; } let deniable_free_pages = (total_free_pages as f32 * (FSCB_FILL_COEFFICIENT + noise)) as usize; - // we're guarantede to have at least one free page, because we errored out if the pages was 0 above. + // we're guaranteed to have at least one free page, because we errored out if the pages was 0 above. let deniable_free_pages = if deniable_free_pages == 0 { 1 } else { deniable_free_pages }; free_pool.truncate(deniable_free_pages); - log::info!("free_pool after PD trim: {}; max pages allowed: {}", free_pool.len(), deniable_free_pages); + log::warn!("total_free: {}; free_pool after PD trim: {}; max pages allowed: {}", + total_free_pages, free_pool.len(), deniable_free_pages); // 5. Take the free_pool and annotate it for writing to disk let mut page_pool = Vec::::new(); @@ -738,7 +739,7 @@ impl PddbOs { } cipher.decrypt_block(&mut block); if let Some(pp) = SpaceUpdate::try_into_phys_page(block.as_slice()) { - log::info!("maybe replacing fspace block: {:x?}", pp); + log::debug!("maybe replacing fspace block: {:x?}", pp); // note: pp.valid() isn't the cryptographic check, the cryptographic check of record validity is in try_into_phys_page() if pp.valid() { // PS: it should always be valid! if let Some(prev_pp) = self.fspace_cache.get(&pp) { @@ -749,7 +750,7 @@ impl PddbOs { panic!("Inconsistent FSCB state"); } } else { - log::info!("Strange...we have a journal entry for a free space page that isn't already in our cache. Guru meditation: {:?}", pp); + log::warn!("Strange...we have a journal entry for a free space page that isn't already in our cache. Guru meditation: {:?}", pp); self.fspace_cache.insert(pp); } } @@ -809,6 +810,7 @@ impl PddbOs { } if is_blank { self.fspace_log_next_addr = Some( (page_start + (1 + index) * aes::BLOCK_SIZE) as PhysAddr ); + log::info!("Next FSCB entry: {:x?}", self.fspace_log_next_addr); return true } } @@ -823,6 +825,7 @@ impl PddbOs { let random_index = self.entropy.borrow_mut().get_u32() as usize % blank_pages.len(); // set the next log address at an offset of one AES block in from the top. self.fspace_log_next_addr = Some((blank_pages[random_index] + aes::BLOCK_SIZE) as PhysAddr); + log::warn!("Moving log to fresh FSCB page: {:x?}", self.fspace_log_next_addr); true } else { false @@ -933,7 +936,7 @@ impl PddbOs { true } else { if self.fast_space_len() <= pages + BUFFER { - log::trace!("alloc forced by lack of free space"); + log::warn!("FastSpace alloc forced by lack of free space"); // if we're really out of space, do an expensive full-space sweep if let Some(used_pages) = self.pddb_generate_used_map(cache) { let free_pool = self.fast_space_generate(used_pages); @@ -966,7 +969,7 @@ impl PddbOs { } else { // log regenration is faster & less intrusive than fastspace regeneration, and we would have // to do this more often. So we have a separate path for this outcome. - log::trace!("alloc forced by lack of log space"); + log::warn!("FastSpace alloc forced by lack of log space"); let mut fast_space = FastSpace { free_pool: [PhysPage(0); FASTSPACE_FREE_POOL_LEN], }; diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 3b1635e41..fc6ed8f20 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -436,6 +436,8 @@ fn xmain() -> ! { const EXTRA_BASIS: &'static str = "Basis2"; const EXTRA_BASIS_PW: &'static str = "some password blah blah"; + log::set_max_level(log::LevelFilter::Info); + pddb_os.test_reset(); log::info!("Creating `basecase1e`"); let mut basis_cache = BasisCache::new(); @@ -466,10 +468,12 @@ fn xmain() -> ! { patch_test(&mut pddb_os, &mut basis_cache, None, None, true); pddb_os.dbg_dump(Some("patche".to_string()), None); + log::set_max_level(log::LevelFilter::Debug); log::info!("Doing delete pattern test"); delete_pattern(&mut pddb_os, &mut basis_cache, None, None, None, None); pddb_os.dbg_dump(Some("patterne".to_string()), None); -/* + log::set_max_level(log::LevelFilter::Info); + // extended tests. // allocation space curtailed to force resource exhaustion faster. // note to self: FSCB_PAGES revert to 16 (hw.rs), FASTSPACE_PAGES revert to 2 (fastspace.rs) @@ -498,7 +502,7 @@ fn xmain() -> ! { Some(50), None, None, None); log::info!("Saving `dachecke4` to local host"); pddb_os.dbg_dump(Some("dachecke4".to_string()), None); -*/ + let mut pre_list = HashSet::::new(); for dict in basis_cache.dict_list(&mut pddb_os, None).iter() { for key in basis_cache.key_list(&mut pddb_os, dict, None).unwrap().iter() { @@ -577,6 +581,7 @@ fn xmain() -> ! { list_all(&mut pddb_os, &mut basis_cache); } + log::set_max_level(log::LevelFilter::Debug); log::info!("CI done"); /* { // a simple case that could be run directly on the hardware diff --git a/services/pddb/src/tests.rs b/services/pddb/src/tests.rs index bdcaadd43..1b27552c4 100644 --- a/services/pddb/src/tests.rs +++ b/services/pddb/src/tests.rs @@ -205,7 +205,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, } for dict in basis_cache.dict_list(hw, None).iter() { let da = basis_cache.dict_attributes(hw, dict, None).unwrap(); - log::info!("{:?}", da); + log::debug!("{:?}", da); } for keynum in 1..=num_keys { @@ -226,7 +226,7 @@ pub(crate) fn delete_pattern(hw: &mut PddbOs, basis_cache: &mut BasisCache, } for dict in basis_cache.dict_list(hw, None).iter() { let da = basis_cache.dict_attributes(hw, dict, None).unwrap(); - log::info!("{:?}", da); + log::debug!("{:?}", da); } log::info!("doing all-basis sync"); @@ -250,14 +250,14 @@ pub(crate) fn list_all(hw: &mut PddbOs, basis_cache: &mut BasisCache) { continue; } }; - log::info!("{:?}", da); + log::debug!("{:?}", da); let mut sanity_count = 0; for key in basis_cache.key_list(hw, dict, None).unwrap().iter() { let attrs = match basis_cache.key_attributes(hw, dict, key, None) { Ok(a) => a, Err(e) => {log::debug!("key not in basis, searching another basis, {:?}", e); continue}, }; - log::info!("{}:{}=>{:?}", dict, key, attrs); + log::debug!("{}:{}=>{:?}", dict, key, attrs); sanity_count += 1; } log::info!("sanity check count: {}", sanity_count);