slow but steady progress toward implemeting key allocation and access

This commit is contained in:
bunnie 2021-11-19 04:00:33 +08:00
parent 14f4beef98
commit 6da984e51e
4 changed files with 536 additions and 107 deletions

View file

@ -6,10 +6,10 @@ use std::rc::Rc;
use core::ops::{Deref, DerefMut}; use core::ops::{Deref, DerefMut};
use core::mem::size_of; use core::mem::size_of;
use std::convert::TryInto; use std::convert::TryInto;
use aes_gcm_siv::{Aes256GcmSiv, Nonce}; use aes_gcm_siv::{Aes256GcmSiv, Nonce, Key};
use aes_gcm_siv::aead::{Aead, Payload}; use aes_gcm_siv::aead::{Aead, Payload, NewAead};
use std::iter::IntoIterator; use std::iter::IntoIterator;
use std::collections::HashMap; use std::collections::{BinaryHeap, HashMap};
use std::io::{Result, Error, ErrorKind}; use std::io::{Result, Error, ErrorKind};
@ -36,10 +36,11 @@ use std::io::{Result, Error, ErrorKind};
/// ///
/// The root basis structure takes up the first valid VPAGE in the virtual memory space. /// The root basis structure takes up the first valid VPAGE in the virtual memory space.
/// It contains a count of the number of valid dictionaries in the Basis. Dictionaries are found at /// It contains a count of the number of valid dictionaries in the Basis. Dictionaries are found at
/// fixed offsets starting at 0xFE_0000 and repeating every 0xFE_0000 intervals. A naive linear search /// fixed offsets starting at 0xFE_0000 and repeating every 0xFE_0000 intervals, with up to 16383 dictionaries
/// is used to scan for dictionaries, starting at the lowest address, scanning every 0xFE_0000, until /// allowed. A naive linear search is used to scan for dictionaries, starting at the lowest address,
/// the number of valid dictionaries have been found that matches the valid dictionary count prescribed /// scanning every 0xFE_0000, until the number of valid dictionaries have been found that matches the valid
/// in the Basis root. A dictionary can be effectively deleted by just marking its descriptor as invalid. /// dictionary count prescribed in the Basis root. A dictionary can be effectively deleted by just marking its
/// descriptor as invalid.
/// ///
/// A stride of 0xFE_0000 means that dictionary descriptors can be up to 4096 VPAGEs long. A dictionary /// A stride of 0xFE_0000 means that dictionary descriptors can be up to 4096 VPAGEs long. A dictionary
/// descriptor consists of a `DictDescriptor` header, some bookkeeping data, plus a count of the number /// descriptor consists of a `DictDescriptor` header, some bookkeeping data, plus a count of the number
@ -53,9 +54,6 @@ use std::io::{Result, Error, ErrorKind};
/// Thus adding a new dictionary always consumes at least one 4k page, but you can have up to 15 keys /// Thus adding a new dictionary always consumes at least one 4k page, but you can have up to 15 keys
/// in that dictionary with no extra bookkeeping cost once the dictionary is added. /// in that dictionary with no extra bookkeeping cost once the dictionary is added.
/// ///
/// Key data storage itself pulls from allocation pools that optimize for large (>64k), medium (~4k) and
/// small (<256 byte) data lengths, using different allocation strategies to reduce fragmentation.
///
/// ///
/// Basis Virtual Memory Layout /// Basis Virtual Memory Layout
/// ///
@ -64,19 +62,20 @@ use std::io::{Result, Error, ErrorKind};
/// | 0x0000_0000_0000_0000 | Invalid -- VPAGE 0 reserved for Option<> | /// | 0x0000_0000_0000_0000 | Invalid -- VPAGE 0 reserved for Option<> |
/// | 0x0000_0000_0000_0FE0 | Basis root page | /// | 0x0000_0000_0000_0FE0 | Basis root page |
/// | 0x0000_0000_00FE_0000 | Dictionary[0] | /// | 0x0000_0000_00FE_0000 | Dictionary[0] |
/// | +0 | - Dict header (127 bytes) |
/// | +7F | - Maybe key entry (127 bytes) |
/// | +FE | - Maybe key entry (127 bytes) |
/// | +FD_FF02 | - Last key entry start (128k possible) |
/// | 0x0000_0000_01FC_0000 | Dictionary[1] | /// | 0x0000_0000_01FC_0000 | Dictionary[1] |
/// | 0x0000_003F_8000_0000 | Dictionary[16383] | /// | 0x0000_003F_7F02_0000 | Dictionary[16382] |
/// | 0x0000_003F_80FE_0F00 | Unused | /// | 0x0000_003F_8000_0000 | Small data pool start (~256GiB) |
/// | 0x0000_0040_0000_0000 | Small data pool start (256GiB) |
/// | | - Dict[0] pool = 16MiB (4k vpages) | /// | | - Dict[0] pool = 16MiB (4k vpages) |
/// | 0x0000_0040_00FE_0000 | - Dict[1] pool = 16MiB | /// | 0x0000_003F_80FE_0000 | - Dict[1] pool = 16MiB |
/// | 0x0000_007F_8000_0000 | - Dict[16383] pool | /// | 0x0000_007E_FE04_0000 | - Dict[16383] pool |
/// | 0x0000_007F_80FE_0000 | Unused | /// | 0x0000_007E_FF02_0000 | Unused |
/// | 0x0000_0080_0000_0000 | Medium data pool start (512GiB) | /// | 0x0000_007F_0000_0000 | Medium data pool start |
/// | | - Dict[0] pool = 32MiB (8k vpages) | /// | | - TBD |
/// | 0x0000_0080_01FC_0000 | - Dict[1] pool = 32MiB | /// | 0x0000_FE00_0000_0000 | Large data pool start (~16mm TiB) |
/// | 0x0000_00FF_0000_0000 | Unused |
/// | 0x0000_0100_0000_0000 | Large data pool start (~16mm TiB) |
/// | | - Demand-allocated, bump-pointer | /// | | - Demand-allocated, bump-pointer |
/// | | currently no defrag | /// | | currently no defrag |
/// ///
@ -86,9 +85,9 @@ use std::io::{Result, Error, ErrorKind};
/// ///
/// Memory Pools /// Memory Pools
/// ///
/// Key data is split into three categories of sizes: small, medium, and large. The thresholds /// Key data is split into three categories of sizes: small, medium, and large; but the implementation
/// are subject to tuning, but roughly speaking, small data are keys <2k bytes; medium are ~4k; /// currently only deals with small and large keys. The thresholds are subject to tuning, but
/// and large are bigger than 32k. /// roughly speaking, small data are keys <4k bytes; large keys are everything else.
/// ///
/// Large keys are the simplest - each key starts at a VPAGE-aligned address, and allocates /// Large keys are the simplest - each key starts at a VPAGE-aligned address, and allocates
/// up from there. Any unused amount is wasted, but with a ~32k threshold you'll have no worse /// up from there. Any unused amount is wasted, but with a ~32k threshold you'll have no worse
@ -140,9 +139,26 @@ use std::io::{Result, Error, ErrorKind};
/// structure and serialize/deserialize that into the correct Rust structure. See /// structure and serialize/deserialize that into the correct Rust structure. See
/// https://doc.rust-lang.org/nomicon/other-reprs.html for a citation on that. /// https://doc.rust-lang.org/nomicon/other-reprs.html for a citation on that.
pub(crate) const SMALL_POOL_START: u64 = 0x0000_003F_8000_0000;
pub(crate) const SMALL_POOL_END: u64 = 0x0000_007E_FF02_0000;
pub(crate) const SMALL_POOL_STRIDE: u64 = 0xFE_0000;
/// we don't want this bigger than VPAGE_SIZE, because a key goal of the small pool is to
/// reduce # of writes to the disk of small data. While we could get some gain in memory efficiency
/// if we made this larger than a VPAGE_SIZE, we don't get much gain in terms of write reduction,
/// and it greatly complicates the implementation. So, SMALL_POOL_MAXENTRY should be less than VPAGE_SIZE.
pub(crate) const SMALL_POOL_MAXENTRY: usize = VPAGE_SIZE;
pub(crate) const LARGE_POOL_START: u64 = 0x0000_FE00_0000_0000;
pub(crate) const KEY_MAXCOUNT: usize = 131_071; // 2^17 - 1
/// The chosen "stride" of a dict/key entry. Drives a lot of key parameters in the database's characteristics. /// The chosen "stride" of a dict/key entry. Drives a lot of key parameters in the database's characteristics.
/// This is chosen such that 32 of these entries fit evenly into a VPAGE. /// This is chosen such that 32 of these entries fit evenly into a VPAGE.
pub(crate) const DK_STRIDE: usize = 127; pub(crate) const DK_STRIDE: usize = 127;
//// DK_STRIDES per VPAGE
pub(crate) const DK_PER_VPAGE: usize = VPAGE_SIZE / DK_STRIDE; // should be 32 - use this for computing modulus on dictionary indices
/// size of a dictionary region in virtual memory
pub(crate) const DICT_VSIZE: u64 = 0xFE_0000;
/// maximum number of dictionaries in a system
pub(crate) const DICT_MAXCOUNT: usize = 16383;
/// This is the format of the Basis as stored on disk /// This is the format of the Basis as stored on disk
#[derive(PartialEq, Debug, Default)] #[derive(PartialEq, Debug, Default)]
@ -196,7 +212,7 @@ pub(crate) struct BasisCache {
} }
impl BasisCache { impl BasisCache {
pub(crate) fn new() -> Self { pub(crate) fn new() -> Self {
BasisCache { cache: Vec::new() } BasisCache { cache: Vec::new(), }
} }
pub(crate) fn add_basis(&mut self, basis: BasisCacheEntry) { pub(crate) fn add_basis(&mut self, basis: BasisCacheEntry) {
self.cache.push(basis); self.cache.push(basis);
@ -250,13 +266,23 @@ impl BasisCache {
for (src, dst) in name.bytes().into_iter().zip(dict_name.iter_mut()) { for (src, dst) in name.bytes().into_iter().zip(dict_name.iter_mut()) {
*dst = src; *dst = src;
} }
let mut my_aad = Vec::<u8>::new();
for &b in basis.aad.iter() {
my_aad.push(b);
}
let mut init_flags = DictFlags(0);
init_flags.set_valid(true);
let dict_cache = DictCacheEntry { let dict_cache = DictCacheEntry {
index: dict_index, index: dict_index,
keys: HashMap::<String, KeyCacheEntry>::new(), keys: HashMap::<String, KeyCacheEntry>::new(),
clean: false, clean: false,
age: 111, age: 0,
flags: 222, flags: init_flags,
key_count: 0, key_count: 0,
free_key_offset: Some(0),
small_pool: Vec::<KeySmallPool>::new(),
small_pool_free: BinaryHeap::<KeySmallPoolOrd>::new(),
aad: my_aad,
}; };
log::info!("adding dictionary {}", name); log::info!("adding dictionary {}", name);
basis.dicts.insert(String::from(name), dict_cache); basis.dicts.insert(String::from(name), dict_cache);
@ -300,54 +326,149 @@ pub(crate) struct BasisCacheEntry {
pub v2p_map: HashMap<VirtAddr, PhysPage>, pub v2p_map: HashMap<VirtAddr, PhysPage>,
/// the last journal rev written to disk /// the last journal rev written to disk
pub journal: u32, pub journal: u32,
/// current allocation pointer for the "large" pool. This just keeps incrementing "forever".
pub large_alloc_ptr: Option<PageAlignedVa>,
} }
impl BasisCacheEntry { impl BasisCacheEntry {
/* /// given a pointer to the hardware, name of the basis, and its cryptographic key, try to derive
/// the basis. If `lazy` is true, it stops with the minimal amount of effort to respond to a query.
/// If it `lazy` is false, it will populate the dictionary cache and key cache entries, as well as
/// discover the location of the `large_alloc_ptr`.
pub(crate) fn mount(hw: &mut PddbOs, name: &str, key: &[u8; AES_KEYSIZE], lazy: bool) -> Option<BasisCacheEntry> {
if let Some(basis_map) = hw.pt_scan_key(key, name) {
let cipher = Aes256GcmSiv::new(Key::from_slice(key));
let aad = hw.data_aad(name);
// get the first page, where the basis root is guaranteed to be
if let Some(root_page) = basis_map.get(&VirtAddr::new(VPAGE_SIZE as u64).unwrap()) {
let vpage = match hw.data_decrypt_page(&cipher, &aad, root_page) {
Some(data) => data,
None => {log::error!("System basis decryption did not authenticate. Unrecoverable error."); return None;},
};
// if the below assertion fails, you will need to re-code this to decrypt more than one VPAGE and stripe into a basis root struct
assert!(size_of::<BasisRoot>() <= VPAGE_SIZE, "BasisRoot has grown past a single VPAGE, this routine needs to be re-coded to accommodate the extra bulk");
let mut basis_root = BasisRoot::default();
for (&src, dst) in vpage[size_of::<JournalType>()..].iter().zip(basis_root.deref_mut().iter_mut()) {
*dst = src;
}
if basis_root.magic != PDDB_MAGIC {
log::error!("Basis root did not deserialize correctly, unrecoverable error.");
return None;
}
if basis_root.version != PDDB_VERSION {
log::error!("PDDB version mismatch in system basis root. Unrecoverable error.");
return None;
}
let basis_name = cstr_to_string(&basis_root.name.0);
if basis_name != String::from(name) {
log::error!("Discovered basis name does not match the requested name: {}; aborting mount operation.", basis_name);
return None;
}
let mut bcache = BasisCacheEntry {
name: basis_name.clone(),
clean: true,
last_sync: Some(hw.timestamp_now()),
num_dicts: basis_root.num_dictionaries,
dicts: HashMap::<String, DictCacheEntry>::new(),
cipher,
aad,
age: basis_root.age,
free_dict_offset: None,
v2p_map: basis_map,
journal: u32::from_le_bytes(vpage[..size_of::<JournalType>()].try_into().unwrap()),
large_alloc_ptr: None,
};
if !lazy {
bcache.populate_caches(hw);
}
log::info!("Basis {} found and reconstructed", name);
return Some(bcache);
} else {
// i guess technically we could try a brute-force search for the page if it went missing, but meh.
log::error!("Basis {} did not contain a root page -- unrecoverable error.", name);
return None;
}
} else {
log::error!("Basis {} has no page table entries -- maybe a bad password?", name);
None
}
}
/// called during the initial basis scan to track where the large allocation pointer end should be.
/// basically try to find the maximal extent of already allocated data, and start allocating from there.
pub(crate) fn large_pool_update(&mut self, maybe_end: u64) {
if maybe_end > self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)).as_u64() {
self.large_alloc_ptr = Some(PageAlignedVa::from(maybe_end));
}
}
/// allocate a pointer data in the large pool, of length `amount`. "always" succeeds because...
/// there's 16 million terabytes of large pool to allocate before you run out?
pub(crate) fn large_pool_alloc(&mut self, amount: u64) -> u64 {
let alloc_ptr = self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START));
self.large_alloc_ptr = Some(self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)) + PageAlignedVa::from(amount));
return alloc_ptr.as_u64()
}
/// do a deep scan of all the dictionaries and keys and attempt to populate all the caches
pub(crate) fn populate_caches(&mut self, hw: &mut PddbOs) {
let mut try_entry = 1;
let mut dict_count = 0;
while try_entry <= DICT_MAXCOUNT && dict_count < self.num_dicts {
let dict_vaddr = VirtAddr::new(try_entry as u64 * DICT_VSIZE).unwrap();
if let Some(pp) = self.v2p_map.get(&dict_vaddr) {
if let Some(dict) = self.dict_decrypt(hw, &pp) {
if dict.flags.valid() {
let dict_name = String::from(cstr_to_string(&dict.name));
let mut dcache = DictCacheEntry::new(dict, try_entry, &self.aad);
let max_large_alloc = dcache.fill(hw, &self.v2p_map, &self.cipher);
self.dicts.insert(dict_name, dcache);
self.large_pool_update(max_large_alloc.get());
dict_count += 1;
} else {
// this is an empty dictionary entry. we could stick a dictionary in here later on, take note if we haven't already computed that
if self.free_dict_offset.is_none() { self.free_dict_offset = Some(try_entry as u32); }
}
} else {
if self.free_dict_offset.is_none() { self.free_dict_offset = Some(try_entry as u32); }
}
} else {
if self.free_dict_offset.is_none() { self.free_dict_offset = Some(try_entry as u32); }
}
try_entry += 1;
}
if try_entry <= DICT_MAXCOUNT {
if self.free_dict_offset.is_none() { self.free_dict_offset = Some(try_entry as u32); }
}
}
/// If `paranoid` is true, it recurses through each key and replaces its data with random junk. /// If `paranoid` is true, it recurses through each key and replaces its data with random junk.
/// Otherwise, it does a "shallow" delete and just removes the directory entry, which is much /// Otherwise, it does a "shallow" delete and just removes the directory entry, which is much
/// more performant. Note that the intended "fast" way to secure-erase data is to store sensitive /// more performant. Note that the intended "fast" way to secure-erase data is to store sensitive
/// data in its own Basis, and then remove the Basis itself. This is much faster than picking /// data in its own Basis, and then remove the Basis itself. This is much faster than picking
/// through compounded data and re-writing partias sectors, and because of this, initially, /// through compounded data and re-writing partias sectors, and because of this, initially,
/// the `paranoid` erase is `unimplemented`. /// the `paranoid` erase is `unimplemented`.
pub(crate) fn dict_delete(&self, name: &str, basis_name: Option<&str>, paranoid: bool) -> Result<()> { pub(crate) fn dict_delete(&mut self, hw: &mut PddbOs, name: &str, paranoid: bool) -> Result<()> {
if self.basis_cache.len() == 0 { if let Some((index, _dict)) = self.dict_deep_search(hw, name) {
return Err(Error::new(ErrorKind::NotConnected, "PDDB is not mounted")); if !paranoid {
} // erase the header by writing over with random data. This makes the dictionary unsearchable, but if you
let maybe_basis = if let Some(n) = basis_name { // have the key, you can of course do a hard-scan and try partially re-assemble the dictionary.
self.basis_cache.iter_mut().filter(|&bc| bc.name == n).next() if let Some(pp) = self.v2p_map.get(&VirtAddr::new(index as u64 * DICT_VSIZE as u64).unwrap()) {
} else { let mut random = [0u8; PAGE_SIZE];
self.basis_cache.last_mut() hw.trng_slice(&mut random);
}; hw.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
if let Some(basis) = maybe_basis {
if let Some((index, _dict)) = self.dict_deep_search(&mut basis, name) {
let map = self.v2p_map.get_mut(&basis.name).expect("No v2p map despite extant BasisCacheEntry record. Shouldn't be possible...");
if !paranoid {
// erase the header by writing over with random data. This makes the dictionary unsearchable, but if you
// have the key, you can of course do a hard-scan and try partially re-assemble the dictionary.
if let Some(pp) = map.get(&VirtAddr::new(index as u64 * DICT_VSIZE as u64).unwrap()) {
let mut random = [0u8; PAGE_SIZE];
self.entropy.borrow_mut().get_slice(&mut random);
self.patch_data(&random, pp.page_number() * PAGE_SIZE as u32);
} else {
log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table.");
}
} else { } else {
unimplemented!("For now store sensitive data in its own Basis, and then delete the Basis."); log::warn!("Inconsistent internal state: requested dictionary didn't have a mapping in the page table.");
// this is a bit of an arduous code path, it involves recursing through all the keys and nuking
// them. Let's write this after we've actually _got keys_ (we're just figuring out how to add a dictionary
// in the first place right now!).
} }
// de-allocate all of the dictionary entries
Ok(())
} else { } else {
Err(Error::new(ErrorKind::NotFound, "Dictionary not found")) unimplemented!("For now store sensitive data in its own Basis, and then delete the Basis.");
// this is a bit of an arduous code path, it involves recursing through all the keys and nuking
// them. Let's write this after we've actually _got keys_ (we're just figuring out how to add a dictionary
// in the first place right now!).
} }
// de-allocate all of the dictionary entries
Ok(())
} else { } else {
Err(Error::new(ErrorKind::NotFound, "Requested basis not found")) Err(Error::new(ErrorKind::NotFound, "Dictionary not found"))
} }
} }
*/
pub(crate) fn dict_get_free_offset(&mut self, hw: &mut PddbOs) -> u32 { pub(crate) fn dict_get_free_offset(&mut self, hw: &mut PddbOs) -> u32 {
if let Some(offset) = self.free_dict_offset.take() { if let Some(offset) = self.free_dict_offset.take() {
@ -407,9 +528,7 @@ impl BasisCacheEntry {
/// dictionary index for this to work. /// dictionary index for this to work.
pub(crate) fn dict_decrypt(&self, hw: &mut PddbOs, pp: &PhysPage) -> Option<Dictionary> { pub(crate) fn dict_decrypt(&self, hw: &mut PddbOs, pp: &PhysPage) -> Option<Dictionary> {
if let Some(data) = hw.data_decrypt_page(&self.cipher, &self.aad, &pp) { if let Some(data) = hw.data_decrypt_page(&self.cipher, &self.aad, &pp) {
let mut dict = Dictionary { let mut dict = Dictionary::default();
flags: 0, age: 0, num_keys: 0, name: [0; DICT_NAME_LEN]
};
for (&src, dst) in data[size_of::<JournalType>()..].iter().zip(dict.deref_mut().iter_mut()) { for (&src, dst) in data[size_of::<JournalType>()..].iter().zip(dict.deref_mut().iter_mut()) {
*dst = src; *dst = src;
} }

View file

@ -2,6 +2,7 @@ use crate::api::*;
use super::*; use super::*;
use core::cell::RefCell; use core::cell::RefCell;
use std::num::NonZeroU32;
use std::rc::Rc; use std::rc::Rc;
use core::ops::{Deref, DerefMut}; use core::ops::{Deref, DerefMut};
use core::mem::size_of; use core::mem::size_of;
@ -9,9 +10,32 @@ use std::convert::TryInto;
use aes_gcm_siv::{Aes256GcmSiv, Nonce}; use aes_gcm_siv::{Aes256GcmSiv, Nonce};
use aes_gcm_siv::aead::{Aead, Payload}; use aes_gcm_siv::aead::{Aead, Payload};
use std::iter::IntoIterator; use std::iter::IntoIterator;
use std::collections::HashMap; use std::collections::{HashMap, BinaryHeap};
use std::io::{Result, Error, ErrorKind}; use std::io::{Result, Error, ErrorKind};
use bitfield::bitfield;
bitfield! {
#[derive(Copy, Clone, PartialEq, Eq)]
pub struct DictFlags(u32);
impl Debug;
pub valid, set_valid: 0;
}
/// stashed copy of a decrypted page. The copy here must always match
/// what's actually on disk; do not mutate it and expect it to sync with the disk.
/// Remember to invalidate this if the data are
/// This is stored with the journal number on top.
/// What the four possibilities of cache vs pp mean:
/// Some(cache) & Some(cache_pp) -> valid cache and pp
/// None(cache) & Some(cache_pp) -> the page was allocated; but never used, or was erased (it's free for you to use it); alternately, it was corrupted
/// Some(cache) & None(cache_pp) -> invalid, internal error
/// None(cache) & None(cache_pp) -> the basis mapping didn't exist: we've never requested this page before.
pub(crate) struct PlaintextCache {
/// a page of data, stored with the Journal rev on top
pub(crate) data: Option<Vec::<u8>>,
/// the page the cache corresponds to
pub(crate) tag: Option<PhysPage>,
}
/// RAM based copy of the dictionary structures on disk. /// RAM based copy of the dictionary structures on disk.
pub(crate) struct DictCacheEntry { pub(crate) struct DictCacheEntry {
/// Use this to compute the virtual address of the dictionary's location /// Use this to compute the virtual address of the dictionary's location
@ -23,14 +47,192 @@ pub(crate) struct DictCacheEntry {
pub(crate) keys: HashMap::<String, KeyCacheEntry>, pub(crate) keys: HashMap::<String, KeyCacheEntry>,
/// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys` /// count of total keys in the dictionary -- may be equal to or larger than the number of elements in `keys`
pub(crate) key_count: u32, pub(crate) key_count: u32,
/// a cached copy of the next free key slot, expressed in units of DK_STRIDE, with 0 being the first valid key offset
pub(crate) free_key_offset: Option<u32>,
/// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified. /// set if synced to disk. should be cleared if the dict is modified, and/or if a subordinate key descriptor is modified.
pub(crate) clean: bool, pub(crate) clean: bool,
/// track modification count /// track modification count
pub(crate) age: u32, pub(crate) age: u32,
/// copy of the flags entry on the Dict on-disk /// copy of the flags entry on the Dict on-disk
pub(crate) flags: u32, pub(crate) flags: DictFlags,
/// small pool data. index corresponds to portion on disk. This structure is built up as the dictionary is
/// read in, and is the "master" for tracking purposes. We always fill this from index 0 and go up; if a KeySmallPool
/// goes completely empty, the entry should still exist but indicate that it's got space. Thus if a key was found allocated
/// to the Nth index position, but the previous N-1 positions are empty, the only way we could have gotten there was if we
/// had allocated lots of small data, filled upo the pool to the Nth position, and then deleted all of that prior data.
/// This situation could create pathologies in the memory usage overhead of the small_pool, which until we have a "defrag"
/// operation for the small pool, we may just have to live with.
pub(crate) small_pool: Vec<KeySmallPool>,
/// free space of each small pool element. It's a collection of free space along with the Vec index of the small_pool.
/// We don't keep the KeySmallPool itself in the small_pool_free directly because it's presumed to be more common
/// that we want to index the data, than it is to need to ask the question of who has the most space free.
/// This stays in lock-step with the small_pool data because we do a .pop() to get the target vector from the small_pool_free,
/// then we modify the pool item, and then we .push() it back into the heap (or if it doesn't fit at all we allocate a new
/// entry and return the original item plus the new one to the heap).
pub(crate) small_pool_free: BinaryHeap<KeySmallPoolOrd>,
/// copy of our AAD, for convenience
pub(crate) aad: Vec::<u8>,
} }
impl DictCacheEntry { impl DictCacheEntry {
pub fn new(dict: Dictionary, index: usize, aad: &Vec<u8>) -> DictCacheEntry {
let mut my_aad = Vec::<u8>::new();
for &b in aad.iter() {
my_aad.push(b);
}
DictCacheEntry {
index: index as u32,
keys: HashMap::<String, KeyCacheEntry>::new(),
key_count: dict.num_keys,
free_key_offset: None,
clean: true,
age: dict.age,
flags: dict.flags,
small_pool: Vec::<KeySmallPool>::new(),
small_pool_free: BinaryHeap::<KeySmallPoolOrd>::new(),
aad: my_aad,
}
}
/// Populates cache entries, reporting the maximum extent of large alloc data seen so far.
/// Requires a descriptor for the hardware, and our virtual to physical page mapping.
pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv) -> VirtAddr {
let dict_vaddr = VirtAddr::new(self.index as u64 * DICT_VSIZE).unwrap();
let mut try_entry = 1;
let mut key_count = 0;
let mut alloc_top = VirtAddr::new(LARGE_POOL_START).unwrap();
let mut index_cache = PlaintextCache { data: None, tag: None };
let mut data_cache = PlaintextCache { data: None, tag: None };
while try_entry < KEY_MAXCOUNT && key_count < self.key_count {
// cache our decryption data -- there's about 32 entries per page, and the scan is largely linear/sequential, so this should
// be a substantial savings in effort.
// Determine the absolute virtual address of the requested entry. It's written a little weird because
// DK_PER_VPAGE is 32, which optimizes cleanly and removes an expensive division step
let req_vaddr = self.index as u64 * DICT_VSIZE + ((try_entry / DK_PER_VPAGE) as u64) * VPAGE_SIZE as u64;
if let Some(pp) = v2p_map.get(&VirtAddr::new(req_vaddr).unwrap()) {
let mut fill_needed = false;
if let Some(tag) = index_cache.tag {
if tag.page_number() != pp.page_number() {
fill_needed = true;
}
} else if index_cache.tag.is_none() {
fill_needed = true;
}
if fill_needed {
index_cache.data = hw.data_decrypt_page(&cipher, &self.aad, pp);
index_cache.tag = Some(*pp);
}
} else {
index_cache.data = None;
index_cache.tag = None;
}
if index_cache.data.is_none() || index_cache.tag.is_none() {
// somehow we hit a page where nothing was allocated (perhaps it was previously deleted?), or less likely, the data was corrupted. Note the isuse, skip past it.
if self.free_key_offset.is_none() { self.free_key_offset = Some(try_entry as u32) }
log::warn!("Dictionary fill op encountered an unallocated page checking entry {} in the dictionary map. Marking it for re-use.", try_entry);
try_entry += DK_PER_VPAGE;
} else {
let cache = index_cache.data.as_ref().expect("Cache should be full, it was already checked...");
let pp = index_cache.tag.as_ref().expect("PP should be in existence, it was already checked...");
let mut keydesc = KeyDescriptor::default();
let start = size_of::<JournalType>() + (try_entry % DK_PER_VPAGE) * DK_STRIDE;
for (&src, dst) in cache[start..start + DK_STRIDE].iter().zip(keydesc.deref_mut().iter_mut()) {
*dst = src;
}
if keydesc.flags.valid() {
let mut kcache = KeyCacheEntry {
start: keydesc.start,
len: keydesc.len,
reserved: keydesc.reserved,
flags: keydesc.flags,
age: keydesc.age,
descriptor_index: NonZeroU32::new(try_entry as u32).unwrap(),
clean: true,
data: None,
};
let kname = cstr_to_string(&keydesc.name);
if keydesc.start + keydesc.reserved > alloc_top.get() {
// if the key is within the large pool space, note its allocation for the basis overall
alloc_top = VirtAddr::new(keydesc.start + keydesc.reserved).unwrap();
// nothing else needs to be done -- we don't pre-cache large key data.
} else if keydesc.start < SMALL_POOL_END {
// if the key is within the small pool space, create a bookkeeping record for it, and pre-cache its data.
// generate the index within the small pool based on the address
assert!(keydesc.start >= SMALL_POOL_START, "Small pool key descriptor has an invalid range");
let rebase = keydesc.start - SMALL_POOL_START;
let derived_dict_index = rebase / SMALL_POOL_STRIDE;
assert!(derived_dict_index == (self.index - 1) as u64, "Small pool key was not stored in our dictionary's designated region");
let pool_index = (rebase - (derived_dict_index * SMALL_POOL_STRIDE)) as usize / VPAGE_SIZE as usize;
while self.small_pool.len() < pool_index {
// fill in the pool with blank entries. In general, we should have a low amount of blank entries, but
// one situation where we could get a leak is if we allocate a large amount of small data, and then delete
// all but the most recently allocated one, leaving an orphan at a high index, which is then subsequently
// treated as read-only so none of the subsequent write/update ops would have occassion to move it. This would
// need to be remedied with a "defrag" operation, but for now, we don't have that.
let ksp = KeySmallPool::new();
self.small_pool.push(ksp);
}
let ksp = &mut self.small_pool[pool_index];
ksp.contents.push(cstr_to_string(&keydesc.name));
assert!(keydesc.reserved >= keydesc.len, "Reserved amount is less than length, this is an error!");
assert!(keydesc.reserved <= VPAGE_SIZE as u64, "Reserved amount is not appropriate for the small pool. Logic error in prior PDDB operation!");
assert!((ksp.avail as u64) < keydesc.reserved, "Total amount allocated to a small pool chunk is incorrect; suspect logic error in prior PDDB operation!");
ksp.avail -= keydesc.reserved as u16;
// note: small_pool_free is updated only after all the entries have been read in
// now grab the *data* referred to by this key. Maybe this is a "bad" idea -- this can really eat up RAM fast to hold
// all the small pool data right away, but let's give it a try and see how it works. Later on we can always skip this.
// manage a separate small cache for data blocks, under the theory that small keys tend to be packed together
let data_vaddr = (keydesc.start / VPAGE_SIZE as u64) * VPAGE_SIZE as u64;
if let Some(pp) = v2p_map.get(&VirtAddr::new(data_vaddr).unwrap()) {
let mut fill_needed = false;
if let Some(tag) = data_cache.tag {
if tag.page_number() != pp.page_number() {
fill_needed = true;
}
} else if data_cache.tag.is_none() {
fill_needed = true;
}
if fill_needed {
data_cache.data = hw.data_decrypt_page(&cipher, &self.aad, pp);
data_cache.tag = Some(*pp);
}
// the data should all be page-aligned, so we can "just" access by the modulus
if let Some(page) = data_cache.data.as_ref() {
let start_offset = size_of::<JournalType>() + (keydesc.start % VPAGE_SIZE as u64) as usize;
let data = page[start_offset..start_offset + keydesc.len as usize].to_vec();
kcache.data = Some(KeyCacheData::Small(
KeySmallData {
clean: true,
data
}
));
} else {
log::error!("Key {}'s data region at pp: {:x?} va: {:x} is unreadable", kname, pp, keydesc.start);
}
} else {
log::error!("Key {} lacks a page table entry. Can't read its data...", kname);
data_cache.data = None;
data_cache.tag = None;
}
}
self.keys.insert(kname, kcache);
key_count += 1;
} else {
if self.free_key_offset.is_none() { self.free_key_offset = Some(try_entry as u32) }
}
try_entry += 1;
}
}
if self.free_key_offset.is_none() { self.free_key_offset = Some(try_entry as u32) }
// now build the small_pool_free binary heap structure
for (index, ksp) in self.small_pool.iter().enumerate() {
self.small_pool_free.push(KeySmallPoolOrd{index, avail: ksp.avail})
}
alloc_top
}
/// Update a key entry. If the key does not already exist, it will create a new one. /// Update a key entry. If the key does not already exist, it will create a new one.
/// ///
/// `key_update` will write `data` starting at `offset`, and will grow the record if data /// `key_update` will write `data` starting at `offset`, and will grow the record if data
@ -57,13 +259,20 @@ impl DictCacheEntry {
pub fn key_erase(&mut self, name: &str, paranoid: bool) { pub fn key_erase(&mut self, name: &str, paranoid: bool) {
} }
/// Synchronize a given small pool key to disk
pub(crate) fn key_small_sync(&self, hw: &PddbOs, smallkey: &mut KeySmallPool) {
// 1. do a quick scan to see if any key entries are dirty. If all are clean, terminate fast
// 2. iterate through all the elements of contents
// 3. write the data to a Vec<u8>, while updating the KeyCacheEntry with the new pointers, and marking the entries dirty as necessary
// 4. write the data to disk
}
} }
#[derive(Debug)] #[derive(Debug)]
/// On-disk representation of the dictionary header. /// On-disk representation of the dictionary header.
#[repr(C, align(8))] #[repr(C, align(8))]
pub(crate) struct Dictionary { pub(crate) struct Dictionary {
/// Reserved for flags on the record entry /// Reserved for flags on the record entry
pub(crate) flags: u32, pub(crate) flags: DictFlags,
/// Access count to the dicitionary /// Access count to the dicitionary
pub(crate) age: u32, pub(crate) age: u32,
/// Number of keys in the dictionary /// Number of keys in the dictionary
@ -71,6 +280,13 @@ pub(crate) struct Dictionary {
/// Name. Length should pad out the record to exactly 127 bytes. /// Name. Length should pad out the record to exactly 127 bytes.
pub(crate) name: [u8; DICT_NAME_LEN], pub(crate) name: [u8; DICT_NAME_LEN],
} }
impl Default for Dictionary {
fn default() -> Dictionary {
let mut flags = DictFlags(0);
flags.set_valid(true);
Dictionary { flags, age: 0, num_keys: 0, name: [0; DICT_NAME_LEN] }
}
}
impl Deref for Dictionary { impl Deref for Dictionary {
type Target = [u8]; type Target = [u8];
fn deref(&self) -> &[u8] { fn deref(&self) -> &[u8] {

View file

@ -25,11 +25,6 @@ pub const PAGE_SIZE: usize = spinor::SPINOR_ERASE_SIZE as usize;
/// size of a virtual page -- after the AES encryption and journaling overhead is subtracted /// size of a virtual page -- after the AES encryption and journaling overhead is subtracted
pub const VPAGE_SIZE: usize = PAGE_SIZE - size_of::<Nonce>() - size_of::<Tag>() - size_of::<JournalType>(); pub const VPAGE_SIZE: usize = PAGE_SIZE - size_of::<Nonce>() - size_of::<Tag>() - size_of::<JournalType>();
/// size of a dictionary region in virtual memory
pub(crate) const DICT_VSIZE: u64 = 0xFE_0000;
/// maximum number of dictionaries in a system
pub(crate) const DICT_MAXCOUNT: usize = 16384;
#[repr(C, packed)] // this can map directly into Flash #[repr(C, packed)] // this can map directly into Flash
pub(crate) struct StaticCryptoData { pub(crate) struct StaticCryptoData {
/// aes-256 key of the system basis, encrypted with the User0 root key /// aes-256 key of the system basis, encrypted with the User0 root key
@ -214,6 +209,10 @@ impl PddbOs {
*Nonce::from_slice(&nonce_array) *Nonce::from_slice(&nonce_array)
} }
pub(crate) fn dna(&self) -> u64 {self.dna} pub(crate) fn dna(&self) -> u64 {self.dna}
pub(crate) fn trng_slice(&mut self, slice: &mut [u8]) {
self.entropy.borrow_mut().get_slice(slice);
}
pub(crate) fn timestamp_now(&self) -> u64 {self.tt.elapsed_ms()}
/// patches data at an offset starting from the data physical base address, which corresponds /// patches data at an offset starting from the data physical base address, which corresponds
/// exactly to the first entry in the page table /// exactly to the first entry in the page table
@ -349,7 +348,7 @@ impl PddbOs {
/// scans the page tables and returns all entries for a given basis /// scans the page tables and returns all entries for a given basis
/// basis_name is needed to decrypt pages in case of a journal conflict /// basis_name is needed to decrypt pages in case of a journal conflict
fn pt_scan_key(&self, key: &[u8; AES_KEYSIZE], basis_name: &str) -> Option<HashMap::<VirtAddr, PhysPage>> { pub(crate) fn pt_scan_key(&self, key: &[u8; AES_KEYSIZE], basis_name: &str) -> Option<HashMap::<VirtAddr, PhysPage>> {
let cipher = Aes256::new(&GenericArray::from_slice(key)); let cipher = Aes256::new(&GenericArray::from_slice(key));
let pt = self.pt_as_slice(); let pt = self.pt_as_slice();
let mut map = HashMap::<VirtAddr, PhysPage>::new(); let mut map = HashMap::<VirtAddr, PhysPage>::new();
@ -941,6 +940,8 @@ impl PddbOs {
/// returns a decrypted page that still includes the journal number at the very beginning /// returns a decrypted page that still includes the journal number at the very beginning
/// We don't clip it off because it would require re-allocating a vector, and it's cheaper (although less elegant) to later /// We don't clip it off because it would require re-allocating a vector, and it's cheaper (although less elegant) to later
/// just index past it. /// just index past it.
/// NB: we don't have an "encrypt" version of the function, because encryption requires a nonce and journal handling,
/// and I think these pull in so much upstream context it's not worth it to create the function here.
pub(crate) fn data_decrypt_page(&self, cipher: &Aes256GcmSiv, aad: &[u8], page: &PhysPage) -> Option<Vec::<u8>> { pub(crate) fn data_decrypt_page(&self, cipher: &Aes256GcmSiv, aad: &[u8], page: &PhysPage) -> Option<Vec::<u8>> {
let ct_slice = &self.pddb_mr.as_slice()[ let ct_slice = &self.pddb_mr.as_slice()[
self.data_phys_base.as_usize() + page.page_number() as usize * PAGE_SIZE .. self.data_phys_base.as_usize() + page.page_number() as usize * PAGE_SIZE ..
@ -964,8 +965,6 @@ impl PddbOs {
} }
} }
} }
// NB: we don't have an "encrypt" version of the function, because encryption requires a nonce and journal handling,
// and I think these pull in so much upstream context it's not worth it to create the function here.
/// Meant to be called on boot. This will read the FastSpace record, and then attempt to load /// Meant to be called on boot. This will read the FastSpace record, and then attempt to load
/// in the system basis. /// in the system basis.
@ -1001,8 +1000,7 @@ impl PddbOs {
log::error!("PDDB system basis name is incorrect: {}; aborting mount operation.", basis_name); log::error!("PDDB system basis name is incorrect: {}; aborting mount operation.", basis_name);
return None; return None;
} }
// at this point, we would fill out the cache more, but let's just put in the minimal info so we can do some testing /*
// for example, perhaps we should recurse through the dictionaries and load some keys...
let bcache = BasisCacheEntry { let bcache = BasisCacheEntry {
name: basis_name.clone(), name: basis_name.clone(),
clean: true, clean: true,
@ -1015,18 +1013,19 @@ impl PddbOs {
free_dict_offset: None, free_dict_offset: None,
v2p_map: sysbasis_map, v2p_map: sysbasis_map,
journal: u32::from_le_bytes(vpage[..size_of::<JournalType>()].try_into().unwrap()), journal: u32::from_le_bytes(vpage[..size_of::<JournalType>()].try_into().unwrap()),
}; large_alloc_ptr: None,
log::info!("System BasisRoot record found, saving in cache"); }; */
// save the v2p reverse mapping too. log::info!("System BasisRoot record found, generating cache entry");
return Some(bcache); BasisCacheEntry::mount(self, &basis_name, &syskey, false)
} else { } else {
// i guess technically we could try a brute-force search for the page, but meh. // i guess technically we could try a brute-force search for the page, but meh.
log::error!("System basis did not contain a root page -- unrecoverable error."); log::error!("System basis did not contain a root page -- unrecoverable error.");
return None; None
} }
} } else { None }
} else {
None
} }
None
} }
/// this function is dangerous in that calling it will completely erase all of the previous data /// this function is dangerous in that calling it will completely erase all of the previous data

View file

@ -2,6 +2,7 @@ use crate::api::*;
use super::*; use super::*;
use core::cell::RefCell; use core::cell::RefCell;
use std::num::NonZeroU32;
use std::rc::Rc; use std::rc::Rc;
use core::ops::{Deref, DerefMut}; use core::ops::{Deref, DerefMut};
use core::mem::size_of; use core::mem::size_of;
@ -11,6 +12,15 @@ use aes_gcm_siv::aead::{Aead, Payload};
use std::iter::IntoIterator; use std::iter::IntoIterator;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::{Result, Error, ErrorKind}; use std::io::{Result, Error, ErrorKind};
use std::cmp::Ordering;
use bitfield::bitfield;
bitfield! {
#[derive(Copy, Clone, PartialEq, Eq)]
pub struct KeyFlags(u32);
impl Debug;
pub valid, set_valid: 0;
}
/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so /// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so
/// any deserialization must essentially come with a copy step to line up the record. /// any deserialization must essentially come with a copy step to line up the record.
@ -23,12 +33,24 @@ pub(crate) struct KeyDescriptor {
/// amount of space reserved for the key. Must be >= len. /// amount of space reserved for the key. Must be >= len.
pub(crate) reserved: u64, pub(crate) reserved: u64,
/// Reserved for flags on the record entry /// Reserved for flags on the record entry
pub(crate) flags: u32, pub(crate) flags: KeyFlags,
/// Access count to the key /// Access count to the key
pub(crate) age: u32, pub(crate) age: u32,
/// Name. Length should pad out the record to exactly 127 bytes. /// Name. Length should pad out the record to exactly 127 bytes.
pub(crate) name: [u8; KEY_NAME_LEN], pub(crate) name: [u8; KEY_NAME_LEN],
} }
impl Default for KeyDescriptor {
fn default() -> Self {
KeyDescriptor {
start: 0,
len: 0,
reserved: 0,
flags: KeyFlags(0),
age: 0,
name: [0; KEY_NAME_LEN],
}
}
}
impl Deref for KeyDescriptor { impl Deref for KeyDescriptor {
type Target = [u8]; type Target = [u8];
fn deref(&self) -> &[u8] { fn deref(&self) -> &[u8] {
@ -51,41 +73,114 @@ pub(crate) struct KeyCacheEntry {
pub(crate) start: u64, pub(crate) start: u64,
pub(crate) len: u64, pub(crate) len: u64,
pub(crate) reserved: u64, pub(crate) reserved: u64,
pub(crate) flags: u32, pub(crate) flags: KeyFlags,
pub(crate) age: u32, pub(crate) age: u32,
/// the current on-disk index of the KeyCacheEntry item, enumerated as "0" being the first DictKeyEntry /// the current on-disk index of the KeyCacheEntry item, enumerated as "0" being the Dict descriptor and "1" being the first valid key
/// slot past the first record which is the descriptor of the dictionary. pub(crate) descriptor_index: NonZeroU32,
pub(crate) descriptor_index: u32, /// indicates if the descriptor cache entry is currently synchronized with what's on disk. Does not imply anything about the data,
/// indicates if the cache entry is currently synchronized with what's on disk. /// but if the `data` field is None then there is nothing to in cache to be dirtied.
pub(crate) clean: bool, pub(crate) clean: bool,
/// if Some, contains the keys data contents. if None, you must refer to the disk contents to retrieve it.
pub(crate) data: Option<KeyCacheData>,
} }
impl KeyCacheEntry { impl KeyCacheEntry {
/// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself. /// Given a base offset of the dictionary containing the key, compute the starting VirtAddr of the key itself.
pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr { pub(crate) fn descriptor_vaddr(&self, dict_offset: VirtAddr) -> VirtAddr {
VirtAddr::new(dict_offset.get() + ((self.descriptor_index as u64 + 1) * DK_STRIDE as u64)).unwrap() VirtAddr::new(dict_offset.get() + ((self.descriptor_index.get() as u64 + 1) * DK_STRIDE as u64)).unwrap()
} }
/// Computes the modular position of the KeyDescriptor within a vpage. /// Computes the modular position of the KeyDescriptor within a vpage.
pub(crate) fn descriptor_modulus(&self) -> usize { pub(crate) fn descriptor_modulus(&self) -> usize {
(self.descriptor_index as usize + 1) % (VPAGE_SIZE / DK_STRIDE) (self.descriptor_index.get() as usize + 1) % (VPAGE_SIZE / DK_STRIDE)
} }
/// Computes the vpage offset as measured from the start of the dictionary storage region /// Computes the vpage offset as measured from the start of the dictionary storage region
pub(crate) fn descriptor_vpage_num(&self) -> usize { pub(crate) fn descriptor_vpage_num(&self) -> usize {
(self.descriptor_index as usize + 1) / (VPAGE_SIZE / DK_STRIDE) (self.descriptor_index.get() as usize + 1) / (VPAGE_SIZE / DK_STRIDE)
} }
} }
/// used to identify cached/chunked data from a key pub (crate) enum KeyCacheData {
/// maybe we should detach the cache_page because this is meant to be a key, and cache_page is the payload. Small(KeySmallData),
pub(crate) struct KeyCacheEntryEntry { // the "Medium" type has a region reserved for it, but we haven't coded a handler for it.
basis: String, Large(KeyLargeData),
dict: String, }
key: KeyDescriptor, /// Small data is optimized for low overhead, and always represent a complete copy of the data.
/// offset of the cached data relative to the KeyDescriptor's start pub(crate) struct KeySmallData {
offset: u64, pub clean: bool,
/// the actual cached data pub(crate) data: Vec::<u8>,
cache_page: [u8; VPAGE_SIZE], }
/// true if clean and synced with the disk /// This can hold just a portion of a large key's data. For now, we now essentially manually
clean: bool, /// encode a sub-slice in parts, but, later on we could get more clever and start to cache
/// general flags, tbd /// multiple disjoint portions of a large key's data...
flags: u8, pub(crate) struct KeyLargeData {
pub clean: bool,
pub(crate) start: u64,
pub(crate) data: Vec::<u8>,
}
pub(crate) const SMALL_CAPACITY: usize = VPAGE_SIZE;
/// A storage pool for data that is strictly smaller than one VPAGE. These element are serialized
/// and stored inside the "small data pool" area.
pub(crate) struct KeySmallPool {
// location of data within the Small memory region. Index is in units of SMALL_CAPACITY. (this should be encoded in the vector position)
//pub(crate) index: u32,
/// list of data actually stored within the pool - resolve against `keys` HashMap.
pub(crate) contents: Vec::<String>,
/// keeps track of the available space within the pool, avoiding an expensive lookup every time we want to query the available space
pub(crate) avail: u16,
pub(crate) clean: bool,
}
impl KeySmallPool {
pub(crate) fn new() -> KeySmallPool {
KeySmallPool {
contents: Vec::<String>::new(),
avail: SMALL_CAPACITY as u16,
clean: true,
}
}
/// Tries to fit the key into the current pool. If there isn't enough space, the function returns false.
/// If `capacity_hint` is provided, it tries to reserve this amount of capacity for the key, even if the
/// data provided is smaller than the hint.
pub(crate) fn try_insert_key(&self, data: &KeyCacheEntry, capacity_hint: Option<usize>) -> bool {
// 1. check if the key will fit
// 2. if so, add its size to the used pool and contents
// 3. if not, return false
false
}
/// Tries to update the key in the pool with the new data. If it doesn't fit,
/// returns Some(key_name). Else it returns None. Note that the data is immediately
/// evicted if it doesn't fit -- the caller must then find a new pool to place the
/// evicted key in.
pub(crate) fn try_update_key(&self, data: &KeyCacheEntry, capacity_hint: Option<usize>) -> Option<String> {
None
}
pub(crate) fn delete_key(&self, data: &KeyCacheEntry) {
// 1. remove the contents string
// 2. update the used tracking information
}
/// Returns the amount of space available in the pool
pub(crate) fn free(&self) -> usize {
self.avail as usize
}
}
/// a bookkeeping structrue to put into a max-heap to figure out who has the most available space
#[derive(Eq)]
pub(crate) struct KeySmallPoolOrd {
pub(crate) avail: u16,
pub(crate) index: usize,
}
// only compare based on the amount of data used
impl Ord for KeySmallPoolOrd {
fn cmp(&self, other: &Self) -> Ordering {
self.avail.cmp(&other.avail)
}
}
impl PartialOrd for KeySmallPoolOrd {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl PartialEq for KeySmallPoolOrd {
fn eq(&self, other: &Self) -> bool {
self.avail == other.avail
}
} }