diff --git a/services/pddb/src/backend/basis.rs b/services/pddb/src/backend/basis.rs index db8e7ac70..bc194967e 100644 --- a/services/pddb/src/backend/basis.rs +++ b/services/pddb/src/backend/basis.rs @@ -1,21 +1,11 @@ -use crate::api::*; -use super::*; - -use core::ops::{Deref, DerefMut}; -use core::mem::size_of; -use std::convert::TryInto; -use aes_gcm_siv::{Aes256GcmSiv, Key}; -use aes_gcm_siv::aead::NewAead; -use std::iter::IntoIterator; -use std::collections::{BinaryHeap, HashMap, HashSet}; -use std::io::{Result, Error, ErrorKind}; - - +/// # The Organization of Basis Data +/// +/// ## Overview /// In basis space, the BasisRoot is located at VPAGE #1 (VPAGE #0 is always invalid). /// A VPAGE is 0xFE0 (4,064) bytes long, which is equal to a PAGE of 4k minus 32 bytes of encryption+journal /// overhead. /// -/// NB: The factors of 4,064 are: 1, 2, 4, 8, 16, 32, 127, 254, 508, 1016, 2032, 4064 +/// *NB: The factors of 4,064 are: 1, 2, 4, 8, 16, 32, 127, 254, 508, 1016, 2032, 4064* /// /// AAD associated with the BasisRoot consist of a bytewise concatenation of: /// - Basis name @@ -53,8 +43,8 @@ use std::io::{Result, Error, ErrorKind}; /// in that dictionary with no extra bookkeeping cost once the dictionary is added. /// /// -/// Basis Virtual Memory Layout -/// +/// ## Basis Virtual Memory Layout +///```Text /// | Start Address | | /// |------------------------|-------------------------------------------| /// | 0x0000_0000_0000_0000 | Invalid -- VPAGE 0 reserved for Option<> | @@ -78,12 +68,13 @@ use std::io::{Result, Error, ErrorKind}; /// | 0x0000_FE00_0000_0000 | Large data pool start (~16mm TiB) | /// | | - Demand-allocated, bump-pointer | /// | | currently no defrag | +/// ``` /// /// Note that each Basis has its own memory section, and you can have "many" orthogonal Basis without /// a collision -- the AES keyspace is 128 bits, so you have a decent chance of no collisions /// even with a few billion Basis concurrently existing in the filesystem. /// -/// Memory Pools +/// ## Memory Pools /// /// Key data is split into three categories of sizes: small, medium, and large; but the implementation /// currently only deals with small and large keys. The thresholds are subject to tuning, but @@ -101,7 +92,7 @@ use std::io::{Result, Error, ErrorKind}; /// /// Medium keys have a TBD implementation, and are currently directed to the large pool for now. /// -/// The Alignment and Serialization Chronicles +/// ## The Alignment and Serialization Chronicles /// /// We're using Repr(C) and alignment to 64-bits to create a consistent "FFI" layout; we use an unsafe cast /// to [u8] as our method to serialize the structure, which means we could be subject to breakage if the Rust @@ -131,7 +122,7 @@ use std::io::{Result, Error, ErrorKind}; /// storage structure that can persist through future versions of Rust and also other implementations in other /// languages. /// -/// Known Repr(C) footguns: +/// ## Known Repr(C) footguns: /// - When you start laying in 64-bit types, stuff has to be 64-bit aligned, or else you'll start to get /// uninitialized padding data inserted, which can leak stack data in the serialization process. /// - Don't use anything that's not native to C. In particular, for primitives that we want to be "Option" @@ -139,6 +130,18 @@ use std::io::{Result, Error, ErrorKind}; /// structure and serialize/deserialize that into the correct Rust structure. See /// https://doc.rust-lang.org/nomicon/other-reprs.html for a citation on that. +use crate::api::*; +use super::*; + +use core::ops::{Deref, DerefMut}; +use core::mem::size_of; +use std::convert::TryInto; +use aes_gcm_siv::{Aes256GcmSiv, Key}; +use aes_gcm_siv::aead::NewAead; +use std::iter::IntoIterator; +use std::collections::{BinaryHeap, HashMap, HashSet}; +use std::io::{Result, Error, ErrorKind}; + pub(crate) const SMALL_POOL_START: u64 = 0x0000_003F_8000_0000; pub(crate) const SMALL_POOL_END: u64 = 0x0000_007E_FF02_0000; pub(crate) const SMALL_POOL_STRIDE: u64 = 0xFE_0000; @@ -243,17 +246,9 @@ impl BasisCache { if let Some(basis_index) = self.select_basis(basis_name) { let basis = &mut self.cache[basis_index]; basis.age = basis.age.saturating_add(1); - if basis.dicts.get(&String::from(name)).is_some() { - // quick exit if we see the dictionary is hot in the cache + + if basis.ensure_dict_in_cache(hw, name) { return Err(Error::new(ErrorKind::AlreadyExists, "Dictionary already exists")); - } else { - // if the dictionary doesn't exist in our cache it doesn't necessarily mean it - // doesn't exist. Do a comprehensive search if our cache isn't complete. - if basis.num_dicts as usize != basis.dicts.len() { - if basis.dict_deep_search(hw, name).is_some() { - return Err(Error::new(ErrorKind::AlreadyExists, "Dictionary already exists")); - } - } } basis.clean = false; // allocate a vpage offset for the dictionary @@ -361,16 +356,8 @@ impl BasisCache { ) -> Result { if let Some(basis_index) = self.select_basis(basis_name) { let basis = &mut self.cache[basis_index]; - if !basis.dicts.contains_key(dict) { - if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) { - if dict_record.flags.valid() { - let dict_name = String::from(cstr_to_string(&dict_record.name)); - let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad); - basis.dicts.insert(dict_name, dcache); - } - } else { - return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); - } + if !basis.ensure_dict_in_cache(hw, dict) { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); } if let Some(dict_entry) = basis.dicts.get_mut(dict) { if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) { @@ -435,16 +422,8 @@ impl BasisCache { ) -> Result<()> { if let Some(basis_index) = self.select_basis(basis_name) { let basis = &mut self.cache[basis_index]; - if !basis.dicts.contains_key(dict) { - if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) { - if dict_record.flags.valid() { - let dict_name = String::from(cstr_to_string(&dict_record.name)); - let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad); - basis.dicts.insert(dict_name, dcache); - } - } else { - return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); - } + if !basis.ensure_dict_in_cache(hw, dict) { + return Err(Error::new(ErrorKind::NotFound, "dictionary not found")); } if let Some(dict_entry) = basis.dicts.get_mut(dict) { basis.age = basis.age.saturating_add(1); @@ -497,17 +476,9 @@ impl BasisCache { }; if let Some(basis_index) = self.select_basis(basis_name) { let basis = &mut self.cache[basis_index]; - if !basis.dicts.contains_key(dict) { - if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) { - if dict_record.flags.valid() { - let dict_name = String::from(cstr_to_string(&dict_record.name)); - let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad); - basis.dicts.insert(dict_name, dcache); - } - } else { - pages_needed += 1; - pages_needed += reserved_pages; - } + if !basis.ensure_dict_in_cache(hw, dict) { + pages_needed += 1; + pages_needed += reserved_pages; } if let Some(dict_entry) = basis.dicts.get(dict) { // see if we need to make a kcache entry @@ -565,24 +536,7 @@ impl BasisCache { } // now actually do the update if let Some(basis_index) = self.select_basis(basis_name) { - let mut dict_found = false; - { // resolve the basis here, potentially mutating things - let basis = &mut self.cache[basis_index]; - basis.age = basis.age.saturating_add(1); - basis.clean = false; - if !basis.dicts.contains_key(dict) { - // if the dictionary doesn't exist in our cache it doesn't necessarily mean it - // doesn't exist. Do a comprehensive search if our cache isn't complete. - if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) { - let dict_name = String::from(cstr_to_string(&dict_record.name)); - let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad); - basis.dicts.insert(dict_name, dcache); - dict_found = true; - } - } else { - dict_found = true; - } - } + let dict_found = (&mut self.cache[basis_index]).ensure_dict_in_cache(hw, dict); if !dict_found { // now that we're clear of the deep search, mutate the basis if we are sure it's not there self.dict_add(hw, dict, basis_name).expect("couldn't add dictionary"); } @@ -594,6 +548,8 @@ impl BasisCache { } // refetch the basis here to avoid the re-borrow problem, now that all the potential dict cache mutations are done let basis = &mut self.cache[basis_index]; + basis.age = basis.age.saturating_add(1); + basis.clean = false; // now do the sync if let Some(dict_entry) = basis.dicts.get_mut(dict) { let updated_ptr = dict_entry.key_update(hw, &mut basis.v2p_map, @@ -727,19 +683,6 @@ impl BasisCacheEntry { } } - /// allocate a pointer data in the large pool, of length `amount`. "always" succeeds because... - /// there's 16 million terabytes of large pool to allocate before you run out? - /* - // this function is actually implemented inside the "key_update()" code - as a large key is allocated, - // the pointer is bumped along within the update code. AFAIK, nobody else should be calling this, so, let's - // comment it out, and if it becomes necessary for some reason let's take a good hard look at assumptions. - // In particular, this function might become necessary if disk caching is implemented for large keys. - pub(crate) fn large_pool_alloc(&mut self, amount: u64) -> u64 { - let alloc_ptr = self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)); - self.large_alloc_ptr = Some(self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)) + PageAlignedVa::from(amount)); - return alloc_ptr.as_u64() - }*/ - /// do a deep scan of all the dictionaries and keys and attempt to populate all the caches pub(crate) fn populate_caches(&mut self, hw: &mut PddbOs) { let mut try_entry = 1; @@ -779,21 +722,7 @@ impl BasisCacheEntry { /// through compounded data and re-writing partias sectors, and because of this, initially, /// the `paranoid` erase is `unimplemented`. pub(crate) fn dict_delete(&mut self, hw: &mut PddbOs, name: &str, paranoid: bool) -> Result<()> { - let mut dict_found = false; - if !self.dicts.contains_key(name) { - log::info!("dict delete: key not in cache {}", name); - // if the dictionary doesn't exist in our cache it doesn't necessarily mean it - // doesn't exist. Do a comprehensive search if our cache isn't complete. - if let Some((index, dict_record)) = self.dict_deep_search(hw, name) { - let dict_name = String::from(cstr_to_string(&dict_record.name)); - let dcache = DictCacheEntry::new(dict_record, index as usize, &self.aad); - self.dicts.insert(dict_name, dcache); - dict_found = true; - } - } else { - dict_found = true; - } - if dict_found { + if self.ensure_dict_in_cache(hw, name) { let dcache = self.dicts.get_mut(name).expect("entry was ensured, but somehow missing"); //log::info!("dcache {} index {}, key_count {}", name, dcache.index, dcache.key_count); // ensure all the keys are in RAM @@ -850,6 +779,8 @@ impl BasisCacheEntry { } } + /// Called to compute the next free index. Normally this will get filled during a disk search for + /// cache entries, but it could be None if a new dict entry was allocated and no other entries were loaded. pub(crate) fn dict_get_free_offset(&mut self, hw: &mut PddbOs) -> u32 { if let Some(offset) = self.free_dict_offset.take() { return offset; @@ -1088,6 +1019,39 @@ impl BasisCacheEntry { self.clean = true; } } + + /// This function ensures a dictionary is in the cache; if not, it will load its entry. + pub(crate) fn ensure_dict_in_cache(&mut self, hw: &mut PddbOs, name: &str) -> bool { + let mut dict_found = false; + if !self.dicts.contains_key(name) { + log::info!("dict: key not in cache {}", name); + // if the dictionary doesn't exist in our cache it doesn't necessarily mean it + // doesn't exist. Do a comprehensive search if our cache isn't complete. + if let Some((index, dict_record)) = self.dict_deep_search(hw, name) { + let dict_name = String::from(cstr_to_string(&dict_record.name)); + let dcache = DictCacheEntry::new(dict_record, index as usize, &self.aad); + self.dicts.insert(dict_name, dcache); + dict_found = true; + } + } else { + dict_found = true; + } + dict_found + } + + // allocate a pointer data in the large pool, of length `amount`. "always" succeeds because... + // there's 16 million terabytes of large pool to allocate before you run out? + /* + // this function is actually implemented inside the "key_update()" code - as a large key is allocated, + // the pointer is bumped along within the update code. AFAIK, nobody else should be calling this, so, let's + // comment it out, and if it becomes necessary for some reason let's take a good hard look at assumptions. + // In particular, this function might become necessary if disk caching is implemented for large keys. + pub(crate) fn large_pool_alloc(&mut self, amount: u64) -> u64 { + let alloc_ptr = self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)); + self.large_alloc_ptr = Some(self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)) + PageAlignedVa::from(amount)); + return alloc_ptr.as_u64() + }*/ + } // **** diff --git a/services/pddb/src/backend/dictionary.rs b/services/pddb/src/backend/dictionary.rs index 7c9282f73..543f5d12c 100644 --- a/services/pddb/src/backend/dictionary.rs +++ b/services/pddb/src/backend/dictionary.rs @@ -17,93 +17,9 @@ bitfield! { pub valid, set_valid: 0; } -#[derive(PartialEq, Eq)] -pub(crate) enum FreeKeyCases { - LeftAdjacent, - RightAdjacent, - Within, - LessThan, - GreaterThan, -} -#[derive(Eq, Copy, Clone)] -pub(crate) struct FreeKeyRange { - /// This index should be free - pub(crate) start: u32, - /// Additional free keys after the start one. Run = 0 means just the start key is free, and the - /// next one should be used. Run = 2 means {start, start+1} are free, etc. - pub(crate) run: u32, -} -impl FreeKeyRange { - pub(crate) fn compare_to(&self, index: u32) -> FreeKeyCases { - if self.start > 1 && index < self.start - 1 { - FreeKeyCases::LessThan - } else if self.start > 0 && index == self.start - 1 { - FreeKeyCases::LeftAdjacent - } else if index >= self.start && index <= self.start + self.run { - FreeKeyCases::Within - } else if index == self.start + self.run + 1 { - FreeKeyCases::RightAdjacent - } else { - FreeKeyCases::GreaterThan - } - } -} -impl Ord for FreeKeyRange { - fn cmp(&self, other: &Self) -> Ordering { - // note the reverse order -- so we can sort as a "min-heap" - other.start.cmp(&self.start) - } -} -impl PartialOrd for FreeKeyRange { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} -impl PartialEq for FreeKeyRange { - fn eq(&self, other: &Self) -> bool { - self.start == other.start - } -} - -/// stashed copy of a decrypted page. The copy here must always match -/// what's actually on disk; do not mutate it and expect it to sync with the disk. -/// Remember to invalidate this if the data are -/// This is stored with the journal number on top. -/// What the four possibilities of cache vs pp mean: -/// Some(cache) & Some(cache_pp) -> valid cache and pp -/// None(cache) & Some(cache_pp) -> the page was allocated; but never used, or was erased (it's free for you to use it); alternately, it was corrupted -/// Some(cache) & None(cache_pp) -> invalid, internal error -/// None(cache) & None(cache_pp) -> the basis mapping didn't exist: we've never requested this page before. -pub(crate) struct PlaintextCache { - /// a page of data, stored with the Journal rev on top - pub(crate) data: Option>, - /// the page the cache corresponds to - pub(crate) tag: Option, -} -impl PlaintextCache { - pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv, aad: &[u8], - req_vaddr: VirtAddr - ) { - if let Some(pp) = v2p_map.get(&req_vaddr) { - let mut fill_needed = false; - if let Some(tag) = self.tag { - if tag.page_number() != pp.page_number() { - fill_needed = true; - } - } else if self.tag.is_none() { - fill_needed = true; - } - if fill_needed { - self.data = hw.data_decrypt_page(&cipher, &aad, pp); - self.tag = Some(*pp); - } - } else { - self.data = None; - self.tag = None; - } - } -} -/// RAM based copy of the dictionary structures on disk. +/// RAM based copy of the dictionary structures on disk. Most of the methods on this function operate on +/// keys within the Dictionary. Operations on the Dictionary itself originate from the containing Basis +/// structure. pub(crate) struct DictCacheEntry { /// Use this to compute the virtual address of the dictionary's location /// multiply this by DICT_VSIZE to get at the virtual address. This /could/ be a @@ -824,6 +740,10 @@ impl DictCacheEntry { } } } + +/// Derives the index of a Small Pool storage block given the key cache entry and the dictionary index. +/// The index maps into the small_pool array, which itself maps 1:1 onto blocks inside the small pool +/// memory space. pub(crate) fn small_storage_index_from_key(kcache: &KeyCacheEntry, dict_index: u32) -> Option { let storage_base = dict_index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START; let storage_end = storage_base + SMALL_POOL_STRIDE; @@ -835,7 +755,8 @@ pub(crate) fn small_storage_index_from_key(kcache: &KeyCacheEntry, dict_index: u } } #[derive(Debug)] -/// On-disk representation of the dictionary header. +/// On-disk representation of the dictionary header. This structure is mainly for archival/unarchival +/// purposes. To "functionalize" a stored disk entry, it needs to be deserialized into a DictionaryCacheEntry. #[repr(C, align(8))] pub(crate) struct Dictionary { /// Reserved for flags on the record entry @@ -899,3 +820,91 @@ impl<'a> Default for DictKeyVpage { } } } + + +#[derive(PartialEq, Eq)] +pub(crate) enum FreeKeyCases { + LeftAdjacent, + RightAdjacent, + Within, + LessThan, + GreaterThan, +} +#[derive(Eq, Copy, Clone)] +pub(crate) struct FreeKeyRange { + /// This index should be free + pub(crate) start: u32, + /// Additional free keys after the start one. Run = 0 means just the start key is free, and the + /// next one should be used. Run = 2 means {start, start+1} are free, etc. + pub(crate) run: u32, +} +impl FreeKeyRange { + pub(crate) fn compare_to(&self, index: u32) -> FreeKeyCases { + if self.start > 1 && index < self.start - 1 { + FreeKeyCases::LessThan + } else if self.start > 0 && index == self.start - 1 { + FreeKeyCases::LeftAdjacent + } else if index >= self.start && index <= self.start + self.run { + FreeKeyCases::Within + } else if index == self.start + self.run + 1 { + FreeKeyCases::RightAdjacent + } else { + FreeKeyCases::GreaterThan + } + } +} +impl Ord for FreeKeyRange { + fn cmp(&self, other: &Self) -> Ordering { + // note the reverse order -- so we can sort as a "min-heap" + other.start.cmp(&self.start) + } +} +impl PartialOrd for FreeKeyRange { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl PartialEq for FreeKeyRange { + fn eq(&self, other: &Self) -> bool { + self.start == other.start + } +} + +/// stashed copy of a decrypted page. The copy here must always match +/// what's actually on disk; do not mutate it and expect it to sync with the disk. +/// Remember to invalidate this if the data are +/// This is stored with the journal number on top. +/// What the four possibilities of cache vs pp mean: +/// Some(cache) & Some(cache_pp) -> valid cache and pp +/// None(cache) & Some(cache_pp) -> the page was allocated; but never used, or was erased (it's free for you to use it); alternately, it was corrupted +/// Some(cache) & None(cache_pp) -> invalid, internal error +/// None(cache) & None(cache_pp) -> the basis mapping didn't exist: we've never requested this page before. +pub(crate) struct PlaintextCache { + /// a page of data, stored with the Journal rev on top + pub(crate) data: Option>, + /// the page the cache corresponds to + pub(crate) tag: Option, +} +impl PlaintextCache { + pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::, cipher: &Aes256GcmSiv, aad: &[u8], + req_vaddr: VirtAddr + ) { + if let Some(pp) = v2p_map.get(&req_vaddr) { + let mut fill_needed = false; + if let Some(tag) = self.tag { + if tag.page_number() != pp.page_number() { + fill_needed = true; + } + } else if self.tag.is_none() { + fill_needed = true; + } + if fill_needed { + self.data = hw.data_decrypt_page(&cipher, &aad, pp); + self.tag = Some(*pp); + } + } else { + self.data = None; + self.tag = None; + } + } +} \ No newline at end of file diff --git a/services/pddb/src/backend/key.rs b/services/pddb/src/backend/key.rs index f9e9d5d80..b058383c1 100644 --- a/services/pddb/src/backend/key.rs +++ b/services/pddb/src/backend/key.rs @@ -16,8 +16,9 @@ bitfield! { pub unresolved, set_unresolved: 1; } -/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so -/// any deserialization must essentially come with a copy step to line up the record. +/// On-disk representation of the Key. Note that the storage on disk is mis-aligned relative +/// to Rust's expecatation of in-RAM format, so any deserialization must essentially come with +/// a copy step to re-align the record to meet Rust's placement rules. #[repr(C, align(8))] pub(crate) struct KeyDescriptor { /// virtual address of the key's start @@ -63,6 +64,11 @@ impl DerefMut for KeyDescriptor { } } +/// In-RAM representation of a key. This file defines the storage for the KeyCacheEntry; most of the structure +/// manipulations happen inside `dictionary.rs`, in part because to locate a Key in absolute memory space you need +/// to know what Dictionary it comes from. This is a point to consider for a refactor: if we pull some info about +/// the containing Dictionary into the key, we could associate more methods with the data structure. However, this +/// means duplicating the dictionary index, a field that can then get out of sync. pub(crate) struct KeyCacheEntry { pub(crate) start: u64, pub(crate) len: u64, diff --git a/services/pddb/src/lib.rs b/services/pddb/src/lib.rs index 601f6742e..25b2fc0e7 100644 --- a/services/pddb/src/lib.rs +++ b/services/pddb/src/lib.rs @@ -1,5 +1,365 @@ #![cfg_attr(target_os = "none", no_std)] +/// # PDDB - Plausibly Deniable DataBase +/// +/// ## Glossary: +/// * Basis - a unionizeable dictionary that can be unlocked with a password-protected key. +/// * Dictionary - similar to a "volume" on a typical filesystem. Contains a group of k/v pairs +/// that share attributes such as access permissions +/// * Key - a unique string that identifies a piece of data in the PDDB +/// * Path - Rust's notion of how to locate a file, except we interpret it as a PDDB key, where +/// the "root" directory is the dictionary, and the remainder is the "key" +/// * Value - the value associated with a key, isomorphic ot a "file" in Rust +/// * Open Basis - A Basis that is decryptable and known to the system. +/// * Closed Basis - A Basis that is not decryptable. It's unknown to the system and potentially treated +/// identically to free disk space, e.g., it could be partially overwritten. +/// * FSCB - Free Space Commit Buffer. A few pages allocated to tracking a subset of free space, +/// meant to accelerate PDDB operations. Creates a side-channel that can reveal that some activity +/// has happened, but without disclosing what and why. Contains FastSpace and SpaceUpdate records. +/// Frequently updated, so the buffer is slightly oversized, and which sector is "hot" is randomized +/// for wear-levelling. +/// * MBBB - Make Before Break Buffer. A set of randomly allocated pages that are a shadow copy +/// of a page table page. If any data exists, its contents override those of a corrupted page table. +/// * FastSpace - a collection of random pages that are known to be empty. The number of pages in FastSpace +/// is reduced from the absolute amount of free space available by at least a factor of FSCB_FILL_COEFFICIENT. +/// * SpaceUpdate - encrypted patches to the FastSpace table. The FastSpace table is "heavyweight", and would +/// be too expensive to update on every page allocation, so SpaceUpdate is used to patch the FastSpace table. +/// +/// A `Path` like the following is deconstructed as follows by the PDDB: +/// +/// ```Text +/// Dictionary +/// | Key +/// | | +/// --+- --------+--------------------- +/// logs:matrix/alice/oct30_2021/bob.txt +/// ``` +/// +/// It could equally have an arbitrary name like `logs:Matrix - alice to bob Oct 30 2021`; +/// as long as the string that identifies a Key is unique, it's stored in the database +/// all the same. Any valid utf-8 unicode characters are acceptable. +/// +/// Likewise, something like this: +/// `settings:wifi/Kosagi.json` +/// Would be deconstructed into the "settings" dictionary with a key of `wifi/Kosagi.json`. +/// +/// ## Code Organization: +/// Accurate as of Nov 2021. May be subject to charge, ymmv. +/// +/// ### `frontend.rs` +/// Defines a set of modules that plug the PDDB into applications (in particular, they +/// attemp to provide a Rust-compatible `read`/`write`/`open` abstraction layer, +/// torturing the notion of `Prefix`, `Path` and `File` in Rust to fit the basis/dict/key +/// format of the PDDB). +/// +/// This is the `lib`-facing set of operations. +/// +/// ### `backend.rs` +/// Defines a set of modules that implement the PDDB itself. This is the hardware-facing set +/// of operations. +/// +/// #### `basis.rs` +/// The set of known `Basis` are tracked in the `BasisCache` structure. This is the "entry point" +/// for most operations on the PDDB, and thus most externally-visible API calls will be revealed +/// on that structure; in fact many calls on that object are just pass-through of lower level calls. +/// +/// A `BasisCach`e consists of one or more `BasisCacheEntries`. This structure manages one or more +/// `DictCacheEntry` within a `BasisCacheEntry`'s dictionary cache. +/// +/// #### `dictionary.rs` +/// The `DictCacheEntry` is defined in this file, along with the on-disk `Dictionary` +/// storage stucture. Most of the methods on `DictCacheEntry` are concerned with managing +/// keys contained within the dictionary cache. +/// +/// #### `keys.rs` +/// The `KeyCacheEntry` is defined in this file. It's a bookkeeping record for the +/// `DictCacheEntry` structure. Keys themselves are split into metadata and data +/// records. The `KeyDescriptor` is the on-disk format for key metadata. The key +/// storage is either described in `KeySmallPool` for keys less than one `VPAGE_SIZE` +/// (just shy of 4kiB), or written directly to disk as fully allocated blocks of +/// `VPAGE_SIZE` for keys larger than one `VPAGE_SIZE` unit. +/// +/// ## Threat model: +/// The user is forced to divulge "all the Basis passwords" on the device, through +/// coercion, blackmail, subpoena, customs checkpoint etc. The adversary has physical +/// access to the device, and is able to take a static disk image; they may even have +/// the opportunity to take several disk images over time and diff the images. +/// The adversary may be able to decrypt the root key of the cryptographic enclave. +/// The adversary may also be able to observe the contents of encrypted data within the +/// "system" basis, which includes some of the bookkeeping information for the PDDB, +/// as the user will have at least been forced to divulge the system basis password as +/// this is a password that every device requires and they cannot deny its existence. +/// +/// Under these conditions, it should be impossible for the adversary to conclusively prove or +/// disprove that every Basis password has been presented and unlocked for inspection +/// through forensic analysis of the PDDB alone (significantly, we cannot prevent disclosure +/// by poorly constructed end-user applications storing things like "last opened Basis" +/// convenience lists, or if the user themselves wrote a note referring to a secret +/// Basis in a less secret area). Furthermore, if a device is permanently seized by +/// the adversary for extensive analysis, any Basis whose password that has not been +/// voluntarily disclosed should be "as good as deleted". +/// +/// The PDDB also cannot protect against key loggers or surveillance cameras recording +/// key strokes as the user operates the device. Resistance to key loggers is instead +/// a problem handled by the OS, and it is up to the user to not type secret passwords +/// in areas that may be under camera surveillance. +/// +/// ## Auditor Notices: +/// There's probably a lot of things the PDDB does wrong. Here's a list of some things +/// to worry about: +/// - The device RootKeys shares one salt across all of its keys, instead of a per-key salt. +/// Currently there are only two keys in the RootKey store sharing the one salt. The concern +/// is that a migration to an ASIC base design would make eFuse bits scarce, so, the initial +/// draft of the RootKeys shares one salt to maximize key capacity. The per-key salt is slightly +/// modified by adding the key index to the salt. This isn't meant to be a robust mitigation: +/// it just prevents a naive rainbow attack from re-using its table. +/// - The bcrypt() implementation is vendored in from a Rust bcrypt crate. It hasn't been audited. +/// - The COST of 7 for bcrypt is relatively low by today's standards (should be 10). However, +/// we can't raise the cost to 10 because our CPU is slower than most modern x86 devices. There is +/// an open issue to try to improve this with hardware acceleration. The mitigation is to use +/// a longer passphrase instead of a 12 or 14-character password. +/// - The RootKey is used to decrypt a locally stored System Basis key. The key is encrypted using +/// straight AES-256 with no authentication. +/// - Secret basis keys are not stored anywhere on the device. They are all derived from a password +/// using bcrypt. The salt for the password is drawn from a "salt pool", whose index is derived from +/// a weak hash of the password itself. This means there is a chance that a salt gets re-used. However, +/// we do not store per-password salts because the existence of the salt would betray the existence of +/// a password. +/// - Disk blocks are encrypted & authenticated on a page-by-page basis using AES-GCM-SIV, with a 256-bit key. +/// - Page table entries and space update entries are salted & hinted using a 64-bit nonce + weak checksum. +/// There is a fairly high chance of a checksum collision, thus PTE decrypts are regarded as advisory and +/// not final; the PTE is not accepted as authentic until the AES-GCM-SIV behind it checks out. Free space +/// entries have no such protection, which means there is a slight chance of data loss due to a checksum +/// collision on the free space entries. +/// +/// ## General Operation: +/// The initial Basis is known as the "System" Basis. It's a low-security framework basis +/// onto which all other Basis overlay. The initial set of Dictionaries, along with their +/// Key/Value pairs, are available to any and all processes. +/// +/// Each new Basis opened will contain zero or more dictionaries. If the dictionary within +/// the new Basis has the same name as an existing dictionary, the Keys are searched in +/// the reverse order of opening. In other words, a new Basis can temporarily override or +/// mask existing Keys. When updating a Key, one may specify the following modes of +/// operation: +/// +/// - UpdateLatest: updates only the copy in the latest Basis to be opened +/// - UpdateOpened: updates copies in other currently opened Basis +/// +/// Note that `UpdateOpened` is dangerous in the sense that if you have a Basis that is +/// is currently Closed, it cannot update copies in the closed Basis. Thus any global +/// update of database schema requires a user to open any and all knows Basis so that +/// synchronization can be maintained. +/// +/// Furthermore, it is possible for a Basis to be closed on a "File" that is currently open. +/// In this case, two things happen: +/// - If a notification callback is registered, it's pinged by the PDDB. The notification +/// of closure callback is an additional feature to the typical Rust File interface. +/// - If the client attempts to read or write to any keys that span a Basis modification, +/// the now-ambiguous key operation will return a `BrokenPipe` error to the caller. +/// +/// ## General flash->key structure +/// +/// The basic unit of memory is a page (4k). Keys shorter than 4k will be packed into a single +/// page, but no allocation will ever be smaller than 4k due to the erase block size of the FLASH. +/// +/// Because the flash is always encrypted, there is also no such thing as "incremental writing of data" +/// to a page, as the presence of free space (all 1's regions) are considered a side-channel. Thus, +/// the PDDB relies heavily on caching data structures in RAM, with periodic commits of encrypted data +/// back to the flash. This means it is also very easy to lose data when the system has a panic and +/// reboots ungracefully. Thus, the structure of data at rest is journaled, so that we have some +/// chance of recovering an older snapshot of the data if there is an ungraceful power-down event. +/// +/// I believe this relative unreliability compared to more traditional storage is just a fact of life for +/// an encrypted, plausibly deniable filesystem, as you have to trade-off the side channel of partially +/// erased blocks versus the finite write lifetime of flash storage. +/// +/// When a Basis is created, it starts with 64k of space allocated to it. Most of this is blank (but encrypted +/// in flash), but it is important in a PD filesystem to claim some freespace for day-to-day operations. This +/// is because when you need to allocate more data, you have to make a guess as to what is free, and what +/// previously allocated but currently denied to exist (locked Basis look identical to free space when locked). +/// +/// Each Dictionary within a Basis starts with a 4k allocation as well, although most of it is free space. +/// Keys are then populated into dictionaries, with additional pages allocated to dictionaries as the keys +/// grow. +/// +/// Blocks on disk are mapped into a virtual PDDB space via a page table. The virtual PDDB space is 64-bit. +/// Each Basis gets its own 48-bit address space, allowing for up to 64k Basis. +/// The PDDB page table guarantees that no matter the order of allocation in FLASH, a Basis can linearly +/// access its blocks through a page table lookup. A reverse-lookup structure is constructed on boot +/// by scanning the entire PDDB, and creating a HashMap of (basis, offset) -> pddb_addr tuples. +/// +/// Physical addresses can be 32 or 64 bit based upon the specific implementation. The physical address +/// type is defined in types.rs. For Precursor, it's defined as a u32, which limits the physical address +/// size to 32 bits. This is important because Precursor operates out of a very small amount of RAM, +/// and doubling the size of the overhead bookkeeping structures to handle disk sizes that are well +/// beyond the current implementation has a real impact on the system's free memory footprint. However, +/// the PDDB is coded in a way such that one "should" be able to swap out the u32 for a u64 in the +/// "newtype" definition for a PhysAddr, and the system will work. +/// +/// Page table entries don't define themselves with a Basis -- every time a Basis is opened, the entire +/// table must be scanned, brute-force decrypting every entry against the Basis key, and seeing if the +/// checksum in the table entry matches the address. If there's a match, then that entry is recorded as +/// belonging to a given Basis. +/// +/// When a system boots, it starts with two default Basis: FastSpace, and System. These Basis are protected +/// with the "device unlock" key. System is a set of low-security data that are used +/// to configure "everyday details". FastSpace Basis is a special-case Basis that is not structured as +/// a Key/Value store. Instead, it tracks a small amount of pre-allocated "free" space, that can be +/// pulled from to grow Dictionaries without requesting that the user unlock every known Basis in the system +/// to ensure none of the valuable data is overwritten. +/// +/// The .FastSpace basis should never be equal to the total amount of free space in the system, as it +/// would betray the actual amount of data available and destroy plausible deniability. The system can +/// operate with .FastSpace set to the nil set, but it would also operate without confidentiality of hidden +/// Basis, because in order to allocate new blocks it must prompt the user to enter all known Basis passwords, +/// to avoid accidentally overwriting data that should be retained (as locked Basis would appear as free space +/// and risk being overwritten). However, in order to accelerate bookkeeping, the .FastSpace basis operates +/// with an encrypted record that rotates through a set of "clean" pages (that is, pages that have been set +/// to all 1's) in a circular buffer basis. The state of where the .FastSpace record is in the "clean" pages +/// disclose nothing other than the fact that the system has been used. +/// +/// ## Basis Deniability +/// +/// The existence of a Basis itself must be confidential; if we stored a list of encrypted Basis passwords +/// somewhere, a rubber-hose attacker would simply need to count the number of encrypted entries and commence +/// the beatings until the number of passwords that have been disclosed under duress match the number of +/// encrypted password entries. +/// +/// Herein lies a challenge. Standard password storage techniques demand that each password be stored +/// with a unique salt. The existence of this salt betrays that a password must, in essence, exist. +/// There are at least two methods to counter this: +/// +/// 1. Dummy salt entries. In this approach, each system is initialized with a random set of additional, dummy +/// salt/encrypted password combos. In the case of a rubber-hose attack, the attacker cannot know precisely +/// when to stop the beatings, as the few remaining entries could be dummies with absolutely no meaning. +/// 2. A single salt entry. In this approach, a single shared salt is used by /all/ passwords, and there is no +/// encrypted password list stored; instead, the the password as presented is directly used to decrypt +/// the Basis page table and if no valid entries are found we may conclude that the password provided has a typo. +/// +/// The advantage of (1) is that the usage of salt falls precisely within the traditional cryptographic +/// specification of bcrypt(). The disadvantages of (1) include: one still has to have a method to match +/// a presented password to a given salt entry. In a traditional user/pass login database, the username is +/// the plaintext key to match the salt. In this case, we would use a nickname used to refer to each Basis +/// to correlate to its salt entry. Thus, we'd have to come up with "garbage" plaintext nicknames +/// for a Basis that are not trivial for a rubber-hose attacker to dismiss as chaffe. I think this is Hard. +/// The alternative is to "brute force" the entire list of salts, guessing each one in sequence until one +/// is found to decrypt entries in the Basis page table. The problem with this is that bcrypt is designed to +/// be computationally slow, even for valid passwords. The complexity parameter is chosen so that it takes +/// about 1 second per iteration through the password function. Note that a faster CPU does not solve this +/// problem: if you CPU gets faster, you should increase your complexity, so one is always suffering the 1 +/// second penalty to try a password. This means brute forcing a list of salts necessarily incurs at +/// least a 1 second penalty per entry; thus each dummy basis adds a 1 second minimum overhead to unlocking +/// a new Basis database. This puts a downward pressure on the number of dummies to be included; however, the +/// number of dummies needs to be at least as large as the number of Basis we wish to plausibly deny. This +/// creates a negative incentive to using plausible deniability. +/// +/// The advantage of (2) is that when a user presents a password, we only need to run the bcrypt routine +/// once, and then we can immediately start checking the Basis page table for valid entries. The disadvantage +/// of (2) is that we are re-using a salt across all of a given user's passwords. Note that the salt itself +/// is from a TRNG, so between user devices, the salt performs its role. However, it means that an attacker +/// can generate a single rainbow table specific to a given user to reverse all of their passwords; and furthermore, +/// re-used passwords are trivially discoverable with the rainbow table. +/// +/// Approach (1) is probably what most crypto purists would adopt: you, dear user, /should/ understand that +/// cryptography is worth the wait. However, a second principle states that "users always pick convenience over +/// security". A 15-second wait is an eternity in the UX world, and would act as an effective deterrent to any +/// user ever using the PD function because it is too slow. Thus for v1 of the PDDB, we're going to try approach +/// (2), where a common salt is used across all the PDDB passwords, but the salt is /not/ re-used between devices. +/// This choice diminishes the overall security of the system in the case that a user chooses weak passwords, or re-uses +/// passwords, but in exchange for a great improvement in responsivity of the implementation. +/// +/// The final implementation uses a slight mod on (2), where the 128-bit common salt stored on disk is XOR'd +/// with the user-provided "basis name". Users are of course allowed to pick crappy names for their basis, and +/// re-use the names, but hopefully this adds a modicum of robustness against rainbow table attacks. +/// +/// +/// ## Basis Unlock Procedure +/// +/// Each Basis has a name and a passcode associated with it. The default Basis name is `.System`. +/// In addition to that, a `.FastSpace` structure is unlocked along side the default Basis. +/// These are both associated with the default system unlock passcode. +/// +/// A newly created Basis will request a name for the Basis, and a password. It is a requirement +/// that the combination of `(name, password)` be unique; this property is enforced and a system will +/// reject attempts to create identically named, identially passworded Basis. However, one can have +/// same-named Basis that have a different password, or diffently-named Basis with the same password +/// (this is generally not recommended, but it's not prohibited). +/// +/// The `name` field is XOR'd with the device-local `salt` field to form the salt for the password, +/// and the plaintext of the password itself is used as the password which is fed into the bcrypt() +/// algorithm to generate a 192-bit encrypted password, which is expanded to 256-bits using SHA-512/256, +/// and then used as the AES key for the given `(name, password)` Basis. +/// +/// +/// ## Journaling +/// +/// A major issue with the implementation is that even a small change in a data structure turns into +/// the mutation of at least an entire sector of data, with a sector being 4k. The reason is that +/// we can't take advantage of FLASH memory's property where 1's (erased) can be set to 0's on a byte-by-byte +/// basis: in order to mask free space signatures, the entire storage area is filled with random bytes. +/// Furthermore, any update to a part of data within a block should necessarily propagate to the entire +/// block changing, in order to avoid attacks where observations on which portion of ciphertext has changed +/// leading to a definitive conclusion about the state of the database records. Therefore, a large portion +/// of the database data will have to persist in RAM, and be updated only at regular, but widely-spaced-out +/// (to avoid FLASH wear-out), intervals. +/// +/// In the case that power is lost during an update, the system uses a 32-bit `journal_rev` number at the +/// top of every major database structure. If two competing copies of data are found, the one with the +/// highest `journal_rev` number wins. It may be possible to overflow the 32-bit number, so at some point a +/// garbage collection step needs to be made where any errant, lower journal rev blocks are overwritten +/// with random data; all the journal revs are confirmed to be the same; and then all set to 0 again. Power +/// should not be lost during this process; but it should be rare, and likely never needed. +/// +/// When writing to disk, a write-then-erase method is used: +/// 1. Required blocks for the update are taken out of the .FastSpace pool +/// 2. Detached data sections of structural records are written first (only database structural records can have detached data; +/// all stored data have individual journal revision fields associated with them), and into blocks +/// taken out of the .FastSpace pool. +/// 3. The head of the updated structural record is written, with the new latest journal rev noted. +/// 4. The page table is updated using the procedure outlined below. +/// 5. The old blocks are erased and overwritten with random data. +/// 6. The .FastSpace pool is updated with the newly freed blocks +/// +/// When updating the page table, it's important not to lose an entire page's worth of entries in case +/// power is lost when updating it. The following process is used to update a page table entry: +/// 0. At the end of the page table there is a circular buffer of blank pages (they are truly blank, +/// but this will not leak metadata about the contents of the PDDB, as it only does bookkeeping +/// on page table updates as a whole). This is known as the make-before-break buffer (mbbb). +/// 1. The previous mbbb entry is erased, if there is any, and the next entry is noted. +/// 2. The new page table entry is written into the mbbb +/// 3. The old page table entry in the page table itself is erased. +/// 4. The old page table entry is populated with the updated page table records. +/// +/// Detecting and Repairing a broken page table: +/// The special case of long run's of 1's (more than 32 bits in a row) is checked during the page table scan. +/// If a long run of 1's is detected, then a suspected corrupted page table update is flagged, and for that page the data +/// is pulled from the mbbb record. +/// +/// ## Precursor's Implementation-Specific Flash Memory Organization: +/// +/// ```Text +/// offset from | +/// pt_phys_base| contents (example for total PDDB len of 0x6f8_0000 or 111 MiB) +/// ------------|--------------------------------------------------- +/// 0x0000_0000 | virtual map for page at (0x0000 + data_phys_base) +/// 0x0000_0010 | virtual map for page at (0x1000 + data_phys_base) +/// 0x0000_0020 | virtual map for page at (0x2000 + data_phys_base) +/// ... +/// 0x0006_F7F0 | virtual map for page at (0x06F7_F000 + data_phys_base) +/// 0x0006_F800 | unused +/// ... +/// 0x0007_0000 | key page +/// ... +/// 0x0007_1000 | mbbb start (example of 10 pages) +/// ... +/// 0x0007_B000 | fscb start (example of 10 pages) +/// ... +/// 0x0008_5000 | data_phys_base - start of basis + dictionary + key data region +/// ``` + +// historical note: 389 hours, 11 mins elapsed since the start of the PDDB coding, and the first attempt at a hardware test -- as evidenced by the uptime of the hardware validation unit. + pub mod api; pub use api::*; pub mod frontend; diff --git a/services/pddb/src/main.rs b/services/pddb/src/main.rs index 74471bac0..a853270d6 100644 --- a/services/pddb/src/main.rs +++ b/services/pddb/src/main.rs @@ -1,67 +1,86 @@ #![cfg_attr(target_os = "none", no_std)] #![cfg_attr(target_os = "none", no_main)] -extern crate bitflags; -extern crate bitfield; - -mod api; -use api::*; -mod backend; -use backend::*; - -#[cfg(not(any(target_os = "none", target_os = "xous")))] -mod tests; -#[cfg(not(any(target_os = "none", target_os = "xous")))] -use tests::*; - -use num_traits::*; -use xous::{msg_blocking_scalar_unpack, msg_scalar_unpack}; -use core::cell::RefCell; -use std::rc::Rc; - -/// PDDB - Plausibly Deniable DataBase +/// # PDDB - Plausibly Deniable DataBase /// -/// Glossary: -/// Basis - a unionizeable dictionary that can be unlocked with a password-protected key. -/// Dictionary - similar to a "volume" on a typical filesystem. Contains a group of k/v pairs +/// ## Glossary: +/// * Basis - a unionizeable dictionary that can be unlocked with a password-protected key. +/// * Dictionary - similar to a "volume" on a typical filesystem. Contains a group of k/v pairs /// that share attributes such as access permissions -/// Key - a unique string that identifies a piece of data in the PDDB -/// Path - Rust's notion of how to locate a file, except we interpret it as a PDDB key, where +/// * Key - a unique string that identifies a piece of data in the PDDB +/// * Path - Rust's notion of how to locate a file, except we interpret it as a PDDB key, where /// the "root" directory is the dictionary, and the remainder is the "key" -/// Value - the value associated with a key, isomorphic ot a "file" in Rust -/// Open Basis - A Basis that is decryptable and known to the system. -/// Closed Basis - A Basis that is not decryptable. It's unknown to the system and potentially treated +/// * Value - the value associated with a key, isomorphic ot a "file" in Rust +/// * Open Basis - A Basis that is decryptable and known to the system. +/// * Closed Basis - A Basis that is not decryptable. It's unknown to the system and potentially treated /// identically to free disk space, e.g., it could be partially overwritten. -/// FSCB - Free Space Commit Buffer. A few pages allocated to tracking a subset of free space, +/// * FSCB - Free Space Commit Buffer. A few pages allocated to tracking a subset of free space, /// meant to accelerate PDDB operations. Creates a side-channel that can reveal that some activity /// has happened, but without disclosing what and why. Contains FastSpace and SpaceUpdate records. /// Frequently updated, so the buffer is slightly oversized, and which sector is "hot" is randomized /// for wear-levelling. -/// MBBB - Make Before Break Buffer. A set of randomly allocated pages that are a shadow copy +/// * MBBB - Make Before Break Buffer. A set of randomly allocated pages that are a shadow copy /// of a page table page. If any data exists, its contents override those of a corrupted page table. -/// FastSpace - a collection of random pages that are known to be empty. The number of pages in FastSpace +/// * FastSpace - a collection of random pages that are known to be empty. The number of pages in FastSpace /// is reduced from the absolute amount of free space available by at least a factor of FSCB_FILL_COEFFICIENT. -/// SpaceUpdate - encrypted patches to the FastSpace table. The FastSpace table is "heavyweight", and would +/// * SpaceUpdate - encrypted patches to the FastSpace table. The FastSpace table is "heavyweight", and would /// be too expensive to update on every page allocation, so SpaceUpdate is used to patch the FastSpace table. /// /// A `Path` like the following is deconstructed as follows by the PDDB: /// +/// ```Text /// Dictionary /// | Key /// | | /// --+- --------+--------------------- /// logs:matrix/alice/oct30_2021/bob.txt +/// ``` /// -/// It could equally have an arbitrary name like "logs:Matrix - alice to bob Oct 30 2021"; +/// It could equally have an arbitrary name like `logs:Matrix - alice to bob Oct 30 2021`; /// as long as the string that identifies a Key is unique, it's stored in the database /// all the same. Any valid utf-8 unicode characters are acceptable. /// /// Likewise, something like this: -/// settings:wifi/Kosagi.json -/// Would be deconstructed into the "settings" dictionary with a key of wifi/Kosagi.json. +/// `settings:wifi/Kosagi.json` +/// Would be deconstructed into the "settings" dictionary with a key of `wifi/Kosagi.json`. /// +/// ## Code Organization: +/// Accurate as of Nov 2021. May be subject to charge, ymmv. /// -/// Threat model: +/// ### `frontend.rs` +/// Defines a set of modules that plug the PDDB into applications (in particular, they +/// attemp to provide a Rust-compatible `read`/`write`/`open` abstraction layer, +/// torturing the notion of `Prefix`, `Path` and `File` in Rust to fit the basis/dict/key +/// format of the PDDB). +/// +/// This is the `lib`-facing set of operations. +/// +/// ### `backend.rs` +/// Defines a set of modules that implement the PDDB itself. This is the hardware-facing set +/// of operations. +/// +/// #### `basis.rs` +/// The set of known `Basis` are tracked in the `BasisCache` structure. This is the "entry point" +/// for most operations on the PDDB, and thus most externally-visible API calls will be revealed +/// on that structure; in fact many calls on that object are just pass-through of lower level calls. +/// +/// A `BasisCach`e consists of one or more `BasisCacheEntries`. This structure manages one or more +/// `DictCacheEntry` within a `BasisCacheEntry`'s dictionary cache. +/// +/// #### `dictionary.rs` +/// The `DictCacheEntry` is defined in this file, along with the on-disk `Dictionary` +/// storage stucture. Most of the methods on `DictCacheEntry` are concerned with managing +/// keys contained within the dictionary cache. +/// +/// #### `keys.rs` +/// The `KeyCacheEntry` is defined in this file. It's a bookkeeping record for the +/// `DictCacheEntry` structure. Keys themselves are split into metadata and data +/// records. The `KeyDescriptor` is the on-disk format for key metadata. The key +/// storage is either described in `KeySmallPool` for keys less than one `VPAGE_SIZE` +/// (just shy of 4kiB), or written directly to disk as fully allocated blocks of +/// `VPAGE_SIZE` for keys larger than one `VPAGE_SIZE` unit. +/// +/// ## Threat model: /// The user is forced to divulge "all the Basis passwords" on the device, through /// coercion, blackmail, subpoena, customs checkpoint etc. The adversary has physical /// access to the device, and is able to take a static disk image; they may even have @@ -86,7 +105,7 @@ use std::rc::Rc; /// a problem handled by the OS, and it is up to the user to not type secret passwords /// in areas that may be under camera surveillance. /// -/// Auditor Notices: +/// ## Auditor Notices: /// There's probably a lot of things the PDDB does wrong. Here's a list of some things /// to worry about: /// - The device RootKeys shares one salt across all of its keys, instead of a per-key salt. @@ -114,7 +133,7 @@ use std::rc::Rc; /// entries have no such protection, which means there is a slight chance of data loss due to a checksum /// collision on the free space entries. /// -/// General Operation: +/// ## General Operation: /// The initial Basis is known as the "System" Basis. It's a low-security framework basis /// onto which all other Basis overlay. The initial set of Dictionaries, along with their /// Key/Value pairs, are available to any and all processes. @@ -140,7 +159,7 @@ use std::rc::Rc; /// - If the client attempts to read or write to any keys that span a Basis modification, /// the now-ambiguous key operation will return a `BrokenPipe` error to the caller. /// -/// General flash->key structure +/// ## General flash->key structure /// /// The basic unit of memory is a page (4k). Keys shorter than 4k will be packed into a single /// page, but no allocation will ever be smaller than 4k due to the erase block size of the FLASH. @@ -201,9 +220,7 @@ use std::rc::Rc; /// to all 1's) in a circular buffer basis. The state of where the .FastSpace record is in the "clean" pages /// disclose nothing other than the fact that the system has been used. /// -/// -/// -/// Basis Deniability +/// ## Basis Deniability /// /// The existence of a Basis itself must be confidential; if we stored a list of encrypted Basis passwords /// somewhere, a rubber-hose attacker would simply need to count the number of encrypted entries and commence @@ -258,7 +275,7 @@ use std::rc::Rc; /// re-use the names, but hopefully this adds a modicum of robustness against rainbow table attacks. /// /// -/// Basis Unlock Procedure +/// ## Basis Unlock Procedure /// /// Each Basis has a name and a passcode associated with it. The default Basis name is `.System`. /// In addition to that, a `.FastSpace` structure is unlocked along side the default Basis. @@ -276,7 +293,7 @@ use std::rc::Rc; /// and then used as the AES key for the given `(name, password)` Basis. /// /// -/// Journaling +/// ## Journaling /// /// A major issue with the implementation is that even a small change in a data structure turns into /// the mutation of at least an entire sector of data, with a sector being 4k. The reason is that @@ -320,8 +337,9 @@ use std::rc::Rc; /// If a long run of 1's is detected, then a suspected corrupted page table update is flagged, and for that page the data /// is pulled from the mbbb record. /// -/// Precursor's Implementation-Specific Flash Memory Organization: +/// ## Precursor's Implementation-Specific Flash Memory Organization: /// +/// ```Text /// offset from | /// pt_phys_base| contents (example for total PDDB len of 0x6f8_0000 or 111 MiB) /// ------------|--------------------------------------------------- @@ -339,9 +357,28 @@ use std::rc::Rc; /// 0x0007_B000 | fscb start (example of 10 pages) /// ... /// 0x0008_5000 | data_phys_base - start of basis + dictionary + key data region +/// ``` // historical note: 389 hours, 11 mins elapsed since the start of the PDDB coding, and the first attempt at a hardware test -- as evidenced by the uptime of the hardware validation unit. +extern crate bitflags; +extern crate bitfield; + +mod api; +use api::*; +mod backend; +use backend::*; + +#[cfg(not(any(target_os = "none", target_os = "xous")))] +mod tests; +#[cfg(not(any(target_os = "none", target_os = "xous")))] +use tests::*; + +use num_traits::*; +use xous::{msg_blocking_scalar_unpack, msg_scalar_unpack}; +use core::cell::RefCell; +use std::rc::Rc; + #[xous::xous_main] fn xmain() -> ! { log_server::init_wait().unwrap(); @@ -411,10 +448,11 @@ fn xmain() -> ! { } */ /* list of test cases: - genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k. - delete/add consistency: general integrity, delete a dictionary, then add a dictionary. + [done] genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k. + [done] delete/add consistency: general integrity, delete a dictionary, then add a dictionary. in-place update consistency: general integrity then patch all keys with a new test pattern extend update consistency: general integrity then patch all keys with a longer test pattern + key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data. basis search: create basis A, populate with general integrity. create basis B, add test entries. hide basis B, confirm original A; mount basis B, confirm B overlay. */