reorganize documentation

This commit is contained in:
bunnie 2021-11-23 15:47:40 +08:00
parent 9c54152f4d
commit cbf7ee032f
5 changed files with 619 additions and 242 deletions

View file

@ -1,21 +1,11 @@
use crate::api::*;
use super::*;
use core::ops::{Deref, DerefMut};
use core::mem::size_of;
use std::convert::TryInto;
use aes_gcm_siv::{Aes256GcmSiv, Key};
use aes_gcm_siv::aead::NewAead;
use std::iter::IntoIterator;
use std::collections::{BinaryHeap, HashMap, HashSet};
use std::io::{Result, Error, ErrorKind};
/// # The Organization of Basis Data
///
/// ## Overview
/// In basis space, the BasisRoot is located at VPAGE #1 (VPAGE #0 is always invalid).
/// A VPAGE is 0xFE0 (4,064) bytes long, which is equal to a PAGE of 4k minus 32 bytes of encryption+journal
/// overhead.
///
/// NB: The factors of 4,064 are: 1, 2, 4, 8, 16, 32, 127, 254, 508, 1016, 2032, 4064
/// *NB: The factors of 4,064 are: 1, 2, 4, 8, 16, 32, 127, 254, 508, 1016, 2032, 4064*
///
/// AAD associated with the BasisRoot consist of a bytewise concatenation of:
/// - Basis name
@ -53,8 +43,8 @@ use std::io::{Result, Error, ErrorKind};
/// in that dictionary with no extra bookkeeping cost once the dictionary is added.
///
///
/// Basis Virtual Memory Layout
///
/// ## Basis Virtual Memory Layout
///```Text
/// | Start Address | |
/// |------------------------|-------------------------------------------|
/// | 0x0000_0000_0000_0000 | Invalid -- VPAGE 0 reserved for Option<> |
@ -78,12 +68,13 @@ use std::io::{Result, Error, ErrorKind};
/// | 0x0000_FE00_0000_0000 | Large data pool start (~16mm TiB) |
/// | | - Demand-allocated, bump-pointer |
/// | | currently no defrag |
/// ```
///
/// Note that each Basis has its own memory section, and you can have "many" orthogonal Basis without
/// a collision -- the AES keyspace is 128 bits, so you have a decent chance of no collisions
/// even with a few billion Basis concurrently existing in the filesystem.
///
/// Memory Pools
/// ## Memory Pools
///
/// Key data is split into three categories of sizes: small, medium, and large; but the implementation
/// currently only deals with small and large keys. The thresholds are subject to tuning, but
@ -101,7 +92,7 @@ use std::io::{Result, Error, ErrorKind};
///
/// Medium keys have a TBD implementation, and are currently directed to the large pool for now.
///
/// The Alignment and Serialization Chronicles
/// ## The Alignment and Serialization Chronicles
///
/// We're using Repr(C) and alignment to 64-bits to create a consistent "FFI" layout; we use an unsafe cast
/// to [u8] as our method to serialize the structure, which means we could be subject to breakage if the Rust
@ -131,7 +122,7 @@ use std::io::{Result, Error, ErrorKind};
/// storage structure that can persist through future versions of Rust and also other implementations in other
/// languages.
///
/// Known Repr(C) footguns:
/// ## Known Repr(C) footguns:
/// - When you start laying in 64-bit types, stuff has to be 64-bit aligned, or else you'll start to get
/// uninitialized padding data inserted, which can leak stack data in the serialization process.
/// - Don't use anything that's not native to C. In particular, for primitives that we want to be "Option"
@ -139,6 +130,18 @@ use std::io::{Result, Error, ErrorKind};
/// structure and serialize/deserialize that into the correct Rust structure. See
/// https://doc.rust-lang.org/nomicon/other-reprs.html for a citation on that.
use crate::api::*;
use super::*;
use core::ops::{Deref, DerefMut};
use core::mem::size_of;
use std::convert::TryInto;
use aes_gcm_siv::{Aes256GcmSiv, Key};
use aes_gcm_siv::aead::NewAead;
use std::iter::IntoIterator;
use std::collections::{BinaryHeap, HashMap, HashSet};
use std::io::{Result, Error, ErrorKind};
pub(crate) const SMALL_POOL_START: u64 = 0x0000_003F_8000_0000;
pub(crate) const SMALL_POOL_END: u64 = 0x0000_007E_FF02_0000;
pub(crate) const SMALL_POOL_STRIDE: u64 = 0xFE_0000;
@ -243,17 +246,9 @@ impl BasisCache {
if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
basis.age = basis.age.saturating_add(1);
if basis.dicts.get(&String::from(name)).is_some() {
// quick exit if we see the dictionary is hot in the cache
if basis.ensure_dict_in_cache(hw, name) {
return Err(Error::new(ErrorKind::AlreadyExists, "Dictionary already exists"));
} else {
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
// doesn't exist. Do a comprehensive search if our cache isn't complete.
if basis.num_dicts as usize != basis.dicts.len() {
if basis.dict_deep_search(hw, name).is_some() {
return Err(Error::new(ErrorKind::AlreadyExists, "Dictionary already exists"));
}
}
}
basis.clean = false;
// allocate a vpage offset for the dictionary
@ -361,16 +356,8 @@ impl BasisCache {
) -> Result<usize> {
if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.dicts.contains_key(dict) {
if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) {
if dict_record.flags.valid() {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad);
basis.dicts.insert(dict_name, dcache);
}
} else {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
if !basis.ensure_dict_in_cache(hw, dict) {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
if let Some(dict_entry) = basis.dicts.get_mut(dict) {
if dict_entry.ensure_key_entry(hw, &mut basis.v2p_map, &basis.cipher, key) {
@ -435,16 +422,8 @@ impl BasisCache {
) -> Result<()> {
if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.dicts.contains_key(dict) {
if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) {
if dict_record.flags.valid() {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad);
basis.dicts.insert(dict_name, dcache);
}
} else {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
if !basis.ensure_dict_in_cache(hw, dict) {
return Err(Error::new(ErrorKind::NotFound, "dictionary not found"));
}
if let Some(dict_entry) = basis.dicts.get_mut(dict) {
basis.age = basis.age.saturating_add(1);
@ -497,17 +476,9 @@ impl BasisCache {
};
if let Some(basis_index) = self.select_basis(basis_name) {
let basis = &mut self.cache[basis_index];
if !basis.dicts.contains_key(dict) {
if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) {
if dict_record.flags.valid() {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad);
basis.dicts.insert(dict_name, dcache);
}
} else {
pages_needed += 1;
pages_needed += reserved_pages;
}
if !basis.ensure_dict_in_cache(hw, dict) {
pages_needed += 1;
pages_needed += reserved_pages;
}
if let Some(dict_entry) = basis.dicts.get(dict) {
// see if we need to make a kcache entry
@ -565,24 +536,7 @@ impl BasisCache {
}
// now actually do the update
if let Some(basis_index) = self.select_basis(basis_name) {
let mut dict_found = false;
{ // resolve the basis here, potentially mutating things
let basis = &mut self.cache[basis_index];
basis.age = basis.age.saturating_add(1);
basis.clean = false;
if !basis.dicts.contains_key(dict) {
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
// doesn't exist. Do a comprehensive search if our cache isn't complete.
if let Some((index, dict_record)) = basis.dict_deep_search(hw, dict) {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &basis.aad);
basis.dicts.insert(dict_name, dcache);
dict_found = true;
}
} else {
dict_found = true;
}
}
let dict_found = (&mut self.cache[basis_index]).ensure_dict_in_cache(hw, dict);
if !dict_found { // now that we're clear of the deep search, mutate the basis if we are sure it's not there
self.dict_add(hw, dict, basis_name).expect("couldn't add dictionary");
}
@ -594,6 +548,8 @@ impl BasisCache {
}
// refetch the basis here to avoid the re-borrow problem, now that all the potential dict cache mutations are done
let basis = &mut self.cache[basis_index];
basis.age = basis.age.saturating_add(1);
basis.clean = false;
// now do the sync
if let Some(dict_entry) = basis.dicts.get_mut(dict) {
let updated_ptr = dict_entry.key_update(hw, &mut basis.v2p_map,
@ -727,19 +683,6 @@ impl BasisCacheEntry {
}
}
/// allocate a pointer data in the large pool, of length `amount`. "always" succeeds because...
/// there's 16 million terabytes of large pool to allocate before you run out?
/*
// this function is actually implemented inside the "key_update()" code - as a large key is allocated,
// the pointer is bumped along within the update code. AFAIK, nobody else should be calling this, so, let's
// comment it out, and if it becomes necessary for some reason let's take a good hard look at assumptions.
// In particular, this function might become necessary if disk caching is implemented for large keys.
pub(crate) fn large_pool_alloc(&mut self, amount: u64) -> u64 {
let alloc_ptr = self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START));
self.large_alloc_ptr = Some(self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)) + PageAlignedVa::from(amount));
return alloc_ptr.as_u64()
}*/
/// do a deep scan of all the dictionaries and keys and attempt to populate all the caches
pub(crate) fn populate_caches(&mut self, hw: &mut PddbOs) {
let mut try_entry = 1;
@ -779,21 +722,7 @@ impl BasisCacheEntry {
/// through compounded data and re-writing partias sectors, and because of this, initially,
/// the `paranoid` erase is `unimplemented`.
pub(crate) fn dict_delete(&mut self, hw: &mut PddbOs, name: &str, paranoid: bool) -> Result<()> {
let mut dict_found = false;
if !self.dicts.contains_key(name) {
log::info!("dict delete: key not in cache {}", name);
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
// doesn't exist. Do a comprehensive search if our cache isn't complete.
if let Some((index, dict_record)) = self.dict_deep_search(hw, name) {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &self.aad);
self.dicts.insert(dict_name, dcache);
dict_found = true;
}
} else {
dict_found = true;
}
if dict_found {
if self.ensure_dict_in_cache(hw, name) {
let dcache = self.dicts.get_mut(name).expect("entry was ensured, but somehow missing");
//log::info!("dcache {} index {}, key_count {}", name, dcache.index, dcache.key_count);
// ensure all the keys are in RAM
@ -850,6 +779,8 @@ impl BasisCacheEntry {
}
}
/// Called to compute the next free index. Normally this will get filled during a disk search for
/// cache entries, but it could be None if a new dict entry was allocated and no other entries were loaded.
pub(crate) fn dict_get_free_offset(&mut self, hw: &mut PddbOs) -> u32 {
if let Some(offset) = self.free_dict_offset.take() {
return offset;
@ -1088,6 +1019,39 @@ impl BasisCacheEntry {
self.clean = true;
}
}
/// This function ensures a dictionary is in the cache; if not, it will load its entry.
pub(crate) fn ensure_dict_in_cache(&mut self, hw: &mut PddbOs, name: &str) -> bool {
let mut dict_found = false;
if !self.dicts.contains_key(name) {
log::info!("dict: key not in cache {}", name);
// if the dictionary doesn't exist in our cache it doesn't necessarily mean it
// doesn't exist. Do a comprehensive search if our cache isn't complete.
if let Some((index, dict_record)) = self.dict_deep_search(hw, name) {
let dict_name = String::from(cstr_to_string(&dict_record.name));
let dcache = DictCacheEntry::new(dict_record, index as usize, &self.aad);
self.dicts.insert(dict_name, dcache);
dict_found = true;
}
} else {
dict_found = true;
}
dict_found
}
// allocate a pointer data in the large pool, of length `amount`. "always" succeeds because...
// there's 16 million terabytes of large pool to allocate before you run out?
/*
// this function is actually implemented inside the "key_update()" code - as a large key is allocated,
// the pointer is bumped along within the update code. AFAIK, nobody else should be calling this, so, let's
// comment it out, and if it becomes necessary for some reason let's take a good hard look at assumptions.
// In particular, this function might become necessary if disk caching is implemented for large keys.
pub(crate) fn large_pool_alloc(&mut self, amount: u64) -> u64 {
let alloc_ptr = self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START));
self.large_alloc_ptr = Some(self.large_alloc_ptr.unwrap_or(PageAlignedVa::from(LARGE_POOL_START)) + PageAlignedVa::from(amount));
return alloc_ptr.as_u64()
}*/
}
// ****

View file

@ -17,93 +17,9 @@ bitfield! {
pub valid, set_valid: 0;
}
#[derive(PartialEq, Eq)]
pub(crate) enum FreeKeyCases {
LeftAdjacent,
RightAdjacent,
Within,
LessThan,
GreaterThan,
}
#[derive(Eq, Copy, Clone)]
pub(crate) struct FreeKeyRange {
/// This index should be free
pub(crate) start: u32,
/// Additional free keys after the start one. Run = 0 means just the start key is free, and the
/// next one should be used. Run = 2 means {start, start+1} are free, etc.
pub(crate) run: u32,
}
impl FreeKeyRange {
pub(crate) fn compare_to(&self, index: u32) -> FreeKeyCases {
if self.start > 1 && index < self.start - 1 {
FreeKeyCases::LessThan
} else if self.start > 0 && index == self.start - 1 {
FreeKeyCases::LeftAdjacent
} else if index >= self.start && index <= self.start + self.run {
FreeKeyCases::Within
} else if index == self.start + self.run + 1 {
FreeKeyCases::RightAdjacent
} else {
FreeKeyCases::GreaterThan
}
}
}
impl Ord for FreeKeyRange {
fn cmp(&self, other: &Self) -> Ordering {
// note the reverse order -- so we can sort as a "min-heap"
other.start.cmp(&self.start)
}
}
impl PartialOrd for FreeKeyRange {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl PartialEq for FreeKeyRange {
fn eq(&self, other: &Self) -> bool {
self.start == other.start
}
}
/// stashed copy of a decrypted page. The copy here must always match
/// what's actually on disk; do not mutate it and expect it to sync with the disk.
/// Remember to invalidate this if the data are
/// This is stored with the journal number on top.
/// What the four possibilities of cache vs pp mean:
/// Some(cache) & Some(cache_pp) -> valid cache and pp
/// None(cache) & Some(cache_pp) -> the page was allocated; but never used, or was erased (it's free for you to use it); alternately, it was corrupted
/// Some(cache) & None(cache_pp) -> invalid, internal error
/// None(cache) & None(cache_pp) -> the basis mapping didn't exist: we've never requested this page before.
pub(crate) struct PlaintextCache {
/// a page of data, stored with the Journal rev on top
pub(crate) data: Option<Vec::<u8>>,
/// the page the cache corresponds to
pub(crate) tag: Option<PhysPage>,
}
impl PlaintextCache {
pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv, aad: &[u8],
req_vaddr: VirtAddr
) {
if let Some(pp) = v2p_map.get(&req_vaddr) {
let mut fill_needed = false;
if let Some(tag) = self.tag {
if tag.page_number() != pp.page_number() {
fill_needed = true;
}
} else if self.tag.is_none() {
fill_needed = true;
}
if fill_needed {
self.data = hw.data_decrypt_page(&cipher, &aad, pp);
self.tag = Some(*pp);
}
} else {
self.data = None;
self.tag = None;
}
}
}
/// RAM based copy of the dictionary structures on disk.
/// RAM based copy of the dictionary structures on disk. Most of the methods on this function operate on
/// keys within the Dictionary. Operations on the Dictionary itself originate from the containing Basis
/// structure.
pub(crate) struct DictCacheEntry {
/// Use this to compute the virtual address of the dictionary's location
/// multiply this by DICT_VSIZE to get at the virtual address. This /could/ be a
@ -824,6 +740,10 @@ impl DictCacheEntry {
}
}
}
/// Derives the index of a Small Pool storage block given the key cache entry and the dictionary index.
/// The index maps into the small_pool array, which itself maps 1:1 onto blocks inside the small pool
/// memory space.
pub(crate) fn small_storage_index_from_key(kcache: &KeyCacheEntry, dict_index: u32) -> Option<usize> {
let storage_base = dict_index as u64 * SMALL_POOL_STRIDE + SMALL_POOL_START;
let storage_end = storage_base + SMALL_POOL_STRIDE;
@ -835,7 +755,8 @@ pub(crate) fn small_storage_index_from_key(kcache: &KeyCacheEntry, dict_index: u
}
}
#[derive(Debug)]
/// On-disk representation of the dictionary header.
/// On-disk representation of the dictionary header. This structure is mainly for archival/unarchival
/// purposes. To "functionalize" a stored disk entry, it needs to be deserialized into a DictionaryCacheEntry.
#[repr(C, align(8))]
pub(crate) struct Dictionary {
/// Reserved for flags on the record entry
@ -899,3 +820,91 @@ impl<'a> Default for DictKeyVpage {
}
}
}
#[derive(PartialEq, Eq)]
pub(crate) enum FreeKeyCases {
LeftAdjacent,
RightAdjacent,
Within,
LessThan,
GreaterThan,
}
#[derive(Eq, Copy, Clone)]
pub(crate) struct FreeKeyRange {
/// This index should be free
pub(crate) start: u32,
/// Additional free keys after the start one. Run = 0 means just the start key is free, and the
/// next one should be used. Run = 2 means {start, start+1} are free, etc.
pub(crate) run: u32,
}
impl FreeKeyRange {
pub(crate) fn compare_to(&self, index: u32) -> FreeKeyCases {
if self.start > 1 && index < self.start - 1 {
FreeKeyCases::LessThan
} else if self.start > 0 && index == self.start - 1 {
FreeKeyCases::LeftAdjacent
} else if index >= self.start && index <= self.start + self.run {
FreeKeyCases::Within
} else if index == self.start + self.run + 1 {
FreeKeyCases::RightAdjacent
} else {
FreeKeyCases::GreaterThan
}
}
}
impl Ord for FreeKeyRange {
fn cmp(&self, other: &Self) -> Ordering {
// note the reverse order -- so we can sort as a "min-heap"
other.start.cmp(&self.start)
}
}
impl PartialOrd for FreeKeyRange {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl PartialEq for FreeKeyRange {
fn eq(&self, other: &Self) -> bool {
self.start == other.start
}
}
/// stashed copy of a decrypted page. The copy here must always match
/// what's actually on disk; do not mutate it and expect it to sync with the disk.
/// Remember to invalidate this if the data are
/// This is stored with the journal number on top.
/// What the four possibilities of cache vs pp mean:
/// Some(cache) & Some(cache_pp) -> valid cache and pp
/// None(cache) & Some(cache_pp) -> the page was allocated; but never used, or was erased (it's free for you to use it); alternately, it was corrupted
/// Some(cache) & None(cache_pp) -> invalid, internal error
/// None(cache) & None(cache_pp) -> the basis mapping didn't exist: we've never requested this page before.
pub(crate) struct PlaintextCache {
/// a page of data, stored with the Journal rev on top
pub(crate) data: Option<Vec::<u8>>,
/// the page the cache corresponds to
pub(crate) tag: Option<PhysPage>,
}
impl PlaintextCache {
pub fn fill(&mut self, hw: &mut PddbOs, v2p_map: &HashMap::<VirtAddr, PhysPage>, cipher: &Aes256GcmSiv, aad: &[u8],
req_vaddr: VirtAddr
) {
if let Some(pp) = v2p_map.get(&req_vaddr) {
let mut fill_needed = false;
if let Some(tag) = self.tag {
if tag.page_number() != pp.page_number() {
fill_needed = true;
}
} else if self.tag.is_none() {
fill_needed = true;
}
if fill_needed {
self.data = hw.data_decrypt_page(&cipher, &aad, pp);
self.tag = Some(*pp);
}
} else {
self.data = None;
self.tag = None;
}
}
}

View file

@ -16,8 +16,9 @@ bitfield! {
pub unresolved, set_unresolved: 1;
}
/// On-disk representation of the Key. Note that the storage on disk is mis-aligned, so
/// any deserialization must essentially come with a copy step to line up the record.
/// On-disk representation of the Key. Note that the storage on disk is mis-aligned relative
/// to Rust's expecatation of in-RAM format, so any deserialization must essentially come with
/// a copy step to re-align the record to meet Rust's placement rules.
#[repr(C, align(8))]
pub(crate) struct KeyDescriptor {
/// virtual address of the key's start
@ -63,6 +64,11 @@ impl DerefMut for KeyDescriptor {
}
}
/// In-RAM representation of a key. This file defines the storage for the KeyCacheEntry; most of the structure
/// manipulations happen inside `dictionary.rs`, in part because to locate a Key in absolute memory space you need
/// to know what Dictionary it comes from. This is a point to consider for a refactor: if we pull some info about
/// the containing Dictionary into the key, we could associate more methods with the data structure. However, this
/// means duplicating the dictionary index, a field that can then get out of sync.
pub(crate) struct KeyCacheEntry {
pub(crate) start: u64,
pub(crate) len: u64,

View file

@ -1,5 +1,365 @@
#![cfg_attr(target_os = "none", no_std)]
/// # PDDB - Plausibly Deniable DataBase
///
/// ## Glossary:
/// * Basis - a unionizeable dictionary that can be unlocked with a password-protected key.
/// * Dictionary - similar to a "volume" on a typical filesystem. Contains a group of k/v pairs
/// that share attributes such as access permissions
/// * Key - a unique string that identifies a piece of data in the PDDB
/// * Path - Rust's notion of how to locate a file, except we interpret it as a PDDB key, where
/// the "root" directory is the dictionary, and the remainder is the "key"
/// * Value - the value associated with a key, isomorphic ot a "file" in Rust
/// * Open Basis - A Basis that is decryptable and known to the system.
/// * Closed Basis - A Basis that is not decryptable. It's unknown to the system and potentially treated
/// identically to free disk space, e.g., it could be partially overwritten.
/// * FSCB - Free Space Commit Buffer. A few pages allocated to tracking a subset of free space,
/// meant to accelerate PDDB operations. Creates a side-channel that can reveal that some activity
/// has happened, but without disclosing what and why. Contains FastSpace and SpaceUpdate records.
/// Frequently updated, so the buffer is slightly oversized, and which sector is "hot" is randomized
/// for wear-levelling.
/// * MBBB - Make Before Break Buffer. A set of randomly allocated pages that are a shadow copy
/// of a page table page. If any data exists, its contents override those of a corrupted page table.
/// * FastSpace - a collection of random pages that are known to be empty. The number of pages in FastSpace
/// is reduced from the absolute amount of free space available by at least a factor of FSCB_FILL_COEFFICIENT.
/// * SpaceUpdate - encrypted patches to the FastSpace table. The FastSpace table is "heavyweight", and would
/// be too expensive to update on every page allocation, so SpaceUpdate is used to patch the FastSpace table.
///
/// A `Path` like the following is deconstructed as follows by the PDDB:
///
/// ```Text
/// Dictionary
/// | Key
/// | |
/// --+- --------+---------------------
/// logs:matrix/alice/oct30_2021/bob.txt
/// ```
///
/// It could equally have an arbitrary name like `logs:Matrix - alice to bob Oct 30 2021`;
/// as long as the string that identifies a Key is unique, it's stored in the database
/// all the same. Any valid utf-8 unicode characters are acceptable.
///
/// Likewise, something like this:
/// `settings:wifi/Kosagi.json`
/// Would be deconstructed into the "settings" dictionary with a key of `wifi/Kosagi.json`.
///
/// ## Code Organization:
/// Accurate as of Nov 2021. May be subject to charge, ymmv.
///
/// ### `frontend.rs`
/// Defines a set of modules that plug the PDDB into applications (in particular, they
/// attemp to provide a Rust-compatible `read`/`write`/`open` abstraction layer,
/// torturing the notion of `Prefix`, `Path` and `File` in Rust to fit the basis/dict/key
/// format of the PDDB).
///
/// This is the `lib`-facing set of operations.
///
/// ### `backend.rs`
/// Defines a set of modules that implement the PDDB itself. This is the hardware-facing set
/// of operations.
///
/// #### `basis.rs`
/// The set of known `Basis` are tracked in the `BasisCache` structure. This is the "entry point"
/// for most operations on the PDDB, and thus most externally-visible API calls will be revealed
/// on that structure; in fact many calls on that object are just pass-through of lower level calls.
///
/// A `BasisCach`e consists of one or more `BasisCacheEntries`. This structure manages one or more
/// `DictCacheEntry` within a `BasisCacheEntry`'s dictionary cache.
///
/// #### `dictionary.rs`
/// The `DictCacheEntry` is defined in this file, along with the on-disk `Dictionary`
/// storage stucture. Most of the methods on `DictCacheEntry` are concerned with managing
/// keys contained within the dictionary cache.
///
/// #### `keys.rs`
/// The `KeyCacheEntry` is defined in this file. It's a bookkeeping record for the
/// `DictCacheEntry` structure. Keys themselves are split into metadata and data
/// records. The `KeyDescriptor` is the on-disk format for key metadata. The key
/// storage is either described in `KeySmallPool` for keys less than one `VPAGE_SIZE`
/// (just shy of 4kiB), or written directly to disk as fully allocated blocks of
/// `VPAGE_SIZE` for keys larger than one `VPAGE_SIZE` unit.
///
/// ## Threat model:
/// The user is forced to divulge "all the Basis passwords" on the device, through
/// coercion, blackmail, subpoena, customs checkpoint etc. The adversary has physical
/// access to the device, and is able to take a static disk image; they may even have
/// the opportunity to take several disk images over time and diff the images.
/// The adversary may be able to decrypt the root key of the cryptographic enclave.
/// The adversary may also be able to observe the contents of encrypted data within the
/// "system" basis, which includes some of the bookkeeping information for the PDDB,
/// as the user will have at least been forced to divulge the system basis password as
/// this is a password that every device requires and they cannot deny its existence.
///
/// Under these conditions, it should be impossible for the adversary to conclusively prove or
/// disprove that every Basis password has been presented and unlocked for inspection
/// through forensic analysis of the PDDB alone (significantly, we cannot prevent disclosure
/// by poorly constructed end-user applications storing things like "last opened Basis"
/// convenience lists, or if the user themselves wrote a note referring to a secret
/// Basis in a less secret area). Furthermore, if a device is permanently seized by
/// the adversary for extensive analysis, any Basis whose password that has not been
/// voluntarily disclosed should be "as good as deleted".
///
/// The PDDB also cannot protect against key loggers or surveillance cameras recording
/// key strokes as the user operates the device. Resistance to key loggers is instead
/// a problem handled by the OS, and it is up to the user to not type secret passwords
/// in areas that may be under camera surveillance.
///
/// ## Auditor Notices:
/// There's probably a lot of things the PDDB does wrong. Here's a list of some things
/// to worry about:
/// - The device RootKeys shares one salt across all of its keys, instead of a per-key salt.
/// Currently there are only two keys in the RootKey store sharing the one salt. The concern
/// is that a migration to an ASIC base design would make eFuse bits scarce, so, the initial
/// draft of the RootKeys shares one salt to maximize key capacity. The per-key salt is slightly
/// modified by adding the key index to the salt. This isn't meant to be a robust mitigation:
/// it just prevents a naive rainbow attack from re-using its table.
/// - The bcrypt() implementation is vendored in from a Rust bcrypt crate. It hasn't been audited.
/// - The COST of 7 for bcrypt is relatively low by today's standards (should be 10). However,
/// we can't raise the cost to 10 because our CPU is slower than most modern x86 devices. There is
/// an open issue to try to improve this with hardware acceleration. The mitigation is to use
/// a longer passphrase instead of a 12 or 14-character password.
/// - The RootKey is used to decrypt a locally stored System Basis key. The key is encrypted using
/// straight AES-256 with no authentication.
/// - Secret basis keys are not stored anywhere on the device. They are all derived from a password
/// using bcrypt. The salt for the password is drawn from a "salt pool", whose index is derived from
/// a weak hash of the password itself. This means there is a chance that a salt gets re-used. However,
/// we do not store per-password salts because the existence of the salt would betray the existence of
/// a password.
/// - Disk blocks are encrypted & authenticated on a page-by-page basis using AES-GCM-SIV, with a 256-bit key.
/// - Page table entries and space update entries are salted & hinted using a 64-bit nonce + weak checksum.
/// There is a fairly high chance of a checksum collision, thus PTE decrypts are regarded as advisory and
/// not final; the PTE is not accepted as authentic until the AES-GCM-SIV behind it checks out. Free space
/// entries have no such protection, which means there is a slight chance of data loss due to a checksum
/// collision on the free space entries.
///
/// ## General Operation:
/// The initial Basis is known as the "System" Basis. It's a low-security framework basis
/// onto which all other Basis overlay. The initial set of Dictionaries, along with their
/// Key/Value pairs, are available to any and all processes.
///
/// Each new Basis opened will contain zero or more dictionaries. If the dictionary within
/// the new Basis has the same name as an existing dictionary, the Keys are searched in
/// the reverse order of opening. In other words, a new Basis can temporarily override or
/// mask existing Keys. When updating a Key, one may specify the following modes of
/// operation:
///
/// - UpdateLatest: updates only the copy in the latest Basis to be opened
/// - UpdateOpened: updates copies in other currently opened Basis
///
/// Note that `UpdateOpened` is dangerous in the sense that if you have a Basis that is
/// is currently Closed, it cannot update copies in the closed Basis. Thus any global
/// update of database schema requires a user to open any and all knows Basis so that
/// synchronization can be maintained.
///
/// Furthermore, it is possible for a Basis to be closed on a "File" that is currently open.
/// In this case, two things happen:
/// - If a notification callback is registered, it's pinged by the PDDB. The notification
/// of closure callback is an additional feature to the typical Rust File interface.
/// - If the client attempts to read or write to any keys that span a Basis modification,
/// the now-ambiguous key operation will return a `BrokenPipe` error to the caller.
///
/// ## General flash->key structure
///
/// The basic unit of memory is a page (4k). Keys shorter than 4k will be packed into a single
/// page, but no allocation will ever be smaller than 4k due to the erase block size of the FLASH.
///
/// Because the flash is always encrypted, there is also no such thing as "incremental writing of data"
/// to a page, as the presence of free space (all 1's regions) are considered a side-channel. Thus,
/// the PDDB relies heavily on caching data structures in RAM, with periodic commits of encrypted data
/// back to the flash. This means it is also very easy to lose data when the system has a panic and
/// reboots ungracefully. Thus, the structure of data at rest is journaled, so that we have some
/// chance of recovering an older snapshot of the data if there is an ungraceful power-down event.
///
/// I believe this relative unreliability compared to more traditional storage is just a fact of life for
/// an encrypted, plausibly deniable filesystem, as you have to trade-off the side channel of partially
/// erased blocks versus the finite write lifetime of flash storage.
///
/// When a Basis is created, it starts with 64k of space allocated to it. Most of this is blank (but encrypted
/// in flash), but it is important in a PD filesystem to claim some freespace for day-to-day operations. This
/// is because when you need to allocate more data, you have to make a guess as to what is free, and what
/// previously allocated but currently denied to exist (locked Basis look identical to free space when locked).
///
/// Each Dictionary within a Basis starts with a 4k allocation as well, although most of it is free space.
/// Keys are then populated into dictionaries, with additional pages allocated to dictionaries as the keys
/// grow.
///
/// Blocks on disk are mapped into a virtual PDDB space via a page table. The virtual PDDB space is 64-bit.
/// Each Basis gets its own 48-bit address space, allowing for up to 64k Basis.
/// The PDDB page table guarantees that no matter the order of allocation in FLASH, a Basis can linearly
/// access its blocks through a page table lookup. A reverse-lookup structure is constructed on boot
/// by scanning the entire PDDB, and creating a HashMap of (basis, offset) -> pddb_addr tuples.
///
/// Physical addresses can be 32 or 64 bit based upon the specific implementation. The physical address
/// type is defined in types.rs. For Precursor, it's defined as a u32, which limits the physical address
/// size to 32 bits. This is important because Precursor operates out of a very small amount of RAM,
/// and doubling the size of the overhead bookkeeping structures to handle disk sizes that are well
/// beyond the current implementation has a real impact on the system's free memory footprint. However,
/// the PDDB is coded in a way such that one "should" be able to swap out the u32 for a u64 in the
/// "newtype" definition for a PhysAddr, and the system will work.
///
/// Page table entries don't define themselves with a Basis -- every time a Basis is opened, the entire
/// table must be scanned, brute-force decrypting every entry against the Basis key, and seeing if the
/// checksum in the table entry matches the address. If there's a match, then that entry is recorded as
/// belonging to a given Basis.
///
/// When a system boots, it starts with two default Basis: FastSpace, and System. These Basis are protected
/// with the "device unlock" key. System is a set of low-security data that are used
/// to configure "everyday details". FastSpace Basis is a special-case Basis that is not structured as
/// a Key/Value store. Instead, it tracks a small amount of pre-allocated "free" space, that can be
/// pulled from to grow Dictionaries without requesting that the user unlock every known Basis in the system
/// to ensure none of the valuable data is overwritten.
///
/// The .FastSpace basis should never be equal to the total amount of free space in the system, as it
/// would betray the actual amount of data available and destroy plausible deniability. The system can
/// operate with .FastSpace set to the nil set, but it would also operate without confidentiality of hidden
/// Basis, because in order to allocate new blocks it must prompt the user to enter all known Basis passwords,
/// to avoid accidentally overwriting data that should be retained (as locked Basis would appear as free space
/// and risk being overwritten). However, in order to accelerate bookkeeping, the .FastSpace basis operates
/// with an encrypted record that rotates through a set of "clean" pages (that is, pages that have been set
/// to all 1's) in a circular buffer basis. The state of where the .FastSpace record is in the "clean" pages
/// disclose nothing other than the fact that the system has been used.
///
/// ## Basis Deniability
///
/// The existence of a Basis itself must be confidential; if we stored a list of encrypted Basis passwords
/// somewhere, a rubber-hose attacker would simply need to count the number of encrypted entries and commence
/// the beatings until the number of passwords that have been disclosed under duress match the number of
/// encrypted password entries.
///
/// Herein lies a challenge. Standard password storage techniques demand that each password be stored
/// with a unique salt. The existence of this salt betrays that a password must, in essence, exist.
/// There are at least two methods to counter this:
///
/// 1. Dummy salt entries. In this approach, each system is initialized with a random set of additional, dummy
/// salt/encrypted password combos. In the case of a rubber-hose attack, the attacker cannot know precisely
/// when to stop the beatings, as the few remaining entries could be dummies with absolutely no meaning.
/// 2. A single salt entry. In this approach, a single shared salt is used by /all/ passwords, and there is no
/// encrypted password list stored; instead, the the password as presented is directly used to decrypt
/// the Basis page table and if no valid entries are found we may conclude that the password provided has a typo.
///
/// The advantage of (1) is that the usage of salt falls precisely within the traditional cryptographic
/// specification of bcrypt(). The disadvantages of (1) include: one still has to have a method to match
/// a presented password to a given salt entry. In a traditional user/pass login database, the username is
/// the plaintext key to match the salt. In this case, we would use a nickname used to refer to each Basis
/// to correlate to its salt entry. Thus, we'd have to come up with "garbage" plaintext nicknames
/// for a Basis that are not trivial for a rubber-hose attacker to dismiss as chaffe. I think this is Hard.
/// The alternative is to "brute force" the entire list of salts, guessing each one in sequence until one
/// is found to decrypt entries in the Basis page table. The problem with this is that bcrypt is designed to
/// be computationally slow, even for valid passwords. The complexity parameter is chosen so that it takes
/// about 1 second per iteration through the password function. Note that a faster CPU does not solve this
/// problem: if you CPU gets faster, you should increase your complexity, so one is always suffering the 1
/// second penalty to try a password. This means brute forcing a list of salts necessarily incurs at
/// least a 1 second penalty per entry; thus each dummy basis adds a 1 second minimum overhead to unlocking
/// a new Basis database. This puts a downward pressure on the number of dummies to be included; however, the
/// number of dummies needs to be at least as large as the number of Basis we wish to plausibly deny. This
/// creates a negative incentive to using plausible deniability.
///
/// The advantage of (2) is that when a user presents a password, we only need to run the bcrypt routine
/// once, and then we can immediately start checking the Basis page table for valid entries. The disadvantage
/// of (2) is that we are re-using a salt across all of a given user's passwords. Note that the salt itself
/// is from a TRNG, so between user devices, the salt performs its role. However, it means that an attacker
/// can generate a single rainbow table specific to a given user to reverse all of their passwords; and furthermore,
/// re-used passwords are trivially discoverable with the rainbow table.
///
/// Approach (1) is probably what most crypto purists would adopt: you, dear user, /should/ understand that
/// cryptography is worth the wait. However, a second principle states that "users always pick convenience over
/// security". A 15-second wait is an eternity in the UX world, and would act as an effective deterrent to any
/// user ever using the PD function because it is too slow. Thus for v1 of the PDDB, we're going to try approach
/// (2), where a common salt is used across all the PDDB passwords, but the salt is /not/ re-used between devices.
/// This choice diminishes the overall security of the system in the case that a user chooses weak passwords, or re-uses
/// passwords, but in exchange for a great improvement in responsivity of the implementation.
///
/// The final implementation uses a slight mod on (2), where the 128-bit common salt stored on disk is XOR'd
/// with the user-provided "basis name". Users are of course allowed to pick crappy names for their basis, and
/// re-use the names, but hopefully this adds a modicum of robustness against rainbow table attacks.
///
///
/// ## Basis Unlock Procedure
///
/// Each Basis has a name and a passcode associated with it. The default Basis name is `.System`.
/// In addition to that, a `.FastSpace` structure is unlocked along side the default Basis.
/// These are both associated with the default system unlock passcode.
///
/// A newly created Basis will request a name for the Basis, and a password. It is a requirement
/// that the combination of `(name, password)` be unique; this property is enforced and a system will
/// reject attempts to create identically named, identially passworded Basis. However, one can have
/// same-named Basis that have a different password, or diffently-named Basis with the same password
/// (this is generally not recommended, but it's not prohibited).
///
/// The `name` field is XOR'd with the device-local `salt` field to form the salt for the password,
/// and the plaintext of the password itself is used as the password which is fed into the bcrypt()
/// algorithm to generate a 192-bit encrypted password, which is expanded to 256-bits using SHA-512/256,
/// and then used as the AES key for the given `(name, password)` Basis.
///
///
/// ## Journaling
///
/// A major issue with the implementation is that even a small change in a data structure turns into
/// the mutation of at least an entire sector of data, with a sector being 4k. The reason is that
/// we can't take advantage of FLASH memory's property where 1's (erased) can be set to 0's on a byte-by-byte
/// basis: in order to mask free space signatures, the entire storage area is filled with random bytes.
/// Furthermore, any update to a part of data within a block should necessarily propagate to the entire
/// block changing, in order to avoid attacks where observations on which portion of ciphertext has changed
/// leading to a definitive conclusion about the state of the database records. Therefore, a large portion
/// of the database data will have to persist in RAM, and be updated only at regular, but widely-spaced-out
/// (to avoid FLASH wear-out), intervals.
///
/// In the case that power is lost during an update, the system uses a 32-bit `journal_rev` number at the
/// top of every major database structure. If two competing copies of data are found, the one with the
/// highest `journal_rev` number wins. It may be possible to overflow the 32-bit number, so at some point a
/// garbage collection step needs to be made where any errant, lower journal rev blocks are overwritten
/// with random data; all the journal revs are confirmed to be the same; and then all set to 0 again. Power
/// should not be lost during this process; but it should be rare, and likely never needed.
///
/// When writing to disk, a write-then-erase method is used:
/// 1. Required blocks for the update are taken out of the .FastSpace pool
/// 2. Detached data sections of structural records are written first (only database structural records can have detached data;
/// all stored data have individual journal revision fields associated with them), and into blocks
/// taken out of the .FastSpace pool.
/// 3. The head of the updated structural record is written, with the new latest journal rev noted.
/// 4. The page table is updated using the procedure outlined below.
/// 5. The old blocks are erased and overwritten with random data.
/// 6. The .FastSpace pool is updated with the newly freed blocks
///
/// When updating the page table, it's important not to lose an entire page's worth of entries in case
/// power is lost when updating it. The following process is used to update a page table entry:
/// 0. At the end of the page table there is a circular buffer of blank pages (they are truly blank,
/// but this will not leak metadata about the contents of the PDDB, as it only does bookkeeping
/// on page table updates as a whole). This is known as the make-before-break buffer (mbbb).
/// 1. The previous mbbb entry is erased, if there is any, and the next entry is noted.
/// 2. The new page table entry is written into the mbbb
/// 3. The old page table entry in the page table itself is erased.
/// 4. The old page table entry is populated with the updated page table records.
///
/// Detecting and Repairing a broken page table:
/// The special case of long run's of 1's (more than 32 bits in a row) is checked during the page table scan.
/// If a long run of 1's is detected, then a suspected corrupted page table update is flagged, and for that page the data
/// is pulled from the mbbb record.
///
/// ## Precursor's Implementation-Specific Flash Memory Organization:
///
/// ```Text
/// offset from |
/// pt_phys_base| contents (example for total PDDB len of 0x6f8_0000 or 111 MiB)
/// ------------|---------------------------------------------------
/// 0x0000_0000 | virtual map for page at (0x0000 + data_phys_base)
/// 0x0000_0010 | virtual map for page at (0x1000 + data_phys_base)
/// 0x0000_0020 | virtual map for page at (0x2000 + data_phys_base)
/// ...
/// 0x0006_F7F0 | virtual map for page at (0x06F7_F000 + data_phys_base)
/// 0x0006_F800 | unused
/// ...
/// 0x0007_0000 | key page
/// ...
/// 0x0007_1000 | mbbb start (example of 10 pages)
/// ...
/// 0x0007_B000 | fscb start (example of 10 pages)
/// ...
/// 0x0008_5000 | data_phys_base - start of basis + dictionary + key data region
/// ```
// historical note: 389 hours, 11 mins elapsed since the start of the PDDB coding, and the first attempt at a hardware test -- as evidenced by the uptime of the hardware validation unit.
pub mod api;
pub use api::*;
pub mod frontend;

View file

@ -1,67 +1,86 @@
#![cfg_attr(target_os = "none", no_std)]
#![cfg_attr(target_os = "none", no_main)]
extern crate bitflags;
extern crate bitfield;
mod api;
use api::*;
mod backend;
use backend::*;
#[cfg(not(any(target_os = "none", target_os = "xous")))]
mod tests;
#[cfg(not(any(target_os = "none", target_os = "xous")))]
use tests::*;
use num_traits::*;
use xous::{msg_blocking_scalar_unpack, msg_scalar_unpack};
use core::cell::RefCell;
use std::rc::Rc;
/// PDDB - Plausibly Deniable DataBase
/// # PDDB - Plausibly Deniable DataBase
///
/// Glossary:
/// Basis - a unionizeable dictionary that can be unlocked with a password-protected key.
/// Dictionary - similar to a "volume" on a typical filesystem. Contains a group of k/v pairs
/// ## Glossary:
/// * Basis - a unionizeable dictionary that can be unlocked with a password-protected key.
/// * Dictionary - similar to a "volume" on a typical filesystem. Contains a group of k/v pairs
/// that share attributes such as access permissions
/// Key - a unique string that identifies a piece of data in the PDDB
/// Path - Rust's notion of how to locate a file, except we interpret it as a PDDB key, where
/// * Key - a unique string that identifies a piece of data in the PDDB
/// * Path - Rust's notion of how to locate a file, except we interpret it as a PDDB key, where
/// the "root" directory is the dictionary, and the remainder is the "key"
/// Value - the value associated with a key, isomorphic ot a "file" in Rust
/// Open Basis - A Basis that is decryptable and known to the system.
/// Closed Basis - A Basis that is not decryptable. It's unknown to the system and potentially treated
/// * Value - the value associated with a key, isomorphic ot a "file" in Rust
/// * Open Basis - A Basis that is decryptable and known to the system.
/// * Closed Basis - A Basis that is not decryptable. It's unknown to the system and potentially treated
/// identically to free disk space, e.g., it could be partially overwritten.
/// FSCB - Free Space Commit Buffer. A few pages allocated to tracking a subset of free space,
/// * FSCB - Free Space Commit Buffer. A few pages allocated to tracking a subset of free space,
/// meant to accelerate PDDB operations. Creates a side-channel that can reveal that some activity
/// has happened, but without disclosing what and why. Contains FastSpace and SpaceUpdate records.
/// Frequently updated, so the buffer is slightly oversized, and which sector is "hot" is randomized
/// for wear-levelling.
/// MBBB - Make Before Break Buffer. A set of randomly allocated pages that are a shadow copy
/// * MBBB - Make Before Break Buffer. A set of randomly allocated pages that are a shadow copy
/// of a page table page. If any data exists, its contents override those of a corrupted page table.
/// FastSpace - a collection of random pages that are known to be empty. The number of pages in FastSpace
/// * FastSpace - a collection of random pages that are known to be empty. The number of pages in FastSpace
/// is reduced from the absolute amount of free space available by at least a factor of FSCB_FILL_COEFFICIENT.
/// SpaceUpdate - encrypted patches to the FastSpace table. The FastSpace table is "heavyweight", and would
/// * SpaceUpdate - encrypted patches to the FastSpace table. The FastSpace table is "heavyweight", and would
/// be too expensive to update on every page allocation, so SpaceUpdate is used to patch the FastSpace table.
///
/// A `Path` like the following is deconstructed as follows by the PDDB:
///
/// ```Text
/// Dictionary
/// | Key
/// | |
/// --+- --------+---------------------
/// logs:matrix/alice/oct30_2021/bob.txt
/// ```
///
/// It could equally have an arbitrary name like "logs:Matrix - alice to bob Oct 30 2021";
/// It could equally have an arbitrary name like `logs:Matrix - alice to bob Oct 30 2021`;
/// as long as the string that identifies a Key is unique, it's stored in the database
/// all the same. Any valid utf-8 unicode characters are acceptable.
///
/// Likewise, something like this:
/// settings:wifi/Kosagi.json
/// Would be deconstructed into the "settings" dictionary with a key of wifi/Kosagi.json.
/// `settings:wifi/Kosagi.json`
/// Would be deconstructed into the "settings" dictionary with a key of `wifi/Kosagi.json`.
///
/// ## Code Organization:
/// Accurate as of Nov 2021. May be subject to charge, ymmv.
///
/// Threat model:
/// ### `frontend.rs`
/// Defines a set of modules that plug the PDDB into applications (in particular, they
/// attemp to provide a Rust-compatible `read`/`write`/`open` abstraction layer,
/// torturing the notion of `Prefix`, `Path` and `File` in Rust to fit the basis/dict/key
/// format of the PDDB).
///
/// This is the `lib`-facing set of operations.
///
/// ### `backend.rs`
/// Defines a set of modules that implement the PDDB itself. This is the hardware-facing set
/// of operations.
///
/// #### `basis.rs`
/// The set of known `Basis` are tracked in the `BasisCache` structure. This is the "entry point"
/// for most operations on the PDDB, and thus most externally-visible API calls will be revealed
/// on that structure; in fact many calls on that object are just pass-through of lower level calls.
///
/// A `BasisCach`e consists of one or more `BasisCacheEntries`. This structure manages one or more
/// `DictCacheEntry` within a `BasisCacheEntry`'s dictionary cache.
///
/// #### `dictionary.rs`
/// The `DictCacheEntry` is defined in this file, along with the on-disk `Dictionary`
/// storage stucture. Most of the methods on `DictCacheEntry` are concerned with managing
/// keys contained within the dictionary cache.
///
/// #### `keys.rs`
/// The `KeyCacheEntry` is defined in this file. It's a bookkeeping record for the
/// `DictCacheEntry` structure. Keys themselves are split into metadata and data
/// records. The `KeyDescriptor` is the on-disk format for key metadata. The key
/// storage is either described in `KeySmallPool` for keys less than one `VPAGE_SIZE`
/// (just shy of 4kiB), or written directly to disk as fully allocated blocks of
/// `VPAGE_SIZE` for keys larger than one `VPAGE_SIZE` unit.
///
/// ## Threat model:
/// The user is forced to divulge "all the Basis passwords" on the device, through
/// coercion, blackmail, subpoena, customs checkpoint etc. The adversary has physical
/// access to the device, and is able to take a static disk image; they may even have
@ -86,7 +105,7 @@ use std::rc::Rc;
/// a problem handled by the OS, and it is up to the user to not type secret passwords
/// in areas that may be under camera surveillance.
///
/// Auditor Notices:
/// ## Auditor Notices:
/// There's probably a lot of things the PDDB does wrong. Here's a list of some things
/// to worry about:
/// - The device RootKeys shares one salt across all of its keys, instead of a per-key salt.
@ -114,7 +133,7 @@ use std::rc::Rc;
/// entries have no such protection, which means there is a slight chance of data loss due to a checksum
/// collision on the free space entries.
///
/// General Operation:
/// ## General Operation:
/// The initial Basis is known as the "System" Basis. It's a low-security framework basis
/// onto which all other Basis overlay. The initial set of Dictionaries, along with their
/// Key/Value pairs, are available to any and all processes.
@ -140,7 +159,7 @@ use std::rc::Rc;
/// - If the client attempts to read or write to any keys that span a Basis modification,
/// the now-ambiguous key operation will return a `BrokenPipe` error to the caller.
///
/// General flash->key structure
/// ## General flash->key structure
///
/// The basic unit of memory is a page (4k). Keys shorter than 4k will be packed into a single
/// page, but no allocation will ever be smaller than 4k due to the erase block size of the FLASH.
@ -201,9 +220,7 @@ use std::rc::Rc;
/// to all 1's) in a circular buffer basis. The state of where the .FastSpace record is in the "clean" pages
/// disclose nothing other than the fact that the system has been used.
///
///
///
/// Basis Deniability
/// ## Basis Deniability
///
/// The existence of a Basis itself must be confidential; if we stored a list of encrypted Basis passwords
/// somewhere, a rubber-hose attacker would simply need to count the number of encrypted entries and commence
@ -258,7 +275,7 @@ use std::rc::Rc;
/// re-use the names, but hopefully this adds a modicum of robustness against rainbow table attacks.
///
///
/// Basis Unlock Procedure
/// ## Basis Unlock Procedure
///
/// Each Basis has a name and a passcode associated with it. The default Basis name is `.System`.
/// In addition to that, a `.FastSpace` structure is unlocked along side the default Basis.
@ -276,7 +293,7 @@ use std::rc::Rc;
/// and then used as the AES key for the given `(name, password)` Basis.
///
///
/// Journaling
/// ## Journaling
///
/// A major issue with the implementation is that even a small change in a data structure turns into
/// the mutation of at least an entire sector of data, with a sector being 4k. The reason is that
@ -320,8 +337,9 @@ use std::rc::Rc;
/// If a long run of 1's is detected, then a suspected corrupted page table update is flagged, and for that page the data
/// is pulled from the mbbb record.
///
/// Precursor's Implementation-Specific Flash Memory Organization:
/// ## Precursor's Implementation-Specific Flash Memory Organization:
///
/// ```Text
/// offset from |
/// pt_phys_base| contents (example for total PDDB len of 0x6f8_0000 or 111 MiB)
/// ------------|---------------------------------------------------
@ -339,9 +357,28 @@ use std::rc::Rc;
/// 0x0007_B000 | fscb start (example of 10 pages)
/// ...
/// 0x0008_5000 | data_phys_base - start of basis + dictionary + key data region
/// ```
// historical note: 389 hours, 11 mins elapsed since the start of the PDDB coding, and the first attempt at a hardware test -- as evidenced by the uptime of the hardware validation unit.
extern crate bitflags;
extern crate bitfield;
mod api;
use api::*;
mod backend;
use backend::*;
#[cfg(not(any(target_os = "none", target_os = "xous")))]
mod tests;
#[cfg(not(any(target_os = "none", target_os = "xous")))]
use tests::*;
use num_traits::*;
use xous::{msg_blocking_scalar_unpack, msg_scalar_unpack};
use core::cell::RefCell;
use std::rc::Rc;
#[xous::xous_main]
fn xmain() -> ! {
log_server::init_wait().unwrap();
@ -411,10 +448,11 @@ fn xmain() -> ! {
}
*/
/* list of test cases:
genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
[done] genenral integrity: allocate 4 dictionaries, each with 34 keys of various sizes ranging from 1k-9k.
[done] delete/add consistency: general integrity, delete a dictionary, then add a dictionary.
in-place update consistency: general integrity then patch all keys with a new test pattern
extend update consistency: general integrity then patch all keys with a longer test pattern
key deletion torture test: delete every other key in a dictionary, then regenerate some of them with new data.
basis search: create basis A, populate with general integrity. create basis B, add test entries.
hide basis B, confirm original A; mount basis B, confirm B overlay.
*/