add more key manipulation APIs

services/pddb/src/api.rs

@@ -1,3 +1,6 @@
+use bitfield::bitfield;
+use std::num::NonZeroU32;
+
 pub(crate) const SERVER_NAME_PDDB: &str     = "_Plausibly Deniable Database_";
 #[allow(dead_code)]
 pub(crate) const BASIS_NAME_LEN: usize = 64; // don't want this too long anyways, because it's not recorded anywhere - users have to type it in.
@@ -69,17 +72,22 @@ pub(crate) enum Opcode {
     CreateBasis,
     OpenBasis,
     CloseBasis,
-    /// warning, the Delete routine has not been well tested
+    /// warning, the Delete routines have not been well tested
     DeleteBasis,
+    DeleteKey,
+    DeleteDict,
 
-    CreateDict,
-
+    /// primary method for accessing the database
     KeyRequest,
 
+    // pddbkey methods
     ReadKey,
     WriteKey,
     WriteKeyFlush,
 
+    /// drops any connection state associated with a given key
+    KeyDrop,
+
     /// quit the server
     Quit,
 }
@@ -112,21 +120,19 @@ pub struct PddbBasisRequest {
     pub code: PddbRequestCode,
 }
 
-#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
-pub struct PddbDictRequest {
-    /// applications shouldn't specify a basis so that the PD mechanism works as intended, but non-sensitive system config keys will want to generally specify the system basis.
-    pub basis_specified: bool,
-    pub basis_name: xous_ipc::String::</* BASIS_NAME_LEN */ 64>, // pending https://github.com/rust-lang/rust/issues/90195
-    pub dict_name: xous_ipc::String::</* DICT_NAME_LEN] */ 111>, // pending https://github.com/rust-lang/rust/issues/90195
-    pub code: PddbRequestCode,
-}
-
 /// A structure for requesting a token to access a particular key/value pair
 #[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
-pub(crate) struct PddbKeyRequest {
-    pub(crate) dict: xous_ipc::String::</*DICT_NAME_LEN*/ 111>, // pending https://github.com/rust-lang/rust/issues/90195
-    pub(crate) key: xous_ipc::String::</*KEY_NAME_LEN*/ 95>, // pending https://github.com/rust-lang/rust/issues/90195
-    pub(crate) token: Option<ApiToken>,
+pub struct PddbKeyRequest {
+    pub basis_specified: bool,
+    pub basis: xous_ipc::String::</* BASIS_NAME_LEN */ 64>, // pending https://github.com/rust-lang/rust/issues/90195
+    pub dict: xous_ipc::String::</*DICT_NAME_LEN*/ 111>, // pending https://github.com/rust-lang/rust/issues/90195
+    pub key: xous_ipc::String::</*KEY_NAME_LEN*/ 95>, // pending https://github.com/rust-lang/rust/issues/90195
+    pub token: Option<ApiToken>,
+    pub create_dict: bool,
+    pub create_key: bool,
+    pub alloc_hint: Option<u64>, // this is a usize but for IPC we must have defined memory sizes, so we pick the big option.
+    pub cb_sid: [u32; 4],
+    pub result: PddbRequestCode,
 }
 
 /// Return codes for Read/Write API calls to the main server
@@ -164,6 +170,36 @@ impl PddbBuf {
         unsafe {core::mem::transmute::<*mut u8, &mut PddbBuf>(slice.as_mut_ptr()) }
     }
 }
+
+bitfield! {
+    #[derive(Copy, Clone, PartialEq, Eq)]
+    pub struct KeyFlags(u32);
+    impl Debug;
+    /// set if the entry is valid -- in the cache, an invalid entry means it was previously allocated but then deleted, and needs a sync
+    pub valid, set_valid: 0;
+    /// resolved indicates that the "start" address isn't fully resolved yet in the cache
+    pub unresolved, set_unresolved: 1;
+}
+
+/// A structure for passing around key metadata
+#[derive(Debug)]
+pub struct KeyAttributes {
+    /// actual length of data in the key
+    pub len: usize,
+    /// pre-reserved storage space for the key (growable to this bound "at no cost")
+    pub reserved: usize,
+    /// access count
+    pub age: usize,
+    /// owning dictionary
+    pub dict: String,
+    /// the specific basis from which this key's metadata was found
+    pub basis: String,
+    /// flags
+    pub flags: KeyFlags,
+    /// descriptor index
+    pub index: NonZeroU32,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

services/pddb/src/backend/key.rs

@@ -4,19 +4,8 @@ use super::*;
 use std::num::NonZeroU32;
 use core::ops::{Deref, DerefMut};
 use std::cmp::Ordering;
-use bitfield::bitfield;
 use std::io::{Result, Error, ErrorKind};
 
-bitfield! {
-    #[derive(Copy, Clone, PartialEq, Eq)]
-    pub struct KeyFlags(u32);
-    impl Debug;
-    /// set if the entry is valid -- in the cache, an invalid entry means it was previously allocated but then deleted, and needs a sync
-    pub valid, set_valid: 0;
-    /// resolved indicates that the "start" address isn't fully resolved yet in the cache
-    pub unresolved, set_unresolved: 1;
-}
-
 #[derive(PartialEq, Eq, Copy, Clone, Debug)]
 #[repr(C, align(8))]
 pub struct KeyName {
@@ -208,22 +197,3 @@ impl PartialEq for KeySmallPoolOrd {
         self.avail == other.avail
     }
 }
-
-/// A structure for passing around key metadata
-#[derive(Debug)]
-pub(crate) struct KeyAttributes {
-    /// actual length of data in the key
-    pub(crate) len: usize,
-    /// pre-reserved storage space for the key (growable to this bound "at no cost")
-    pub(crate) reserved: usize,
-    /// access count
-    pub(crate) age: usize,
-    /// owning dictionary
-    pub(crate) dict: String,
-    /// the specific basis from which this key's metadata was found
-    pub(crate) basis: String,
-    /// flags
-    pub(crate) flags: KeyFlags,
-    /// descriptor index
-    pub(crate) index: NonZeroU32,
-}

services/pddb/src/frontend/pddbkey.rs

@@ -11,91 +11,23 @@ use std::io::{Read, Write};
 use std::string::String;
 
 pub struct PddbKey<'a> {
-    conn: CID,
-    dict: String,
-    key: String,
-    token: ApiToken,
+    pub(crate) dict: String,
+    pub(crate) key: String,
+    pub(crate) basis: Option<String>,
+    pub(crate) token: ApiToken,
     /// position in the key's data "stream"
-    pos: u64,
-    buf: Buffer<'a>,
+    pub(crate) pos: u64,
+    pub(crate) buf: Buffer<'a>,
+    pub(crate) conn: CID,
 }
+/// PddbKeys are created by Pddb
 impl<'a> PddbKey<'a> {
-    pub fn get<P: AsRef<Path>>(path: P) -> Result<PddbKey<'a>> {
-        let xns = xous_names::XousNames::new().unwrap();
-        REFCOUNT.store(REFCOUNT.load(Ordering::Relaxed) + 1, Ordering::Relaxed);
-        let conn = xns.request_connection_blocking(api::SERVER_NAME_PDDB).expect("Can't connect to Pddb server");
-
-        if !path.as_ref().is_absolute() {
-            return Err(Error::new(ErrorKind::InvalidInput, "All PDDB keys must be fully specified relative to a dictionary"));
-        }
-        let mut dict = String::new();
-        let mut key = String::new();
-        let mut components = path.as_ref().components();
-        match components.next().unwrap() {
-            Component::Prefix(prefix_component) => {
-                if let Some(dictstr) = prefix_component.as_os_str().to_str() {
-                    if dictstr.len() <= DICT_NAME_LEN {
-                        dict.push_str(dictstr);
-                    } else {
-                        return Err(Error::new(ErrorKind::InvalidInput, format!("PDDB dictionary names must be shorter than {} bytes", DICT_NAME_LEN)));
-                    }
-                } else {
-                    return Err(Error::new(ErrorKind::InvalidInput, "PDDB dictionary names must valid UTF-8"));
-                }
-            }
-            _ => {
-                return Err(Error::new(ErrorKind::InvalidInput, "All PDDB entries must be of the format `dict:key`, where `dict` is treated as a Prefix"));
-            }
-        }
-        // collect the remaining components into the key
-        for comps in components {
-            if let Some(keystr) = comps.as_os_str().to_str() {
-                key.push_str(keystr);
-            } else {
-                return Err(Error::new(ErrorKind::InvalidInput, "PDDB dictionary names must valid UTF-8"));
-            }
-        }
-
-        if key.len() > KEY_NAME_LEN {
-            return Err(Error::new(ErrorKind::InvalidInput, format!("PDDB key names must be shorter than {} bytes", DICT_NAME_LEN)));
-        }
-
-        let request = PddbKeyRequest {
-            dict: xous_ipc::String::<DICT_NAME_LEN>::from_str(dict.as_str()),
-            key: xous_ipc::String::<KEY_NAME_LEN>::from_str(key.as_str()),
-            token: None,
-        };
-        let mut buf = Buffer::into_buf(request)
-            .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
-        buf.lend_mut(conn, Opcode::KeyRequest.to_u32().unwrap())
-            .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
-
-        let response = buf.to_original::<PddbKeyRequest, _>().unwrap();
-
-        // we probably should never remove this check -- the code may compile correctly and
-        // "work" without this being an even page size, but it's pretty easy to get this wrong,
-        // and if it's wrong we can lose a lot in terms of efficiency of execution.
-        assert!(core::mem::size_of::<PddbBuf>() == 4096, "PddBuf record has the wrong size");
-        if let Some(token) = response.token {
-            Ok(PddbKey {
-                conn,
-                dict,
-                key,
-                token,
-                pos: 0,
-                buf: Buffer::new(core::mem::size_of::<PddbBuf>()),
-            })
-        } else {
-            Err(Error::new(ErrorKind::PermissionDenied, "Dict/Key access denied"))
-        }
-    }
     /// this will clear all residual values in the buffer. Should be called whenever the Basis set changes.
     pub fn volatile_clear(&mut self) {
         self.buf.volatile_clear();
     }
-
-    pub(crate) fn conn(&self) -> CID {
-        self.conn
+    pub fn attributes(&self) -> KeyAttributes {
+        unimplemented!()
     }
 }
 
@@ -209,21 +141,19 @@ impl<'a> Write for PddbKey<'a> {
     }
 }
 
-use core::sync::atomic::{AtomicU32, Ordering};
-static REFCOUNT: AtomicU32 = AtomicU32::new(0);
+use core::sync::atomic::Ordering;
 impl<'a> Drop for PddbKey<'a> {
     fn drop(&mut self) {
         self.buf.volatile_clear(); // clears any confidential data in our memory buffer
 
-        // the connection to the server side must be reference counted, so that multiple instances of this object within
-        // a single process do not end up de-allocating the CID on other threads before they go out of scope.
-        // Note to future me: you want this. Don't get rid of it because you think, "nah, nobody will ever make more than one copy of this object".
+        // notify the server that we can drop the connection state when our object goes out of scope
+        send_message(self.conn, Message::new_blocking_scalar(Opcode::KeyDrop.to_usize().unwrap(),
+        self.token[0] as usize, self.token[1] as usize, self.token[2] as usize, 0)).expect("couldn't send KeyDrop message");
+
         REFCOUNT.store(REFCOUNT.load(Ordering::Relaxed) - 1, Ordering::Relaxed);
         if REFCOUNT.load(Ordering::Relaxed) == 0 {
             unsafe{xous::disconnect(self.conn).unwrap();}
         }
-        // if there was object-specific state (such as a one-time use server for async callbacks, specific to the object instance),
-        // de-allocate those items here. They don't need a reference count because they are object-specific
     }
 }
 

services/pddb/src/lib.rs

@@ -7,24 +7,83 @@ pub use frontend::*;
 
 use num_traits::*;
 use std::io::{Result, Error, ErrorKind};
-use xous::CID;
+use xous::{CID, SID, msg_scalar_unpack, send_message, Message};
 use xous_ipc::Buffer;
 
 use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::thread;
+use std::thread::JoinHandle;
 
 use core::sync::atomic::{AtomicU32, Ordering};
-static REFCOUNT: AtomicU32 = AtomicU32::new(0);
+pub(crate) static REFCOUNT: AtomicU32 = AtomicU32::new(0);
 
-pub struct PddbBasisManager {
-    conn: CID,
+#[derive(num_derive::FromPrimitive, num_derive::ToPrimitive, Debug)]
+enum CbOp {
+    Change,
+    Quit
 }
-impl PddbBasisManager {
+
+/// The intention is that one Pddb management object is made per process, and this serves
+/// as the gateway for parcelling out PddbKey objects, which are the equivalent of a File
+/// in a convention system that implements read/write operations.
+///
+/// The Pddb management object also handles meta-issues such as basis creation, unlock/lock,
+/// and callbacks in case data changes.
+pub struct Pddb {
+    conn: CID,
+    /// a SID that we can directly share with the PDDB server for the purpose of handling key change callbacks
+    cb_sid: SID,
+    cb_handle: Option<JoinHandle::<()>>,
+    /// Handle key change updates. The general intent is that the closure implements a
+    /// `send` of a message to the server to deal with a key change appropriately,
+    /// but no mutable data is allowed within the closure itself due to safety problems.
+    /// Thus, the closure might encode something like whether the message is blocking or nonblocking;
+    /// the CID of the message; and the opcode and arguments, as static variables. An implementation
+    /// with many keys might, for example, keep a lookup table of indices to keys to track which
+    /// ones need clearing if you're working at a very fine granularity, but more generally,
+    /// the application behavior might be something like a refresh of the data from storage
+    /// in the case of a basis change. Basis changes are thought to be rare; so, big changes
+    /// like this are probably OK.
+    keys: Arc<Mutex<HashMap<ApiToken, Box<dyn Fn() + 'static + Send> >>>,
+}
+impl Pddb {
     pub fn new() -> Self {
         REFCOUNT.store(REFCOUNT.load(Ordering::Relaxed) + 1, Ordering::Relaxed);
         let xns = xous_names::XousNames::new().unwrap();
         let conn = xns.request_connection_blocking(api::SERVER_NAME_PDDB).expect("can't connect to Pddb server");
-        PddbBasisManager {
-            conn
+        let sid = xous::create_server().unwrap();
+        let keys: Arc<Mutex<HashMap<ApiToken, Box<dyn Fn() + 'static + Send> >>> = Arc::new(Mutex::new(HashMap::new()));
+        let handle = thread::spawn({
+            let keys = Arc::clone(&keys);
+            let sid = sid.clone();
+            move || {
+                loop {
+                    let msg = xous::receive_message(sid).unwrap();
+                    match FromPrimitive::from_usize(msg.body.id()) {
+                        Some(CbOp::Change) => msg_scalar_unpack!(msg, t0, t1, t2, _, {
+                            let token: ApiToken = [t0 as u32, t1 as u32, t2 as u32];
+                            if let Some(cb) = keys.lock().unwrap().get(&token) {
+                                cb();
+                            } else {
+                                log::warn!("Key changed but no callback was hooked to receive it");
+                            }
+                        }),
+                        Some(CbOp::Quit) => { // blocking scalar
+                            xous::return_scalar(msg.sender, 0).unwrap();
+                            break;
+                        },
+                        _ =>log::warn!("Got unknown opcode: {:?}", msg),
+                    }
+                }
+                xous::destroy_server(sid).unwrap();
+            }
+        });
+        Pddb {
+            conn,
+            cb_sid: sid,
+            cb_handle: Some(handle),
+            keys,
         }
     }
     /// return a list of all open bases
@@ -71,7 +130,10 @@ impl PddbBasisManager {
             }
         }
     }
-    pub fn create(&self, basis_name: &str) -> Result<()> {
+    pub fn create_basis(&self, basis_name: &str) -> Result<()> {
+        if basis_name.len() > BASIS_NAME_LEN - 1 {
+            return Err(Error::new(ErrorKind::InvalidInput, "basis name too long"));
+        }
         let mgmt = PddbBasisRequest {
             name: xous_ipc::String::<BASIS_NAME_LEN>::from_str(basis_name),
             code: PddbRequestCode::Create,
@@ -89,7 +151,10 @@ impl PddbBasisManager {
             }
         }
     }
-    pub fn open(&self, basis_name: &str) -> Result<()> {
+    pub fn unlock_basis(&self, basis_name: &str) -> Result<()> {
+        if basis_name.len() > BASIS_NAME_LEN - 1 {
+            return Err(Error::new(ErrorKind::InvalidInput, "basis name too long"));
+        }
         let mgmt = PddbBasisRequest {
             name: xous_ipc::String::<BASIS_NAME_LEN>::from_str(basis_name),
             code: PddbRequestCode::Open,
@@ -107,7 +172,10 @@ impl PddbBasisManager {
             }
         }
     }
-    pub fn close(&self, basis_name: &str) -> Result<()> {
+    pub fn lock_basis(&self, basis_name: &str) -> Result<()> {
+        if basis_name.len() > BASIS_NAME_LEN - 1 {
+            return Err(Error::new(ErrorKind::InvalidInput, "basis name too long"));
+        }
         let mgmt = PddbBasisRequest {
             name: xous_ipc::String::<BASIS_NAME_LEN>::from_str(basis_name),
             code: PddbRequestCode::Close,
@@ -125,7 +193,10 @@ impl PddbBasisManager {
             }
         }
     }
-    pub fn delete(&self, basis_name: &str) -> Result<()> {
+    pub fn delete_basis(&self, basis_name: &str) -> Result<()> {
+        if basis_name.len() > BASIS_NAME_LEN - 1 {
+            return Err(Error::new(ErrorKind::InvalidInput, "basis name too long"));
+        }
         let mgmt = PddbBasisRequest {
             name: xous_ipc::String::<BASIS_NAME_LEN>::from_str(basis_name),
             code: PddbRequestCode::Delete,
@@ -143,55 +214,175 @@ impl PddbBasisManager {
             }
         }
     }
-}
 
-pub struct Pddb<'a> {
-    conn: CID,
-    contents: HashMap<PddbKey<'a>, &'a [u8]>,
-    callback: Option<Box<dyn FnMut() + 'a>>,
-}
-impl<'a> Pddb<'a> {
-    // creates a dictionary only if it does not already exist
-    pub fn create(dict_name: &str, basis_name: Option<&str>) -> Option<Self> {
-        REFCOUNT.store(REFCOUNT.load(Ordering::Relaxed) + 1, Ordering::Relaxed);
-        let xns = xous_names::XousNames::new().unwrap();
-        let conn = xns.request_connection_blocking(api::SERVER_NAME_PDDB).expect("can't connect to Pddb server");
-        /*let mut request = PddbDictRequest {
+    /// If the `create_*` flags are set, creates the asset if they do not exist, otherwise if false, returns
+    /// an error if the asset does not exist.
+    /// `alloc_hint` is an optional field to guide the PDDB allocator to put the key in the right pool. Setting it to `None`
+    /// is perfectly fine, it just has a potential performance impact, especially for very large keys.
+    /// `key_changed_cb` is a static function meant to initiate a message to a server in case the key in question
+    /// goes away due to a basis locking.
+    pub fn get(&mut self, dict_name: &str, key_name: &str, basis_name: Option<&str>,
+        create_dict: bool, create_key: bool, alloc_hint: Option<usize>, key_changed_cb: Option<impl Fn() + 'static + Send>) -> Result<PddbKey> {
+        if key_name.len() > (KEY_NAME_LEN - 1) {
+            return Err(Error::new(ErrorKind::InvalidInput, "key name too long"));
+        }
+        if dict_name.len() > (DICT_NAME_LEN - 1) {
+            return Err(Error::new(ErrorKind::InvalidInput, "dictionary name too long"));
+        }
+        let bname = if let Some(bname) = basis_name {
+            if bname.len() > BASIS_NAME_LEN - 1 {
+                return Err(Error::new(ErrorKind::InvalidInput, "basis name too long"));
+            }
+            xous_ipc::String::<BASIS_NAME_LEN>::from_str(bname)
+        } else {
+            xous_ipc::String::<BASIS_NAME_LEN>::new()
+        };
+
+        let request = PddbKeyRequest {
             basis_specified: basis_name.is_some(),
-        }*/
+            basis: xous_ipc::String::<BASIS_NAME_LEN>::from_str(&bname),
+            dict: xous_ipc::String::<DICT_NAME_LEN>::from_str(dict_name),
+            key: xous_ipc::String::<KEY_NAME_LEN>::from_str(key_name),
+            create_dict,
+            create_key,
+            token: None,
+            result: PddbRequestCode::Uninit,
+            cb_sid: self.cb_sid.to_array(),
+            alloc_hint: if let Some(a) = alloc_hint {Some(a as u64)} else {None},
+        };
+        let mut buf = Buffer::into_buf(request)
+            .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
+        buf.lend_mut(self.conn, Opcode::KeyRequest.to_u32().unwrap())
+            .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
 
-        None
+        let response = buf.to_original::<PddbKeyRequest, _>().unwrap();
+
+        // we probably should never remove this check -- the code may compile correctly and
+        // "work" without this being an even page size, but it's pretty easy to get this wrong,
+        // and if it's wrong we can lose a lot in terms of efficiency of execution.
+        assert!(core::mem::size_of::<PddbBuf>() == 4096, "PddBuf record has the wrong size");
+        match response.result {
+            PddbRequestCode::NoErr => {
+                if let Some(token) = response.token {
+                    if let Some(cb) = key_changed_cb {
+                        self.keys.lock().unwrap().insert(token, Box::new(cb));
+                    }
+                    REFCOUNT.store(REFCOUNT.load(Ordering::Relaxed) + 1, Ordering::Relaxed);
+                    let pk = PddbKey {
+                        dict: String::from(dict_name),
+                        key: String::from(key_name),
+                        basis: if basis_name.is_some() {Some(String::from(bname.as_str().unwrap()))} else {None},
+                        pos: 0,
+                        token,
+                        buf: Buffer::new(core::mem::size_of::<PddbBuf>()),
+                        conn: self.conn,
+                    };
+                    Ok(pk)
+                } else {
+                    Err(Error::new(ErrorKind::PermissionDenied, "Dict/Key access denied"))
+                }
+            }
+            PddbRequestCode::AccessDenied => Err(Error::new(ErrorKind::PermissionDenied, "Dict/Key access denied")),
+            PddbRequestCode::NoFreeSpace => Err(Error::new(ErrorKind::OutOfMemory, "No more space on disk")),
+            PddbRequestCode::NotMounted => Err(Error::new(ErrorKind::ConnectionReset, "PDDB was unmounted")),
+            PddbRequestCode::NotFound => Err(Error::new(ErrorKind::NotFound, "Dictionary or key was not found")),
+            _ => Err(Error::new(ErrorKind::Other, "Internal error"))
+        }
     }
 
-    /// returns a key only if it exists
-    pub fn get(&mut self, dict_name: &str, key_name: &str, basis_name: Option<&str>, key_changed_cb: impl FnMut() + 'a) -> Result<Option<PddbKey>> {
-        self.callback = Some(Box::new(key_changed_cb));
-        Ok(None)
+    /// deletes a key within the dictionary
+    pub fn delete_key(&mut self, dict_name: &str, key_name: &str, basis_name: Option<&str>) -> Result<()> {
+        if key_name.len() > (KEY_NAME_LEN - 1) {
+            return Err(Error::new(ErrorKind::InvalidInput, "key name too long"));
+        }
+        if dict_name.len() > (DICT_NAME_LEN - 1) {
+            return Err(Error::new(ErrorKind::InvalidInput, "dictionary name too long"));
+        }
+        let bname = if let Some(bname) = basis_name {
+            if bname.len() > BASIS_NAME_LEN - 1 {
+                return Err(Error::new(ErrorKind::InvalidInput, "basis name too long"));
+            }
+            xous_ipc::String::<BASIS_NAME_LEN>::from_str(bname)
+        } else {
+            xous_ipc::String::<BASIS_NAME_LEN>::new()
+        };
+
+        let request = PddbKeyRequest {
+            basis_specified: basis_name.is_some(),
+            basis: xous_ipc::String::<BASIS_NAME_LEN>::from_str(&bname),
+            dict: xous_ipc::String::<DICT_NAME_LEN>::from_str(dict_name),
+            key: xous_ipc::String::<KEY_NAME_LEN>::from_str(key_name),
+            create_dict: false,
+            create_key: false,
+            token: None,
+            result: PddbRequestCode::Uninit,
+            cb_sid: self.cb_sid.to_array(),
+            alloc_hint: None,
+        };
+        let mut buf = Buffer::into_buf(request)
+            .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
+        buf.lend_mut(self.conn, Opcode::DeleteKey.to_u32().unwrap())
+            .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
+
+        let response = buf.to_original::<PddbKeyRequest, _>().unwrap();
+        match response.result {
+            PddbRequestCode::NoErr => Ok(()),
+            PddbRequestCode::NotFound => Err(Error::new(ErrorKind::NotFound, "Dictionary or key was not found")),
+            _ => Err(Error::new(ErrorKind::Other, "Internal error"))
+        }
     }
-    /// closes the key, de-allocating the OS memory to track it.
-    pub fn close(&mut self, _key: PddbKey) -> Result<()> {
-        Ok(())
+    /// deletes the entire dictionary
+    pub fn delete_dict(&mut self, dict_name: &str, basis_name: Option<&str>) -> Result<()> {
+        if dict_name.len() > (DICT_NAME_LEN - 1) {
+            return Err(Error::new(ErrorKind::InvalidInput, "dictionary name too long"));
+        }
+        let bname = if let Some(bname) = basis_name {
+            if bname.len() > BASIS_NAME_LEN - 1 {
+                return Err(Error::new(ErrorKind::InvalidInput, "basis name too long"));
+            }
+            xous_ipc::String::<BASIS_NAME_LEN>::from_str(bname)
+        } else {
+            xous_ipc::String::<BASIS_NAME_LEN>::new()
+        };
+
+        let request = PddbKeyRequest {
+            basis_specified: basis_name.is_some(),
+            basis: xous_ipc::String::<BASIS_NAME_LEN>::from_str(&bname),
+            dict: xous_ipc::String::<DICT_NAME_LEN>::from_str(dict_name),
+            key: xous_ipc::String::<KEY_NAME_LEN>::new(),
+            create_dict: false,
+            create_key: false,
+            token: None,
+            result: PddbRequestCode::Uninit,
+            cb_sid: self.cb_sid.to_array(),
+            alloc_hint: None,
+        };
+        let mut buf = Buffer::into_buf(request)
+            .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
+        buf.lend_mut(self.conn, Opcode::DeleteKey.to_u32().unwrap())
+            .or(Err(Error::new(ErrorKind::Other, "Xous internal error")))?;
+
+        let response = buf.to_original::<PddbKeyRequest, _>().unwrap();
+        match response.result {
+            PddbRequestCode::NoErr => Ok(()),
+            PddbRequestCode::NotFound => Err(Error::new(ErrorKind::NotFound, "Dictionary or key was not found")),
+            _ => Err(Error::new(ErrorKind::Other, "Internal error"))
+        }
     }
-    // updates an existing key's value. mainly used by write().
-    pub fn update(&mut self, _key: PddbKey) -> Result<Option<PddbKey>> { Ok(None) }
-    // creates a key or overwrites it
-    pub fn insert(&mut self, _key: PddbKey) -> Option<PddbKey> { None } // may return the displaced key
-    // deletes a key within the dictionary
-    pub fn remove(&mut self, _key: PddbKey) -> Result<()> { Ok(()) }
-    // deletes the entire dictionary
-    pub fn delete(&mut self) {}
 }
 
-impl<'a> Drop for Pddb<'a> {
+impl Drop for Pddb {
     fn drop(&mut self) {
-        // the connection to the server side must be reference counted, so that multiple instances of this object within
-        // a single process do not end up de-allocating the CID on other threads before they go out of scope.
-        // Note to future me: you want this. Don't get rid of it because you think, "nah, nobody will ever make more than one copy of this object".
+        let cid = xous::connect(self.cb_sid).unwrap();
+        send_message(cid, Message::new_blocking_scalar(CbOp::Quit.to_usize().unwrap(), 0, 0, 0, 0)).unwrap();
+        unsafe{xous::disconnect(cid).unwrap();}
+        if let Some(handle) = self.cb_handle.take() {
+            handle.join().expect("couldn't terminate callback helper thread");
+        }
+
         REFCOUNT.store(REFCOUNT.load(Ordering::Relaxed) - 1, Ordering::Relaxed);
         if REFCOUNT.load(Ordering::Relaxed) == 0 {
             unsafe{xous::disconnect(self.conn).unwrap();}
         }
-        // if there was object-specific state (such as a one-time use server for async callbacks, specific to the object instance),
-        // de-allocate those items here. They don't need a reference count because they are object-specific
     }
 }

services/pddb/src/main.rs

@@ -378,12 +378,12 @@ mod tests;
 use tests::*;
 
 use num_traits::*;
-use xous::{send_message, Message};
+use xous::{send_message, Message, msg_blocking_scalar_unpack};
 use xous_ipc::Buffer;
 use core::cell::RefCell;
 use std::rc::Rc;
 use std::thread;
-use std::collections::HashSet;
+use std::collections::HashMap;
 use std::hash::{Hash, Hasher};
 use std::io::ErrorKind;
 use core::fmt::Write;
@@ -396,13 +396,14 @@ pub(crate) struct BasisRequestPassword {
     plaintext_pw: Option<xous_ipc::String::<{crate::api::PASSWORD_LEN}>>,
 }
 
-#[derive(Eq)]
+//#[derive(Eq)]
 struct TokenRecord {
-    token: ApiToken,
-    dict: String,
-    key: String,
-    basis: Option<String>,
+    pub dict: String,
+    pub key: String,
+    pub basis: Option<String>,
+    pub conn: xous::CID, // callback connection
 }
+/*
 impl Hash for TokenRecord {
     fn hash<H: Hasher>(&self, state: &mut H) {
         self.token.hash(state);
@@ -412,7 +413,7 @@ impl PartialEq for TokenRecord {
     fn eq(&self, other: &Self) -> bool {
         self.token == other.token
     }
-}
+}*/
 
 
 #[xous::xous_main]
@@ -439,7 +440,7 @@ fn xmain() -> ! {
     // storage for the basis cache
     let mut basis_cache = BasisCache::new();
     // storage for the token lookup: given an ApiToken, return a dict/key/basis set. Basis can be None or specified.
-    let mut token_dict = HashSet::<TokenRecord>::new();
+    let mut token_dict = HashMap::<ApiToken, TokenRecord>::new();
 
     // run the CI tests if the option has been selected
     #[cfg(all(
@@ -689,12 +690,172 @@ fn xmain() -> ! {
                     }
                 }
                 buffer.replace(mgmt).unwrap();
-            }
-            Some(Opcode::CreateDict) => {
-
             }
             Some(Opcode::KeyRequest) => {
-                // placeholder
+                let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
+                let mut req: PddbKeyRequest = buffer.to_original::<PddbKeyRequest, _>().unwrap();
+                let bname = if req.basis_specified {
+                    Some(req.basis.as_str().unwrap())
+                } else {
+                    None
+                };
+                let dict = req.dict.as_str().expect("dict utf-8 decode error");
+                let key = req.key.as_str().expect("key utf-8 decode error");
+                if basis_cache.dict_attributes(&mut pddb_os, dict, bname).is_err() {
+                    if req.create_dict {
+                        match basis_cache.dict_add(&mut pddb_os, dict, bname) {
+                            Ok(_) => (),
+                            Err(e) => {
+                                match e.kind() {
+                                    std::io::ErrorKind::OutOfMemory => {req.result = PddbRequestCode::NoFreeSpace; buffer.replace(req).unwrap(); continue}
+                                    std::io::ErrorKind::NotFound => {req.result = PddbRequestCode::NotMounted; buffer.replace(req).unwrap(); continue}
+                                    _ => {req.result = PddbRequestCode::InternalError; buffer.replace(req).unwrap(); continue}
+                                }
+                            }
+                        }
+                    } else {
+                        req.result = PddbRequestCode::NotFound;
+                        buffer.replace(req).unwrap(); continue
+                    }
+                }
+                if basis_cache.key_attributes(&mut pddb_os, dict, key, bname).is_err() {
+                    if !req.create_key {
+                        req.result = PddbRequestCode::NotFound;
+                        buffer.replace(req).unwrap(); continue
+                    }
+                    // by default keys are created when they are written on the first try.
+                    // ...do need to remember to create an "empty" key if we try to read a key that doesn't already exist, tho...
+                }
+                // at this point, we have established a basis/dict/key tuple.
+                let token: ApiToken = [pddb_os.trng_u32(), pddb_os.trng_u32(), pddb_os.trng_u32()];
+                let cid = xous::connect(xous::SID::from_array(req.cb_sid)).expect("couldn't connect for callback");
+                let token_record = TokenRecord {
+                    dict: String::from(dict),
+                    key: String::from(key),
+                    basis: if let Some(name) = bname {Some(String::from(name))} else {None},
+                    conn: cid,
+                };
+                token_dict.insert(token, token_record);
+                req.token = Some(token);
+                req.result = PddbRequestCode::NoErr;
+                buffer.replace(req).unwrap();
+            }
+            Some(Opcode::KeyDrop) => msg_blocking_scalar_unpack!(msg, t0, t1, t2, _, {
+                let token: ApiToken = [t0 as u32, t1 as u32, t2 as u32];
+                if let Some(rec) = token_dict.remove(&token) {
+                    // now check if we can safely disconnect and recycle our connection number.
+                    // This is important because we can only have 32 outgoing connections...
+                    let mut has_cid = false;
+                    for r in token_dict.values() {
+                        if r.conn == rec.conn {
+                            has_cid = true;
+                            break;
+                        }
+                    }
+                    if !has_cid {
+                        unsafe{xous::disconnect(rec.conn).expect("couldn't disconnect from callback server")};
+                    }
+                }
+                xous::return_scalar(msg.sender, 1).expect("couldn't ack KeyDrop");
+            }),
+            Some(Opcode::DeleteKey) => {
+                let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
+                let mut req: PddbKeyRequest = buffer.to_original::<PddbKeyRequest, _>().unwrap();
+                let bname = if req.basis_specified {
+                    Some(req.basis.as_str().unwrap())
+                } else {
+                    None
+                };
+                let dict = req.dict.as_str().expect("dict utf-8 decode error");
+                let key = req.key.as_str().expect("key utf-8 decode error");
+                match basis_cache.key_remove(&mut pddb_os, dict, key, bname, false) {
+                    Ok(_) => {
+                        let mut evict_list = Vec::<ApiToken>::new();
+                        // check to see if we need to eliminate any ApiTokens as a result of this.
+                        for (token, rec) in token_dict.iter() {
+                            if (rec.dict == dict) && (rec.key == key) {
+                                // check the basis union rules
+                                let mut matching = false;
+                                if rec.basis.is_none() && bname.is_none() {
+                                    matching = true;
+                                }
+                                if let Some(breq) = bname {
+                                    if rec.basis.is_none() {
+                                        matching = true;
+                                    }
+                                    if let Some(brec) = &rec.basis {
+                                        if brec == breq {
+                                            matching = true;
+                                        }
+                                    }
+                                }
+                                if matching {
+                                    evict_list.push(*token);
+                                }
+                            }
+                        }
+                        for token in evict_list {
+                            token_dict.remove(&token);
+                        }
+                        req.result = PddbRequestCode::NoErr;
+                    }
+                    Err(e) => {
+                        match e.kind() {
+                            std::io::ErrorKind::NotFound => req.result = PddbRequestCode::NotFound,
+                            _ => req.result = PddbRequestCode::InternalError,
+                        }
+                    }
+                }
+                buffer.replace(req).unwrap();
+            }
+            Some(Opcode::DeleteDict) => {
+                let mut buffer = unsafe { Buffer::from_memory_message_mut(msg.body.memory_message_mut().unwrap()) };
+                let mut req: PddbKeyRequest = buffer.to_original::<PddbKeyRequest, _>().unwrap();
+                let bname = if req.basis_specified {
+                    Some(req.basis.as_str().unwrap())
+                } else {
+                    None
+                };
+                let dict = req.dict.as_str().expect("dict utf-8 decode error");
+                match basis_cache.dict_remove(&mut pddb_os, dict, bname, false) {
+                    Ok(_) => {
+                        let mut evict_list = Vec::<ApiToken>::new();
+                        // check to see if we need to eliminate any ApiTokens as a result of this.
+                        for (token, rec) in token_dict.iter() {
+                            if rec.dict == dict {
+                                // check the basis union rules
+                                let mut matching = false;
+                                if rec.basis.is_none() && bname.is_none() {
+                                    matching = true;
+                                }
+                                if let Some(breq) = bname {
+                                    if rec.basis.is_none() {
+                                        matching = true;
+                                    }
+                                    if let Some(brec) = &rec.basis {
+                                        if brec == breq {
+                                            matching = true;
+                                        }
+                                    }
+                                }
+                                if matching {
+                                    evict_list.push(*token);
+                                }
+                            }
+                        }
+                        for token in evict_list {
+                            token_dict.remove(&token);
+                        }
+                        req.result = PddbRequestCode::NoErr;
+                    }
+                    Err(e) => {
+                        match e.kind() {
+                            std::io::ErrorKind::NotFound => req.result = PddbRequestCode::NotFound,
+                            _ => req.result = PddbRequestCode::InternalError,
+                        }
+                    }
+                }
+                buffer.replace(req).unwrap();
             }
             Some(Opcode::ReadKey) => {
                 // placeholder

services/shellchat/src/cmds/pddb_cmd.rs

@@ -2,12 +2,12 @@ use crate::{ShellCmdApi, CommonEnv};
 use xous_ipc::String;
 
 pub struct PddbCmd {
-    manager: pddb::PddbBasisManager,
+    pddb: pddb::Pddb,
 }
 impl PddbCmd {
     pub fn new(xns: &xous_names::XousNames) -> PddbCmd {
         PddbCmd {
-            manager: pddb::PddbBasisManager::new(),
+            pddb: pddb::Pddb::new(),
         }
     }
 }
@@ -21,14 +21,25 @@ impl<'a> ShellCmdApi<'a> for PddbCmd {
         let helpstring = "pddb [basislist]";
 
         let mut tokens = args.as_str().unwrap().split(' ');
+        let sid = xous::create_server().unwrap();
+        let cid = xous::connect(sid).unwrap();
 
         if let Some(sub_cmd) = tokens.next() {
             match sub_cmd {
                 "basislist" => {
-                    let bases = self.manager.list_basis();
+                    let bases = self.pddb.list_basis();
                     for basis in bases {
                         write!(ret, "{}\n", basis).unwrap();
                     }
+                    /*
+                    self.pddb.get("foo", "bar", None, false, false,
+                        Some({
+                            let cid = cid.clone();
+                            let counter = self.counter.clone();
+                            move || {
+                            xous::send_message(cid, xous::Message::new_scalar(0, counter as usize, 0, 0, 0)).expect("couldn't send");
+                        }})
+                    ).unwrap();*/
                 }
                 _ => {
                     write!(ret, "{}", helpstring).unwrap();