dht fixes

Cargo.lock

@@ -5232,6 +5232,12 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "shell-words"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
+
 [[package]]
 name = "shlex"
 version = "0.1.1"
@@ -6493,6 +6499,7 @@ dependencies = [
  "serde-big-array",
  "serde_json",
  "serial_test",
+ "shell-words",
  "simplelog 0.12.1",
  "socket2 0.5.3",
  "static_assertions",

veilid-core/Cargo.toml

@@ -56,6 +56,7 @@ enumset = { version= "^1", features = ["serde"] }
 backtrace = { version = "^0" }
 stop-token = { version = "^0", default-features = false }
 num-traits = "0.2.15"
+shell-words = "1.1.0"
 
 ed25519-dalek = { version = "^1", default_features = false, features = ["alloc", "u64_backend"] }
 x25519-dalek = { version = "^1", default_features = false, features = ["u64_backend"] }

veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs

@@ -91,11 +91,6 @@ impl RPCOperationGetValueA {
         if peers.len() > MAX_GET_VALUE_A_PEERS_LEN {
             return Err(RPCError::protocol("GetValueA peers length too long"));
         }
-        if descriptor.is_some() && !value.is_some() {
-            return Err(RPCError::protocol(
-                "GetValueA should not return descriptor without value",
-            ));
-        }
         Ok(Self {
             value,
             peers,
@@ -144,11 +139,6 @@ impl RPCOperationGetValueA {
                     get_value_context.vcrypto.clone(),
                 )
                 .map_err(RPCError::protocol)?;
-        } else {
-            // No value, should not have descriptor
-            if self.descriptor.is_some() {
-                return Err(RPCError::protocol("descriptor returned without a value"));
-            }
         }
 
         PeerInfo::validate_vec(&mut self.peers, validate_context.crypto.clone());

veilid-core/src/rpc_processor/rpc_get_value.rs

@@ -100,8 +100,9 @@ impl RPCProcessor {
         let (value, peers, descriptor) = get_value_a.destructure();
 
         let debug_string_value = value.as_ref().map(|v| {
-            format!(" len={} writer={}",
+            format!(" len={} seq={} writer={}",
                 v.value_data().data().len(),
+                v.value_data().seq(),
                 v.value_data().writer(),
             )
         }).unwrap_or_default();
@@ -210,8 +211,9 @@ impl RPCProcessor {
             .map_err(RPCError::internal)?);
         
         let debug_string_value = subkey_result.value.as_ref().map(|v| {
-            format!(" len={} writer={}",
+            format!(" len={} seq={} writer={}",
                 v.value_data().data().len(),
+                v.value_data().seq(),
                 v.value_data().writer(),
             )
         }).unwrap_or_default();

veilid-core/src/rpc_processor/rpc_set_value.rs

@@ -202,10 +202,11 @@ impl RPCProcessor {
         let closer_to_key_peers = network_result_try!(routing_table.find_peers_closer_to_key(key));
 
         let debug_string = format!(
-            "IN <=== SetValueQ({} #{} len={} writer={}{}) <== {}",
+            "IN <=== SetValueQ({} #{} len={} seq={} writer={}{}) <== {}",
             key,
             subkey,
             value.value_data().data().len(),
+            value.value_data().seq(),
             value.value_data().writer(),
             if descriptor.is_some() {
                 " +desc"
@@ -239,8 +240,9 @@ impl RPCProcessor {
         };
 
         let debug_string_value = new_value.as_ref().map(|v| {
-            format!(" len={} writer={}",
+            format!(" len={} seq={} writer={}",
                 v.value_data().data().len(),
+                v.value_data().seq(),
                 v.value_data().writer(),
             )
         }).unwrap_or_default();

veilid-core/src/storage_manager/debug.rs

@@ -35,4 +35,44 @@ impl StorageManager {
             .await;
         return format!("Remote records purged: reclaimed {} bytes", reclaimed);
     }
+    pub(crate) async fn debug_local_record_subkey_info(
+        &self,
+        key: TypedKey,
+        subkey: ValueSubkey,
+    ) -> String {
+        let inner = self.inner.lock().await;
+        let Some(local_record_store) = &inner.local_record_store else {
+            return "not initialized".to_owned();
+        };
+        local_record_store
+            .debug_record_subkey_info(key, subkey)
+            .await
+    }
+    pub(crate) async fn debug_remote_record_subkey_info(
+        &self,
+        key: TypedKey,
+        subkey: ValueSubkey,
+    ) -> String {
+        let inner = self.inner.lock().await;
+        let Some(remote_record_store) = &inner.remote_record_store else {
+            return "not initialized".to_owned();
+        };
+        remote_record_store
+            .debug_record_subkey_info(key, subkey)
+            .await
+    }
+    pub(crate) async fn debug_local_record_info(&self, key: TypedKey) -> String {
+        let inner = self.inner.lock().await;
+        let Some(local_record_store) = &inner.local_record_store else {
+            return "not initialized".to_owned();
+        };
+        local_record_store.debug_record_info(key)
+    }
+    pub(crate) async fn debug_remote_record_info(&self, key: TypedKey) -> String {
+        let inner = self.inner.lock().await;
+        let Some(remote_record_store) = &inner.remote_record_store else {
+            return "not initialized".to_owned();
+        };
+        remote_record_store.debug_record_info(key)
+    }
 }

veilid-core/src/storage_manager/mod.rs

@@ -144,6 +144,7 @@ impl StorageManager {
         // The initial writer is the owner of the record
         inner
             .open_existing_record(key, Some(owner), safety_selection)
+            .await
             .map(|r| r.unwrap())
     }
 
@@ -159,7 +160,10 @@ impl StorageManager {
         let mut inner = self.lock().await?;
 
         // See if we have a local record already or not
-        if let Some(res) = inner.open_existing_record(key, writer, safety_selection)? {
+        if let Some(res) = inner
+            .open_existing_record(key, writer, safety_selection)
+            .await?
+        {
             return Ok(res);
         }
 

veilid-core/src/storage_manager/record_store.rs

@@ -9,7 +9,7 @@ use hashlink::LruCache;
 
 pub struct RecordStore<D>
 where
-    D: Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
+    D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
     for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
     <D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
 {
@@ -41,7 +41,7 @@ pub struct SubkeyResult {
 
 impl<D> RecordStore<D>
 where
-    D: Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
+    D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
     for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
     <D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
 {
@@ -363,6 +363,20 @@ where
         out
     }
 
+    pub(super) fn peek_record<R, F>(&self, key: TypedKey, f: F) -> Option<R>
+    where
+        F: FnOnce(&Record<D>) -> R,
+    {
+        // Get record from index
+        let mut out = None;
+        let rtk = RecordTableKey { key };
+        if let Some(record) = self.record_index.peek(&rtk) {
+            // Callback
+            out = Some(f(record));
+        }
+        out
+    }
+
     pub(super) fn with_record_mut<R, F>(&mut self, key: TypedKey, f: F) -> Option<R>
     where
         F: FnOnce(&mut Record<D>) -> R,
@@ -454,6 +468,69 @@ where
         }));
     }
 
+    pub(crate) async fn peek_subkey(
+        &self,
+        key: TypedKey,
+        subkey: ValueSubkey,
+        want_descriptor: bool,
+    ) -> VeilidAPIResult<Option<SubkeyResult>> {
+        // record from index
+        let Some((subkey_count, has_subkey, opt_descriptor)) = self.peek_record(key, |record| {
+            (record.subkey_count(), record.stored_subkeys().contains(subkey), if want_descriptor {
+                Some(record.descriptor().clone())
+            } else {
+                None
+            })
+        }) else {
+            // Record not available
+            return Ok(None);
+        };
+
+        // Check if the subkey is in range
+        if subkey as usize >= subkey_count {
+            apibail_invalid_argument!("subkey out of range", "subkey", subkey);
+        }
+
+        // See if we have this subkey stored
+        if !has_subkey {
+            // If not, return no value but maybe with descriptor
+            return Ok(Some(SubkeyResult {
+                value: None,
+                descriptor: opt_descriptor,
+            }));
+        }
+
+        // Get subkey table
+        let Some(subkey_table) = self.subkey_table.clone() else {
+            apibail_internal!("record store not initialized");
+        };
+
+        // If subkey exists in subkey cache, use that
+        let stk = SubkeyTableKey { key, subkey };
+        if let Some(record_data) = self.subkey_cache.peek(&stk) {
+            let out = record_data.signed_value_data().clone();
+
+            return Ok(Some(SubkeyResult {
+                value: Some(out),
+                descriptor: opt_descriptor,
+            }));
+        }
+        // If not in cache, try to pull from table store if it is in our stored subkey set
+        let Some(record_data) = subkey_table
+            .load_rkyv::<RecordData>(0, &stk.bytes())
+            .await
+            .map_err(VeilidAPIError::internal)? else {
+                apibail_internal!("failed to peek subkey that was stored");
+            };
+
+        let out = record_data.signed_value_data().clone();
+
+        return Ok(Some(SubkeyResult {
+            value: Some(out),
+            descriptor: opt_descriptor,
+        }));
+    }
+
     pub async fn set_subkey(
         &mut self,
         key: TypedKey,
@@ -599,4 +676,23 @@ where
 
         out
     }
+
+    pub(super) fn debug_record_info(&self, key: TypedKey) -> String {
+        self.peek_record(key, |r| format!("{:#?}", r))
+            .unwrap_or("Not found".to_owned())
+    }
+
+    pub(super) async fn debug_record_subkey_info(
+        &self,
+        key: TypedKey,
+        subkey: ValueSubkey,
+    ) -> String {
+        match self.peek_subkey(key, subkey, true).await {
+            Ok(Some(v)) => {
+                format!("{:#?}", v)
+            }
+            Ok(None) => "Subkey not available".to_owned(),
+            Err(e) => format!("{}", e),
+        }
+    }
 }

veilid-core/src/storage_manager/set_value.rs

@@ -166,8 +166,19 @@ impl StorageManager {
     pub async fn inbound_set_value(&self, key: TypedKey, subkey: ValueSubkey, value: SignedValueData, descriptor: Option<SignedValueDescriptor>) -> VeilidAPIResult<NetworkResult<Option<SignedValueData>>> {
         let mut inner = self.lock().await?;
 
+        // See if this is a remote or local value
+        let (is_local, last_subkey_result) = {
             // See if the subkey we are modifying has a last known local value
             let last_subkey_result = inner.handle_get_local_value(key, subkey, true).await?;
+            // If this is local, it must have a descriptor already
+            if last_subkey_result.descriptor.is_some() {
+                (true, last_subkey_result)
+            } else {
+                // See if the subkey we are modifying has a last known remote value
+                let last_subkey_result = inner.handle_get_remote_value(key, subkey, true).await?;
+                (false, last_subkey_result)
+            }
+        };
 
         // Make sure this value would actually be newer
         if let Some(last_value) = &last_subkey_result.value {
@@ -210,7 +221,12 @@ impl StorageManager {
         }
 
         // Do the set and return no new value
-        match inner.handle_set_remote_value(key, subkey, value, actual_descriptor).await {            
+        let res = if is_local {
+            inner.handle_set_local_value(key, subkey, value).await
+        } else {
+            inner.handle_set_remote_value(key, subkey, value, actual_descriptor).await
+        };
+        match res {            
             Ok(()) => {},
             Err(VeilidAPIError::Internal { message }) => {
                 apibail_internal!(message);

veilid-core/src/storage_manager/storage_manager_inner.rs

@@ -209,7 +209,57 @@ impl StorageManagerInner {
         Ok((dht_key, owner))
     }
 
-    pub fn open_existing_record(
+    async fn move_remote_record_to_local(&mut self, key: TypedKey, safety_selection: SafetySelection) -> VeilidAPIResult<Option<(PublicKey, DHTSchema)>>
+    {
+        // Get local record store
+        let Some(local_record_store) = self.local_record_store.as_mut() else {
+            apibail_not_initialized!();
+        };
+        
+        // Get remote record store
+        let Some(remote_record_store) = self.remote_record_store.as_mut() else {
+            apibail_not_initialized!();
+        };
+
+        let rcb = |r: &Record<RemoteRecordDetail>| {
+            // Return record details
+            r.clone()
+        };
+        let Some(remote_record) = remote_record_store.with_record(key, rcb) else {
+            // No local or remote record found, return None
+            return Ok(None);
+        };
+
+        // Make local record
+        let cur_ts = get_aligned_timestamp();
+        let local_record = Record::new(cur_ts, remote_record.descriptor().clone(), LocalRecordDetail {
+            safety_selection
+        })?;
+        local_record_store.new_record(key, local_record).await?;
+
+        // Move copy subkey data from remote to local store
+        for subkey in remote_record.stored_subkeys().iter() {
+            let Some(subkey_result) = remote_record_store.get_subkey(key, subkey, false).await? else {
+                // Subkey was missing
+                warn!("Subkey was missing: {} #{}",key, subkey);
+                continue;
+            };
+            let Some(subkey_data) = subkey_result.value else {
+                // Subkey was missing
+                warn!("Subkey data was missing: {} #{}",key, subkey);
+                continue;
+            };
+            local_record_store.set_subkey(key, subkey, subkey_data).await?;
+        }
+
+        // Delete remote record from store
+        remote_record_store.delete_record(key).await?;
+
+        // Return record information as transferred to local record
+        Ok(Some((remote_record.owner().clone(), remote_record.schema())))
+    }
+
+    pub async fn open_existing_record(
         &mut self,
         key: TypedKey,
         writer: Option<KeyPair>,
@@ -235,9 +285,18 @@ impl StorageManagerInner {
             // Return record details
             (r.owner().clone(), r.schema())
         };
-        let Some((owner, schema)) = local_record_store.with_record_mut(key, cb) else {
+        let (owner, schema) = match local_record_store.with_record_mut(key, cb){
+            Some(v) => v,
+            None => {
+                // If we don't have a local record yet, check to see if we have a remote record
+                // if so, migrate it to a local record
+                let Some(v) = self.move_remote_record_to_local(key, safety_selection).await? else {
+                    // No remote record either
                     return Ok(None);
                 };
+                v
+            }
+        };
         // Had local record
 
         // If the writer we chose is also the owner, we have the owner secret
@@ -424,7 +483,7 @@ impl StorageManagerInner {
     /// # DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ]
     fn get_key<D>(vcrypto: CryptoSystemVersion, record: &Record<D>) -> TypedKey
     where
-        D: Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
+        D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
         for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
         <D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
     {

veilid-core/src/storage_manager/types/record.rs

@@ -6,7 +6,7 @@ use super::*;
 #[archive_attr(repr(C), derive(CheckBytes))]
 pub struct Record<D>
 where
-    D: Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
+    D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
     for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
     <D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
 {
@@ -20,7 +20,7 @@ where
 
 impl<D> Record<D>
 where
-    D: Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
+    D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
     for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
     <D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
 {

veilid-core/src/storage_manager/types/signed_value_descriptor.rs

@@ -5,7 +5,6 @@ use super::*;
 
 #[derive(
     Clone,
-    Debug,
     PartialOrd,
     PartialEq,
     Eq,
@@ -79,3 +78,13 @@ impl SignedValueDescriptor {
         self.schema_data.cmp(&other.schema_data)
     }
 }
+
+impl fmt::Debug for SignedValueDescriptor {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt.debug_struct("SignedValueDescriptor")
+            .field("owner", &self.owner)
+            .field("schema_data", &format!("{:?}", &self.schema_data))
+            .field("signature", &self.signature)
+            .finish()
+    }
+}

veilid-core/src/veilid_api/debug.rs

@@ -31,6 +31,19 @@ fn get_string(text: &str) -> Option<String> {
     Some(text.to_owned())
 }
 
+fn get_data(text: &str) -> Option<Vec<u8>> {
+    if text.starts_with("#") {
+        hex::decode(&text[1..]).ok()
+    } else if text.starts_with("\"") || text.starts_with("'") {
+        json::parse(text)
+            .ok()?
+            .as_str()
+            .map(|x| x.to_owned().as_bytes().to_vec())
+    } else {
+        Some(text.as_bytes().to_vec())
+    }
+}
+
 fn get_subkeys(text: &str) -> Option<ValueSubkeyRangeSet> {
     if let Some(n) = get_number(text) {
         Some(ValueSubkeyRangeSet::single(n.try_into().ok()?))
@@ -88,7 +101,12 @@ fn get_route_id(
     };
 }
 
-fn get_safety_selection(text: &str, routing_table: RoutingTable) -> Option<SafetySelection> {
+fn get_dht_schema(text: &str) -> Option<DHTSchema> {
+    deserialize_json::<DHTSchema>(text).ok()
+}
+
+fn get_safety_selection(routing_table: RoutingTable) -> impl Fn(&str) -> Option<SafetySelection> {
+    move |text| {
         let rss = routing_table.route_spec_store();
         let default_route_hop_count =
             routing_table.with_config(|c| c.network.rpc.default_route_hop_count as usize);
@@ -127,6 +145,7 @@ fn get_safety_selection(text: &str, routing_table: RoutingTable) -> Option<Safet
             };
             Some(SafetySelection::Safe(ss))
         }
+    }
 }
 
 fn get_node_ref_modifiers(mut node_ref: NodeRef) -> impl FnOnce(&str) -> Option<NodeRef> {
@@ -150,7 +169,7 @@ fn get_destination(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option<D
     move |text| {
         // Safety selection
         let (text, ss) = if let Some((first, second)) = text.split_once('+') {
-            let ss = get_safety_selection(second, routing_table.clone())?;
+            let ss = get_safety_selection(routing_table.clone())(second)?;
             (first, Some(ss))
         } else {
             (text, None)
@@ -234,6 +253,9 @@ fn get_typed_key(text: &str) -> Option<TypedKey> {
 fn get_public_key(text: &str) -> Option<PublicKey> {
     PublicKey::from_str(text).ok()
 }
+fn get_keypair(text: &str) -> Option<KeyPair> {
+    KeyPair::from_str(text).ok()
+}
 
 fn get_crypto_system_version(crypto: Crypto) -> impl FnOnce(&str) -> Option<CryptoSystemVersion> {
     move |text| {
@@ -249,7 +271,7 @@ fn get_dht_key(
     move |text| {
         // Safety selection
         let (text, ss) = if let Some((first, second)) = text.split_once('+') {
-            let ss = get_safety_selection(second, routing_table.clone())?;
+            let ss = get_safety_selection(routing_table.clone())(second)?;
             (first, Some(ss))
         } else {
             (text, None)
@@ -389,25 +411,32 @@ fn get_debug_argument_at<T, G: FnOnce(&str) -> Option<T>>(
     Ok(val)
 }
 
-fn print_data_truncated(data: &[u8]) -> String {
+pub fn print_data(data: &[u8], truncate_len: Option<usize>) -> String {
     // check is message body is ascii printable
     let mut printable = true;
     for c in data {
         if *c < 32 || *c > 126 {
             printable = false;
+            break;
         }
     }
 
-    let (data, truncated) = if data.len() > 64 {
+    let (data, truncated) = if truncate_len.is_some() && data.len() > truncate_len.unwrap() {
         (&data[0..64], true)
     } else {
         (&data[..], false)
     };
 
     let strdata = if printable {
-        format!("\"{}\"", String::from_utf8_lossy(&data).to_string())
+        format!("{}", String::from_utf8_lossy(&data).to_string())
     } else {
-        hex::encode(data)
+        let sw = shell_words::quote(&String::from_utf8_lossy(&data).to_string()).to_string();
+        let h = hex::encode(data);
+        if h.len() < sw.len() {
+            h
+        } else {
+            sw
+        }
     };
     if truncated {
         format!("{}...", strdata)
@@ -416,14 +445,6 @@ fn print_data_truncated(data: &[u8]) -> String {
     }
 }
 
-fn print_value_data(value_data: ValueData) -> String {
-    format!(
-        "ValueData {{\n  seq: {},\n  writer: {},\n  data: {}\n}}\n",
-        value_data.seq(),
-        value_data.writer(),
-        print_data_truncated(value_data.data())
-    )
-}
 impl VeilidAPI {
     async fn debug_buckets(&self, args: String) -> VeilidAPIResult<String> {
         let args: Vec<String> = args.split_whitespace().map(|s| s.to_owned()).collect();
@@ -1010,6 +1031,62 @@ impl VeilidAPI {
         };
         return Ok(out);
     }
+
+    async fn debug_record_create(&self, args: Vec<String>) -> VeilidAPIResult<String> {
+        let netman = self.network_manager()?;
+        let routing_table = netman.routing_table();
+        let crypto = self.crypto()?;
+
+        let csv = get_debug_argument_at(
+            &args,
+            1,
+            "debug_record_create",
+            "kind",
+            get_crypto_system_version(crypto.clone()),
+        )
+        .unwrap_or_else(|_| crypto.best());
+        let schema = get_debug_argument_at(
+            &args,
+            2,
+            "debug_record_create",
+            "dht_schema",
+            get_dht_schema,
+        )
+        .unwrap_or_else(|_| DHTSchema::dflt(1));
+        let ss = get_debug_argument_at(
+            &args,
+            3,
+            "debug_record_create",
+            "safety_selection",
+            get_safety_selection(routing_table),
+        )
+        .ok();
+
+        // Get routing context with optional privacy
+        let rc = self.routing_context();
+        let rc = if let Some(ss) = ss {
+            let rcp = match rc.with_custom_privacy(ss) {
+                Err(e) => return Ok(format!("Can't use safety selection: {}", e)),
+                Ok(v) => v,
+            };
+            rcp
+        } else {
+            rc
+        };
+
+        // Do a record get
+        let record = match rc.create_dht_record(csv.kind(), schema).await {
+            Err(e) => return Ok(format!("Can't open DHT record: {}", e)),
+            Ok(v) => v,
+        };
+        match rc.close_dht_record(*record.key()).await {
+            Err(e) => return Ok(format!("Can't close DHT record: {}", e)),
+            Ok(v) => v,
+        };
+        debug!("DHT Record Created:\n{:#?}", record);
+        return Ok(format!("{:?}", record));
+    }
+
     async fn debug_record_get(&self, args: Vec<String>) -> VeilidAPIResult<String> {
         let netman = self.network_manager()?;
         let routing_table = netman.routing_table();
@@ -1080,7 +1157,66 @@ impl VeilidAPI {
             Ok(v) => v,
         };
         let out = if let Some(value) = value {
-            print_value_data(value)
+            format!("{:?}", value)
+        } else {
+            "No value data returned".to_owned()
+        };
+        match rc.close_dht_record(key).await {
+            Err(e) => return Ok(format!("Can't close DHT record: {}", e)),
+            Ok(v) => v,
+        };
+        return Ok(out);
+    }
+
+    async fn debug_record_set(&self, args: Vec<String>) -> VeilidAPIResult<String> {
+        let netman = self.network_manager()?;
+        let routing_table = netman.routing_table();
+
+        let (key, ss) = get_debug_argument_at(
+            &args,
+            1,
+            "debug_record_set",
+            "key",
+            get_dht_key(routing_table),
+        )?;
+        let subkey = get_debug_argument_at(&args, 2, "debug_record_set", "subkey", get_number)?;
+        let writer = get_debug_argument_at(&args, 3, "debug_record_set", "writer", get_keypair)?;
+        let data = get_debug_argument_at(&args, 4, "debug_record_set", "data", get_data)?;
+
+        // Get routing context with optional privacy
+        let rc = self.routing_context();
+        let rc = if let Some(ss) = ss {
+            let rcp = match rc.with_custom_privacy(ss) {
+                Err(e) => return Ok(format!("Can't use safety selection: {}", e)),
+                Ok(v) => v,
+            };
+            rcp
+        } else {
+            rc
+        };
+
+        // Do a record get
+        let _record = match rc.open_dht_record(key, Some(writer)).await {
+            Err(e) => return Ok(format!("Can't open DHT record: {}", e)),
+            Ok(v) => v,
+        };
+        let value = match rc.set_dht_value(key, subkey as ValueSubkey, data).await {
+            Err(e) => {
+                match rc.close_dht_record(key).await {
+                    Err(e) => {
+                        return Ok(format!(
+                            "Can't set DHT value and can't close DHT record: {}",
+                            e
+                        ))
+                    }
+                    Ok(v) => v,
+                };
+                return Ok(format!("Can't set DHT value: {}", e));
+            }
+            Ok(v) => v,
+        };
+        let out = if let Some(value) = value {
+            format!("{:?}", value)
         } else {
             "No value data returned".to_owned()
         };
@@ -1104,46 +1240,35 @@ impl VeilidAPI {
     }
 
     async fn debug_record_info(&self, args: Vec<String>) -> VeilidAPIResult<String> {
-        let netman = self.network_manager()?;
-        let routing_table = netman.routing_table();
+        let storage_manager = self.storage_manager()?;
 
-        let (key, ss) = get_debug_argument_at(
-            &args,
-            1,
-            "debug_record_info",
-            "key",
-            get_dht_key(routing_table),
-        )?;
+        let key = get_debug_argument_at(&args, 1, "debug_record_info", "key", get_typed_key)?;
 
-        // Get routing context with optional privacy
-        let rc = self.routing_context();
-        let rc = if let Some(ss) = ss {
-            let rcp = match rc.with_custom_privacy(ss) {
-                Err(e) => return Ok(format!("Can't use safety selection: {}", e)),
-                Ok(v) => v,
-            };
-            rcp
+        let subkey =
+            get_debug_argument_at(&args, 2, "debug_record_info", "subkey", get_number).ok();
+
+        let out = if let Some(subkey) = subkey {
+            let li = storage_manager
+                .debug_local_record_subkey_info(key, subkey as ValueSubkey)
+                .await;
+            let ri = storage_manager
+                .debug_remote_record_subkey_info(key, subkey as ValueSubkey)
+                .await;
+            format!(
+                "Local Subkey Info:\n{}\n\nRemote Subkey Info:\n{}\n",
+                li, ri
+            )
         } else {
-            rc
-        };
-
-        // Do a record get
-        let record = match rc.open_dht_record(key, None).await {
-            Err(e) => return Ok(format!("Can't open DHT record: {}", e)),
-            Ok(v) => v,
-        };
-
-        let out = format!("{:#?}", record);
-
-        match rc.close_dht_record(key).await {
-            Err(e) => return Ok(format!("Can't close DHT record: {}", e)),
-            Ok(v) => v,
+            let li = storage_manager.debug_local_record_info(key).await;
+            let ri = storage_manager.debug_remote_record_info(key).await;
+            format!("Local Info:\n{}\n\nRemote Info:\n{}\n", li, ri)
         };
         return Ok(out);
     }
 
     async fn debug_record(&self, args: String) -> VeilidAPIResult<String> {
-        let args: Vec<String> = args.split_whitespace().map(|s| s.to_owned()).collect();
+        let args: Vec<String> =
+            shell_words::split(&args).map_err(|e| VeilidAPIError::parse_error(e, args))?;
 
         let command = get_debug_argument_at(&args, 0, "debug_record", "command", get_string)?;
 
@@ -1151,8 +1276,12 @@ impl VeilidAPI {
             self.debug_record_list(args).await
         } else if command == "purge" {
             self.debug_record_purge(args).await
+        } else if command == "create" {
+            self.debug_record_create(args).await
         } else if command == "get" {
             self.debug_record_get(args).await
+        } else if command == "set" {
+            self.debug_record_set(args).await
         } else if command == "delete" {
             self.debug_record_delete(args).await
         } else if command == "info" {
@@ -1187,9 +1316,11 @@ route allocate [ord|*ord] [rel] [<count>] [in|out]
       test <route>
 record list <local|remote>
        purge <local|remote> [bytes]
+       create <cryptokind> <dhtschema> <safety>
+       set <key>[+<safety>] <subkey> <writer> <data> 
        get <key>[+<safety>] <subkey> [force]
        delete <key>
-       info <key>
+       info <key> [subkey]
 --------------------------------------------------------------------
 <key> is: VLD0:GsgXCRPrzSK6oBNgxhNpm-rTYFd02R0ySx6j9vbQBG4
     * also <node>, <relay>, <target>, <route>
@@ -1205,10 +1336,16 @@ record list <local|remote>
 <protocoltype> is: udp|tcp|ws|wss
 <addresstype> is: ipv4|ipv6
 <routingdomain> is: public|local
+<cryptokind> is: VLD0
+<dhtschema> is: a json dht schema, default is '{"kind":"DFLT","o_cnt":1}'
 <subkey> is: a number: 2
 <subkeys> is: 
     * a number: 2
     * a comma-separated inclusive range list: 1..=3,5..=8
+<data> is:
+    * a single-word string: foobar
+    * a shell-quoted string: "foo\nbar\n"
+    * a '#' followed by hex data: #12AB34CD...
 "#
         .to_owned())
     }

veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs

@@ -46,6 +46,9 @@ impl DHTRecordDescriptor {
         }
     }
 
+    pub fn key(&self) -> &TypedKey {
+        &self.key
+    }
     pub fn owner(&self) -> &PublicKey {
         &self.owner
     }

veilid-core/src/veilid_api/types/dht/value_data.rs

@@ -2,7 +2,6 @@ use super::*;
 
 #[derive(
     Clone,
-    Debug,
     Default,
     PartialOrd,
     PartialEq,
@@ -61,3 +60,13 @@ impl ValueData {
         mem::size_of::<Self>() + self.data.len()
     }
 }
+
+impl fmt::Debug for ValueData {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt.debug_struct("ValueData")
+            .field("seq", &self.seq)
+            .field("data", &print_data(&self.data, None))
+            .field("writer", &self.writer)
+            .finish()
+    }
+}

veilid-python/tests/test_dht.py

@@ -193,18 +193,12 @@ async def test_open_writer_dht_value(api_connection: veilid.VeilidAPI):
         assert rec.schema.o_cnt == 2
 
         # Verify subkey 1 can NOT be set because we have the wrong writer
-        vdtemp = await rec.set_dht_value(key, 1, va)
-        assert vdtemp != None
-        assert vdtemp.data == vc
-        assert vdtemp.seq == 2
-        assert vdtemp.writer == owner        
+        with pytest.raises(veilid.VeilidAPIError):
+            vdtemp = await rc.set_dht_value(key, 1, va)
     
         # Verify subkey 0 can NOT be set because we have the wrong writer
-        vdtemp = await rec.set_dht_value(key, 0, va)
-        assert vdtemp != None
-        assert vdtemp.data == vb
-        assert vdtemp.seq == 1
-        assert vdtemp.writer == owner
+        with pytest.raises(veilid.VeilidAPIError):
+            vdtemp = await rc.set_dht_value(key, 0, va)
         
         # Clean up
         await rc.close_dht_record(key)