json schema generation

veilid-core/proto/veilid.capnp

@@ -496,8 +496,9 @@ struct Question @0xd8510bc33492ef70 {
         getValueQ           @5  :OperationGetValueQ;
         setValueQ           @6  :OperationSetValueQ;
         watchValueQ         @7  :OperationWatchValueQ;
-        supplyBlockQ        @8  :OperationSupplyBlockQ;
+        # #[cfg(feature="unstable-blockstore")]
-        findBlockQ          @9  :OperationFindBlockQ;
+        # supplyBlockQ        @8  :OperationSupplyBlockQ;
+        # findBlockQ          @9  :OperationFindBlockQ;
         
         # Tunnel operations
         # #[cfg(feature="unstable-tunnels")]
@@ -534,8 +535,9 @@ struct Answer @0xacacb8b6988c1058 {
         getValueA           @3  :OperationGetValueA;
         setValueA           @4  :OperationSetValueA;
         watchValueA         @5  :OperationWatchValueA;    
-        supplyBlockA        @6  :OperationSupplyBlockA; 
+        # #[cfg(feature="unstable-blockstore")]
-        findBlockA          @7  :OperationFindBlockA;
+        #supplyBlockA        @6  :OperationSupplyBlockA; 
+        #findBlockA          @7  :OperationFindBlockA;
     
         # Tunnel operations
         # #[cfg(feature="unstable-tunnels")]

veilid-core/src/attachment_manager.rs

@@ -30,7 +30,7 @@ impl AttachmentManager {
         storage_manager: StorageManager,
         protected_store: ProtectedStore,
         table_store: TableStore,
-        block_store: BlockStore,
+        #[cfg(feature = "unstable-blockstore")] block_store: BlockStore,
         crypto: Crypto,
     ) -> AttachmentManagerUnlockedInner {
         AttachmentManagerUnlockedInner {
@@ -40,6 +40,7 @@ impl AttachmentManager {
                 storage_manager,
                 protected_store,
                 table_store,
+                #[cfg(feature = "unstable-blockstore")]
                 block_store,
                 crypto,
             ),
@@ -60,7 +61,7 @@ impl AttachmentManager {
         storage_manager: StorageManager,
         protected_store: ProtectedStore,
         table_store: TableStore,
-        block_store: BlockStore,
+        #[cfg(feature = "unstable-blockstore")] block_store: BlockStore,
         crypto: Crypto,
     ) -> Self {
         Self {
@@ -70,6 +71,7 @@ impl AttachmentManager {
                 storage_manager,
                 protected_store,
                 table_store,
+                #[cfg(feature = "unstable-blockstore")]
                 block_store,
                 crypto,
             )),

veilid-core/src/core_context.rs

@@ -17,6 +17,7 @@ struct ServicesContext {
 
     pub protected_store: Option<ProtectedStore>,
     pub table_store: Option<TableStore>,
+    #[cfg(feature = "unstable-blockstore")]
     pub block_store: Option<BlockStore>,
     pub crypto: Option<Crypto>,
     pub attachment_manager: Option<AttachmentManager>,
@@ -30,6 +31,7 @@ impl ServicesContext {
             update_callback,
             protected_store: None,
             table_store: None,
+            #[cfg(feature = "unstable-blockstore")]
             block_store: None,
             crypto: None,
             attachment_manager: None,
@@ -42,7 +44,7 @@ impl ServicesContext {
         update_callback: UpdateCallback,
         protected_store: ProtectedStore,
         table_store: TableStore,
-        block_store: BlockStore,
+        #[cfg(feature = "unstable-blockstore")] block_store: BlockStore,
         crypto: Crypto,
         attachment_manager: AttachmentManager,
         storage_manager: StorageManager,
@@ -52,6 +54,7 @@ impl ServicesContext {
             update_callback,
             protected_store: Some(protected_store),
             table_store: Some(table_store),
+            #[cfg(feature = "unstable-blockstore")]
             block_store: Some(block_store),
             crypto: Some(crypto),
             attachment_manager: Some(attachment_manager),
@@ -103,14 +106,17 @@ impl ServicesContext {
         self.crypto = Some(crypto.clone());
 
         // Set up block store
-        trace!("init block store");
+        #[cfg(feature = "unstable-blockstore")]
-        let block_store = BlockStore::new(self.config.clone());
+        {
-        if let Err(e) = block_store.init().await {
+            trace!("init block store");
-            error!("failed to init block store: {}", e);
+            let block_store = BlockStore::new(self.config.clone());
-            self.shutdown().await;
+            if let Err(e) = block_store.init().await {
-            return Err(e);
+                error!("failed to init block store: {}", e);
+                self.shutdown().await;
+                return Err(e);
+            }
+            self.block_store = Some(block_store.clone());
         }
-        self.block_store = Some(block_store.clone());
 
         // Set up storage manager
         trace!("init storage manager");
@@ -119,6 +125,7 @@ impl ServicesContext {
             self.crypto.clone().unwrap(),
             self.protected_store.clone().unwrap(),
             self.table_store.clone().unwrap(),
+            #[cfg(feature = "unstable-blockstore")]
             self.block_store.clone().unwrap(),
         );
         if let Err(e) = storage_manager.init().await {
@@ -136,6 +143,7 @@ impl ServicesContext {
             storage_manager,
             protected_store,
             table_store,
+            #[cfg(feature = "unstable-blockstore")]
             block_store,
             crypto,
         );
@@ -162,6 +170,7 @@ impl ServicesContext {
             trace!("terminate storage manager");
             storage_manager.terminate().await;
         }
+        #[cfg(feature = "unstable-blockstore")]
         if let Some(block_store) = &mut self.block_store {
             trace!("terminate block store");
             block_store.terminate().await;
@@ -198,6 +207,7 @@ pub struct VeilidCoreContext {
     pub storage_manager: StorageManager,
     pub protected_store: ProtectedStore,
     pub table_store: TableStore,
+    #[cfg(feature = "unstable-blockstore")]
     pub block_store: BlockStore,
     pub crypto: Crypto,
     pub attachment_manager: AttachmentManager,
@@ -251,6 +261,7 @@ impl VeilidCoreContext {
             storage_manager: sc.storage_manager.unwrap(),
             protected_store: sc.protected_store.unwrap(),
             table_store: sc.table_store.unwrap(),
+            #[cfg(feature = "unstable-blockstore")]
             block_store: sc.block_store.unwrap(),
             crypto: sc.crypto.unwrap(),
             attachment_manager: sc.attachment_manager.unwrap(),
@@ -264,6 +275,7 @@ impl VeilidCoreContext {
             self.update_callback.clone(),
             self.protected_store,
             self.table_store,
+            #[cfg(feature = "unstable-blockstore")]
             self.block_store,
             self.crypto,
             self.attachment_manager,

veilid-core/src/intf/native/mod.rs

@@ -1,8 +1,12 @@
+#[cfg(feature = "unstable-blockstore")]
 mod block_store;
+
 mod protected_store;
 mod system;
 
+#[cfg(feature = "unstable-blockstore")]
 pub use block_store::*;
+
 pub use protected_store::*;
 pub use system::*;
 

veilid-core/src/intf/wasm/mod.rs

@@ -1,8 +1,12 @@
+#[cfg(feature = "unstable-blockstore")]
 mod block_store;
+
 mod protected_store;
 mod system;
 
+#[cfg(feature = "unstable-blockstore")]
 pub use block_store::*;
+
 pub use protected_store::*;
 pub use system::*;
 

veilid-core/src/network_manager/mod.rs

@@ -150,6 +150,7 @@ struct NetworkManagerUnlockedInner {
     storage_manager: StorageManager,
     protected_store: ProtectedStore,
     table_store: TableStore,
+    #[cfg(feature="unstable-blockstore")]
     block_store: BlockStore,
     crypto: Crypto,
     // Accessors
@@ -181,6 +182,7 @@ impl NetworkManager {
         storage_manager: StorageManager,
         protected_store: ProtectedStore,
         table_store: TableStore,
+        #[cfg(feature="unstable-blockstore")]
         block_store: BlockStore,
         crypto: Crypto,
     ) -> NetworkManagerUnlockedInner {
@@ -189,6 +191,7 @@ impl NetworkManager {
             storage_manager,
             protected_store,
             table_store,
+            #[cfg(feature="unstable-blockstore")]
             block_store,
             crypto,
             routing_table: RwLock::new(None),
@@ -204,6 +207,7 @@ impl NetworkManager {
         storage_manager: StorageManager,
         protected_store: ProtectedStore,
         table_store: TableStore,
+        #[cfg(feature="unstable-blockstore")]
         block_store: BlockStore,
         crypto: Crypto,
     ) -> Self {
@@ -214,6 +218,7 @@ impl NetworkManager {
                 storage_manager,
                 protected_store,
                 table_store,
+                #[cfg(feature="unstable-blockstore")]
                 block_store,
                 crypto,
             )),
@@ -241,6 +246,7 @@ impl NetworkManager {
     pub fn table_store(&self) -> TableStore {
         self.unlocked_inner.table_store.clone()
     }
+    #[cfg(feature="unstable-blockstore")]
     pub fn block_store(&self) -> BlockStore {
         self.unlocked_inner.block_store.clone()
     }

veilid-core/src/routing_table/tests/test_serialize.rs

@@ -2,6 +2,7 @@ use crate::*;
 
 fn fake_routing_table() -> routing_table::RoutingTable {
     let veilid_config = VeilidConfig::new();
+    #[cfg(feature = "unstable-blockstore")]
     let block_store = BlockStore::new(veilid_config.clone());
     let protected_store = ProtectedStore::new(veilid_config.clone());
     let table_store = TableStore::new(veilid_config.clone(), protected_store.clone());
@@ -11,6 +12,7 @@ fn fake_routing_table() -> routing_table::RoutingTable {
         crypto.clone(),
         protected_store.clone(),
         table_store.clone(),
+        #[cfg(feature = "unstable-blockstore")]
         block_store.clone(),
     );
     let network_manager = network_manager::NetworkManager::new(
@@ -18,6 +20,7 @@ fn fake_routing_table() -> routing_table::RoutingTable {
         storage_manager,
         protected_store.clone(),
         table_store.clone(),
+        #[cfg(feature = "unstable-blockstore")]
         block_store.clone(),
         crypto.clone(),
     );

veilid-core/src/rpc_processor/coders/operations/answer.rs

@@ -37,7 +37,9 @@ pub enum RPCAnswerDetail {
     GetValueA(RPCOperationGetValueA),
     SetValueA(RPCOperationSetValueA),
     WatchValueA(RPCOperationWatchValueA),
+    #[cfg(feature = "unstable-blockstore")]
     SupplyBlockA(RPCOperationSupplyBlockA),
+    #[cfg(feature = "unstable-blockstore")]
     FindBlockA(RPCOperationFindBlockA),
     #[cfg(feature = "unstable-tunnels")]
     StartTunnelA(RPCOperationStartTunnelA),
@@ -56,7 +58,9 @@ impl RPCAnswerDetail {
             RPCAnswerDetail::GetValueA(_) => "GetValueA",
             RPCAnswerDetail::SetValueA(_) => "SetValueA",
             RPCAnswerDetail::WatchValueA(_) => "WatchValueA",
+            #[cfg(feature = "unstable-blockstore")]
             RPCAnswerDetail::SupplyBlockA(_) => "SupplyBlockA",
+            #[cfg(feature = "unstable-blockstore")]
             RPCAnswerDetail::FindBlockA(_) => "FindBlockA",
             #[cfg(feature = "unstable-tunnels")]
             RPCAnswerDetail::StartTunnelA(_) => "StartTunnelA",
@@ -74,7 +78,9 @@ impl RPCAnswerDetail {
             RPCAnswerDetail::GetValueA(r) => r.validate(validate_context),
             RPCAnswerDetail::SetValueA(r) => r.validate(validate_context),
             RPCAnswerDetail::WatchValueA(r) => r.validate(validate_context),
+            #[cfg(feature = "unstable-blockstore")]
             RPCAnswerDetail::SupplyBlockA(r) => r.validate(validate_context),
+            #[cfg(feature = "unstable-blockstore")]
             RPCAnswerDetail::FindBlockA(r) => r.validate(validate_context),
             #[cfg(feature = "unstable-tunnels")]
             RPCAnswerDetail::StartTunnelA(r) => r.validate(validate_context),
@@ -119,11 +125,13 @@ impl RPCAnswerDetail {
                 let out = RPCOperationWatchValueA::decode(&op_reader)?;
                 RPCAnswerDetail::WatchValueA(out)
             }
+            #[cfg(feature = "unstable-blockstore")]
             veilid_capnp::answer::detail::SupplyBlockA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
                 let out = RPCOperationSupplyBlockA::decode(&op_reader)?;
                 RPCAnswerDetail::SupplyBlockA(out)
             }
+            #[cfg(feature = "unstable-blockstore")]
             veilid_capnp::answer::detail::FindBlockA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
                 let out = RPCOperationFindBlockA::decode(&op_reader)?;
@@ -163,9 +171,11 @@ impl RPCAnswerDetail {
             RPCAnswerDetail::WatchValueA(d) => {
                 d.encode(&mut builder.reborrow().init_watch_value_a())
             }
+            #[cfg(feature = "unstable-blockstore")]
             RPCAnswerDetail::SupplyBlockA(d) => {
                 d.encode(&mut builder.reborrow().init_supply_block_a())
             }
+            #[cfg(feature = "unstable-blockstore")]
             RPCAnswerDetail::FindBlockA(d) => d.encode(&mut builder.reborrow().init_find_block_a()),
             #[cfg(feature = "unstable-tunnels")]
             RPCAnswerDetail::StartTunnelA(d) => {

veilid-core/src/rpc_processor/coders/operations/mod.rs

@@ -2,7 +2,6 @@ mod answer;
 mod operation;
 mod operation_app_call;
 mod operation_app_message;
-mod operation_find_block;
 mod operation_find_node;
 mod operation_get_value;
 mod operation_return_receipt;
@@ -10,7 +9,7 @@ mod operation_route;
 mod operation_set_value;
 mod operation_signal;
 mod operation_status;
-mod operation_supply_block;
+
 mod operation_validate_dial_info;
 mod operation_value_changed;
 mod operation_watch_value;
@@ -18,6 +17,11 @@ mod question;
 mod respond_to;
 mod statement;
 
+#[cfg(feature = "unstable-blockstore")]
+mod operation_find_block;
+#[cfg(feature = "unstable-blockstore")]
+mod operation_supply_block;
+
 #[cfg(feature = "unstable-tunnels")]
 mod operation_cancel_tunnel;
 #[cfg(feature = "unstable-tunnels")]
@@ -29,7 +33,6 @@ pub use answer::*;
 pub use operation::*;
 pub use operation_app_call::*;
 pub use operation_app_message::*;
-pub use operation_find_block::*;
 pub use operation_find_node::*;
 pub use operation_get_value::*;
 pub use operation_return_receipt::*;
@@ -37,7 +40,6 @@ pub use operation_route::*;
 pub use operation_set_value::*;
 pub use operation_signal::*;
 pub use operation_status::*;
-pub use operation_supply_block::*;
 pub use operation_validate_dial_info::*;
 pub use operation_value_changed::*;
 pub use operation_watch_value::*;
@@ -45,6 +47,11 @@ pub use question::*;
 pub use respond_to::*;
 pub use statement::*;
 
+#[cfg(feature = "unstable-blockstore")]
+pub use operation_find_block::*;
+#[cfg(feature = "unstable-blockstore")]
+pub use operation_supply_block::*;
+
 #[cfg(feature = "unstable-tunnels")]
 pub use operation_cancel_tunnel::*;
 #[cfg(feature = "unstable-tunnels")]

veilid-core/src/rpc_processor/coders/operations/question.rs

@@ -49,7 +49,9 @@ pub enum RPCQuestionDetail {
     GetValueQ(RPCOperationGetValueQ),
     SetValueQ(RPCOperationSetValueQ),
     WatchValueQ(RPCOperationWatchValueQ),
+    #[cfg(feature = "unstable-blockstore")]
     SupplyBlockQ(RPCOperationSupplyBlockQ),
+    #[cfg(feature = "unstable-blockstore")]
     FindBlockQ(RPCOperationFindBlockQ),
     #[cfg(feature = "unstable-tunnels")]
     StartTunnelQ(RPCOperationStartTunnelQ),
@@ -68,7 +70,9 @@ impl RPCQuestionDetail {
             RPCQuestionDetail::GetValueQ(_) => "GetValueQ",
             RPCQuestionDetail::SetValueQ(_) => "SetValueQ",
             RPCQuestionDetail::WatchValueQ(_) => "WatchValueQ",
+            #[cfg(feature = "unstable-blockstore")]
             RPCQuestionDetail::SupplyBlockQ(_) => "SupplyBlockQ",
+            #[cfg(feature = "unstable-blockstore")]
             RPCQuestionDetail::FindBlockQ(_) => "FindBlockQ",
             #[cfg(feature = "unstable-tunnels")]
             RPCQuestionDetail::StartTunnelQ(_) => "StartTunnelQ",
@@ -86,7 +90,9 @@ impl RPCQuestionDetail {
             RPCQuestionDetail::GetValueQ(r) => r.validate(validate_context),
             RPCQuestionDetail::SetValueQ(r) => r.validate(validate_context),
             RPCQuestionDetail::WatchValueQ(r) => r.validate(validate_context),
+            #[cfg(feature = "unstable-blockstore")]
             RPCQuestionDetail::SupplyBlockQ(r) => r.validate(validate_context),
+            #[cfg(feature = "unstable-blockstore")]
             RPCQuestionDetail::FindBlockQ(r) => r.validate(validate_context),
             #[cfg(feature = "unstable-tunnels")]
             RPCQuestionDetail::StartTunnelQ(r) => r.validate(validate_context),
@@ -132,11 +138,13 @@ impl RPCQuestionDetail {
                 let out = RPCOperationWatchValueQ::decode(&op_reader)?;
                 RPCQuestionDetail::WatchValueQ(out)
             }
+            #[cfg(feature = "unstable-blockstore")]
             veilid_capnp::question::detail::SupplyBlockQ(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
                 let out = RPCOperationSupplyBlockQ::decode(&op_reader)?;
                 RPCQuestionDetail::SupplyBlockQ(out)
             }
+            #[cfg(feature = "unstable-blockstore")]
             veilid_capnp::question::detail::FindBlockQ(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
                 let out = RPCOperationFindBlockQ::decode(&op_reader)?;
@@ -176,9 +184,11 @@ impl RPCQuestionDetail {
             RPCQuestionDetail::WatchValueQ(d) => {
                 d.encode(&mut builder.reborrow().init_watch_value_q())
             }
+            #[cfg(feature = "unstable-blockstore")]
             RPCQuestionDetail::SupplyBlockQ(d) => {
                 d.encode(&mut builder.reborrow().init_supply_block_q())
             }
+            #[cfg(feature = "unstable-blockstore")]
             RPCQuestionDetail::FindBlockQ(d) => {
                 d.encode(&mut builder.reborrow().init_find_block_q())
             }

veilid-core/src/rpc_processor/mod.rs

@@ -5,7 +5,6 @@ mod operation_waiter;
 mod rpc_app_call;
 mod rpc_app_message;
 mod rpc_error;
-mod rpc_find_block;
 mod rpc_find_node;
 mod rpc_get_value;
 mod rpc_return_receipt;
@@ -13,11 +12,15 @@ mod rpc_route;
 mod rpc_set_value;
 mod rpc_signal;
 mod rpc_status;
-mod rpc_supply_block;
 mod rpc_validate_dial_info;
 mod rpc_value_changed;
 mod rpc_watch_value;
 
+#[cfg(feature = "unstable-blockstore")]
+mod rpc_find_block;
+#[cfg(feature = "unstable-blockstore")]
+mod rpc_supply_block;
+
 #[cfg(feature = "unstable-tunnels")]
 mod rpc_cancel_tunnel;
 #[cfg(feature = "unstable-tunnels")]
@@ -1412,7 +1415,9 @@ impl RPCProcessor {
                 RPCQuestionDetail::GetValueQ(_) => self.process_get_value_q(msg).await,
                 RPCQuestionDetail::SetValueQ(_) => self.process_set_value_q(msg).await,
                 RPCQuestionDetail::WatchValueQ(_) => self.process_watch_value_q(msg).await,
+                #[cfg(feature = "unstable-blockstore")]
                 RPCQuestionDetail::SupplyBlockQ(_) => self.process_supply_block_q(msg).await,
+                #[cfg(feature = "unstable-blockstore")]
                 RPCQuestionDetail::FindBlockQ(_) => self.process_find_block_q(msg).await,
                 #[cfg(feature = "unstable-tunnels")]
                 RPCQuestionDetail::StartTunnelQ(_) => self.process_start_tunnel_q(msg).await,

veilid-core/src/storage_manager/mod.rs

@@ -30,6 +30,7 @@ struct StorageManagerUnlockedInner {
     crypto: Crypto,
     protected_store: ProtectedStore,
     table_store: TableStore,
+    #[cfg(feature = "unstable-blockstore")]
     block_store: BlockStore,
 
     // Background processes
@@ -48,13 +49,14 @@ impl StorageManager {
         crypto: Crypto,
         protected_store: ProtectedStore,
         table_store: TableStore,
-        block_store: BlockStore,
+        #[cfg(feature = "unstable-blockstore")] block_store: BlockStore,
     ) -> StorageManagerUnlockedInner {
         StorageManagerUnlockedInner {
             config,
             crypto,
             protected_store,
             table_store,
+            #[cfg(feature = "unstable-blockstore")]
             block_store,
             flush_record_stores_task: TickTask::new(FLUSH_RECORD_STORES_INTERVAL_SECS),
         }
@@ -68,13 +70,14 @@ impl StorageManager {
         crypto: Crypto,
         protected_store: ProtectedStore,
         table_store: TableStore,
-        block_store: BlockStore,
+        #[cfg(feature = "unstable-blockstore")] block_store: BlockStore,
     ) -> StorageManager {
         let unlocked_inner = Arc::new(Self::new_unlocked_inner(
             config,
             crypto,
             protected_store,
             table_store,
+            #[cfg(feature = "unstable-blockstore")]
             block_store,
         ));
         let this = StorageManager {

veilid-core/src/veilid_api/api.rs

@@ -70,6 +70,7 @@ impl VeilidAPI {
         }
         Err(VeilidAPIError::not_initialized())
     }
+    #[cfg(feature = "unstable-blockstore")]
     pub fn block_store(&self) -> VeilidAPIResult<BlockStore> {
         let inner = self.inner.lock();
         if let Some(context) = &inner.context {

veilid-core/src/veilid_api/json_api/crypto_system.rs

@@ -18,49 +18,210 @@ pub struct CryptoSystemResponse {
 #[serde(tag = "cs_op")]
 pub enum CryptoSystemRequestOp {
     Release,
-    CachedDh,
+    CachedDh {
-    ComputeDh,
+        #[schemars(with = "String")]
-    RandomBytes,
+        key: PublicKey,
+        #[schemars(with = "String")]
+        secret: SecretKey,
+    },
+    ComputeDh {
+        #[schemars(with = "String")]
+        key: PublicKey,
+        #[schemars(with = "String")]
+        secret: SecretKey,
+    },
+    RandomBytes {
+        len: u32,
+    },
     DefaultSaltLength,
-    HashPassword,
+    HashPassword {
-    VerifyPassword,
+        #[serde(with = "json_as_base64")]
-    DeriveSharedSecret,
+        #[schemars(with = "String")]
+        password: Vec<u8>,
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        salt: Vec<u8>,
+    },
+    VerifyPassword {
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        password: Vec<u8>,
+        password_hash: String,
+    },
+    DeriveSharedSecret {
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        password: Vec<u8>,
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        salt: Vec<u8>,
+    },
     RandomNonce,
     RandomSharedSecret,
     GenerateKeyPair,
-    GenerateHash,
+    GenerateHash {
-    ValidateKeyPair,
+        #[serde(with = "json_as_base64")]
-    ValidateHash,
+        #[schemars(with = "String")]
-    Distance,
+        data: Vec<u8>,
-    Sign,
+    },
-    Verify,
+    ValidateKeyPair {
+        #[schemars(with = "String")]
+        key: PublicKey,
+        #[schemars(with = "String")]
+        secret: SecretKey,
+    },
+    ValidateHash {
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        data: Vec<u8>,
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        hash_digest: Vec<u8>,
+    },
+    Distance {
+        #[schemars(with = "String")]
+        key1: CryptoKey,
+        #[schemars(with = "String")]
+        key2: CryptoKey,
+    },
+    Sign {
+        #[schemars(with = "String")]
+        key: PublicKey,
+        #[schemars(with = "String")]
+        secret: SecretKey,
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        data: Vec<u8>,
+    },
+    Verify {
+        #[schemars(with = "String")]
+        key: PublicKey,
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        data: Vec<u8>,
+        #[schemars(with = "String")]
+        secret: Signature,
+    },
     AeadOverhead,
-    DecryptAead,
+    DecryptAead {
-    EncryptAead,
+        #[serde(with = "json_as_base64")]
-    CryptNoAuth,
+        #[schemars(with = "String")]
+        body: Vec<u8>,
+        #[schemars(with = "String")]
+        nonce: Nonce,
+        #[schemars(with = "String")]
+        shared_secret: SharedSecret,
+        #[serde(with = "opt_json_as_base64")]
+        #[schemars(with = "Option<String>")]
+        associated_data: Option<Vec<u8>>,
+    },
+    EncryptAead {
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        body: Vec<u8>,
+        #[schemars(with = "String")]
+        nonce: Nonce,
+        #[schemars(with = "String")]
+        shared_secret: SharedSecret,
+        #[serde(with = "opt_json_as_base64")]
+        #[schemars(with = "Option<String>")]
+        associated_data: Option<Vec<u8>>,
+    },
+    CryptNoAuth {
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        body: Vec<u8>,
+        #[schemars(with = "String")]
+        nonce: Nonce,
+        #[schemars(with = "String")]
+        shared_secret: SharedSecret,
+    },
 }
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 #[serde(tag = "cs_op")]
 pub enum CryptoSystemResponseOp {
     Release,
-    CachedDh,
+    CachedDh {
-    ComputeDh,
+        #[serde(flatten)]
-    RandomBytes,
+        #[schemars(with = "ApiResult<String>")]
-    DefaultSaltLength,
+        result: ApiResultWithString<SharedSecret>,
-    HashPassword,
+    },
-    VerifyPassword,
+    ComputeDh {
-    DeriveSharedSecret,
+        #[serde(flatten)]
-    RandomNonce,
+        #[schemars(with = "ApiResult<String>")]
-    RandomSharedSecret,
+        result: ApiResultWithString<SharedSecret>,
-    GenerateKeyPair,
+    },
-    GenerateHash,
+    RandomBytes {
-    ValidateKeyPair,
+        #[serde(flatten)]
-    ValidateHash,
+        #[schemars(with = "ApiResult<String>")]
-    Distance,
+        result: ApiResultWithVecU8,
-    Sign,
+    },
-    Verify,
+    DefaultSaltLength {
-    AeadOverhead,
+        value: u32,
-    DecryptAead,
+    },
-    EncryptAead,
+    HashPassword {
-    CryptNoAuth,
+        #[serde(flatten)]
+        result: ApiResult<String>,
+    },
+    VerifyPassword {
+        #[serde(flatten)]
+        result: ApiResult<bool>,
+    },
+    DeriveSharedSecret {
+        #[serde(flatten)]
+        #[schemars(with = "ApiResult<String>")]
+        result: ApiResultWithString<SharedSecret>,
+    },
+    RandomNonce {
+        #[schemars(with = "String")]
+        value: Nonce,
+    },
+    RandomSharedSecret {
+        #[schemars(with = "String")]
+        value: SharedSecret,
+    },
+    GenerateKeyPair {
+        #[schemars(with = "String")]
+        value: KeyPair,
+    },
+    GenerateHash {
+        #[schemars(with = "String")]
+        value: HashDigest,
+    },
+    ValidateKeyPair {
+        value: bool,
+    },
+    ValidateHash {
+        value: bool,
+    },
+    Distance {
+        #[schemars(with = "String")]
+        value: CryptoKeyDistance,
+    },
+    Sign {
+        #[schemars(with = "String")]
+        value: Signature,
+    },
+    Verify {
+        #[serde(flatten)]
+        result: ApiResult<()>,
+    },
+    AeadOverhead {
+        value: u32,
+    },
+    DecryptAead {
+        #[serde(flatten)]
+        #[schemars(with = "ApiResult<String>")]
+        result: ApiResultWithVecU8,
+    },
+    EncryptAead {
+        #[serde(flatten)]
+        #[schemars(with = "ApiResult<String>")]
+        result: ApiResultWithVecU8,
+    },
+    CryptNoAuth {
+        #[serde(flatten)]
+        #[schemars(with = "ApiResult<String>")]
+        result: ApiResultWithVecU8,
+    },
 }

veilid-core/src/veilid_api/json_api/mod.rs

@@ -11,7 +11,8 @@ pub use crypto_system::*;
 
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 pub struct Request {
-    /// Operation Id (pairs with Response)
+    /// Operation Id (pairs with Response, or empty if unidirectional)
+    #[serde(default)]
     id: String,
     /// The request operation variant
     #[serde(flatten)]
@@ -20,7 +21,8 @@ pub struct Request {
 
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 pub struct Response {
-    /// Operation Id (pairs with Request)
+    /// Operation Id (pairs with Request, or empty if unidirectional)
+    #[serde(default)]
     id: String,
     /// The response operation variant
     #[serde(flatten)]
@@ -36,17 +38,14 @@ pub enum RequestOp {
     NewPrivateRoute,
     NewCustomPrivateRoute {
         #[schemars(with = "Vec<String>")]
-        crypto_kinds: Vec<CryptoKind>,
+        kinds: Vec<CryptoKind>,
         #[serde(default)]
         stability: Stability,
         #[serde(default)]
         sequencing: Sequencing,
     },
     ImportRemotePrivateRoute {
-        #[serde(
+        #[serde(with = "json_as_base64")]
-            serialize_with = "json_as_base64::serialize",
-            deserialize_with = "json_as_base64::deserialize"
-        )]
         #[schemars(with = "String")]
         blob: Vec<u8>,
     },
@@ -57,10 +56,7 @@ pub enum RequestOp {
     AppCallReply {
         #[schemars(with = "String")]
         call_id: OperationId,
-        #[serde(
+        #[serde(with = "json_as_base64")]
-            serialize_with = "json_as_base64::serialize",
-            deserialize_with = "json_as_base64::deserialize"
-        )]
         #[schemars(with = "String")]
         message: Vec<u8>,
     },
@@ -79,27 +75,21 @@ pub enum RequestOp {
     // Crypto
     GetCryptoSystem {
         #[schemars(with = "String")]
-        crypto_kind: CryptoKind,
+        kind: CryptoKind,
     },
     BestCryptoSystem,
     CryptoSystem(CryptoSystemRequest),
     VerifySignatures {
         #[schemars(with = "Vec<String>")]
         node_ids: Vec<TypedKey>,
-        #[serde(
+        #[serde(with = "json_as_base64")]
-            serialize_with = "json_as_base64::serialize",
-            deserialize_with = "json_as_base64::deserialize"
-        )]
         #[schemars(with = "String")]
         data: Vec<u8>,
         #[schemars(with = "Vec<String>")]
         signatures: Vec<TypedSignature>,
     },
     GenerateSignatures {
-        #[serde(
+        #[serde(with = "json_as_base64")]
-            serialize_with = "json_as_base64::serialize",
-            deserialize_with = "json_as_base64::deserialize"
-        )]
         #[schemars(with = "String")]
         data: Vec<u8>,
         #[schemars(with = "Vec<String>")]
@@ -107,7 +97,7 @@ pub enum RequestOp {
     },
     GenerateKeyPair {
         #[schemars(with = "String")]
-        crypto_kind: CryptoKind,
+        kind: CryptoKind,
     },
     // Misc
     Now,
@@ -122,10 +112,7 @@ pub enum RequestOp {
 pub struct NewPrivateRouteResult {
     #[schemars(with = "String")]
     route_id: RouteId,
-    #[serde(
+    #[serde(with = "json_as_base64")]
-        serialize_with = "json_as_base64::serialize",
-        deserialize_with = "json_as_base64::deserialize"
-    )]
     #[schemars(with = "String")]
     blob: Vec<u8>,
 }
@@ -133,6 +120,9 @@ pub struct NewPrivateRouteResult {
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 #[serde(tag = "op")]
 pub enum ResponseOp {
+    Update {
+        value: VeilidUpdate,
+    },
     GetState {
         #[serde(flatten)]
         result: ApiResult<VeilidState>,
@@ -153,28 +143,74 @@ pub enum ResponseOp {
         #[serde(flatten)]
         result: ApiResult<NewPrivateRouteResult>,
     },
-    ImportRemotePrivateRoute,
+    ImportRemotePrivateRoute {
-    ReleasePrivateRoute,
+        #[serde(flatten)]
-    AppCallReply,
+        #[schemars(with = "ApiResult<String>")]
+        result: ApiResultWithString<RouteId>,
+    },
+    ReleasePrivateRoute {
+        #[serde(flatten)]
+        result: ApiResult<()>,
+    },
+    AppCallReply {
+        #[serde(flatten)]
+        result: ApiResult<()>,
+    },
     // Routing Context
-    NewRoutingContext,
+    NewRoutingContext {
+        value: String,
+    },
     RoutingContext(RoutingContextResponse),
     // TableDb
-    OpenTableDb,
+    OpenTableDb {
-    DeleteTableDb,
+        #[serde(flatten)]
+        result: ApiResult<String>,
+    },
+    DeleteTableDb {
+        #[serde(flatten)]
+        result: ApiResult<bool>,
+    },
     TableDb(TableDbResponse),
     // Crypto
-    GetCryptoSystem,
+    GetCryptoSystem {
-    BestCryptoSystem,
+        #[serde(flatten)]
+        result: ApiResult<String>,
+    },
+    BestCryptoSystem {
+        value: String,
+    },
     CryptoSystem(CryptoSystemResponse),
-    VerifySignatures,
+    VerifySignatures {
-    GenerateSignatures,
+        #[serde(flatten)]
-    GenerateKeyPair,
+        #[schemars(with = "ApiResult<Vec<String>>")]
+        result: ApiResultWithVecString<TypedKeySet>,
+    },
+    GenerateSignatures {
+        #[serde(flatten)]
+        #[schemars(with = "ApiResult<Vec<String>>")]
+        result: ApiResultWithVecString<TypedSignatureSet>,
+    },
+    GenerateKeyPair {
+        #[serde(flatten)]
+        #[schemars(with = "ApiResult<String>")]
+        result: ApiResultWithString<TypedKeyPair>,
+    },
     // Misc
-    Now,
+    Now {
-    Debug,
+        #[schemars(with = "String")]
-    VeilidVersionString,
+        value: Timestamp,
-    VeilidVersion,
+    },
+    Debug {
+        value: String,
+    },
+    VeilidVersionString {
+        value: String,
+    },
+    VeilidVersion {
+        major: u32,
+        minor: u32,
+        patch: u32,
+    },
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
@@ -187,6 +223,49 @@ where
     Err { error: VeilidAPIError },
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(untagged)]
+pub enum ApiResultWithString<T>
+where
+    T: Clone + fmt::Debug,
+{
+    Ok {
+        #[schemars(with = "String")]
+        value: T,
+    },
+    Err {
+        error: VeilidAPIError,
+    },
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(untagged)]
+pub enum ApiResultWithVecU8 {
+    Ok {
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        value: Vec<u8>,
+    },
+    Err {
+        error: VeilidAPIError,
+    },
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(untagged)]
+pub enum ApiResultWithVecString<T>
+where
+    T: Clone + fmt::Debug,
+{
+    Ok {
+        #[schemars(with = "Vec<String>")]
+        value: T,
+    },
+    Err {
+        error: VeilidAPIError,
+    },
+}
+
 pub fn emit_schemas(out: &mut HashMap<String, String>) {
     let schema_request = schema_for!(Request);
     let schema_response = schema_for!(Response);

veilid-core/src/veilid_api/json_api/routing_context.rs

@@ -19,34 +19,122 @@ pub struct RoutingContextResponse {
 pub enum RoutingContextRequestOp {
     Release,
     WithPrivacy,
-    WithCustomPrivacy,
+    WithCustomPrivacy {
-    WithSequencing,
+        stability: Stability,
-    AppCall,
+    },
-    AppMessage,
+    WithSequencing {
-    CreateDhtRecord,
+        sequencing: Sequencing,
-    OpenDhtRecord,
+    },
-    CloseDhtRecord,
+    AppCall {
-    DeleteDhtRecord,
+        target: String,
-    GetDhtValue,
+        #[serde(with = "json_as_base64")]
-    SetDhtValue,
+        #[schemars(with = "String")]
-    WatchDhtValues,
+        request: Vec<u8>,
-    CancelDhtWatch,
+    },
+    AppMessage {
+        target: String,
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        message: Vec<u8>,
+    },
+    CreateDhtRecord {
+        #[schemars(with = "String")]
+        kind: CryptoKind,
+        schema: DHTSchema,
+    },
+    OpenDhtRecord {
+        #[schemars(with = "String")]
+        key: TypedKey,
+        #[schemars(with = "Option<String>")]
+        writer: Option<KeyPair>,
+    },
+    CloseDhtRecord {
+        #[schemars(with = "String")]
+        key: TypedKey,
+    },
+    DeleteDhtRecord {
+        #[schemars(with = "String")]
+        key: TypedKey,
+    },
+    GetDhtValue {
+        #[schemars(with = "String")]
+        key: TypedKey,
+        subkey: ValueSubkey,
+        force_refresh: bool,
+    },
+    SetDhtValue {
+        #[schemars(with = "String")]
+        key: TypedKey,
+        subkey: ValueSubkey,
+        #[serde(with = "json_as_base64")]
+        #[schemars(with = "String")]
+        data: Vec<u8>,
+    },
+    WatchDhtValues {
+        #[schemars(with = "String")]
+        key: TypedKey,
+        subkeys: ValueSubkeyRangeSet,
+        expiration: Timestamp,
+        count: u32,
+    },
+    CancelDhtWatch {
+        #[schemars(with = "String")]
+        key: TypedKey,
+        subkeys: ValueSubkeyRangeSet,
+    },
 }
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 #[serde(tag = "rc_op")]
 pub enum RoutingContextResponseOp {
     Release,
-    WithPrivacy,
+    WithPrivacy {
-    WithCustomPrivacy,
+        value: String,
-    WithSequencing,
+    },
-    AppCall,
+    WithCustomPrivacy {
-    AppMessage,
+        value: String,
-    CreateDhtRecord,
+    },
-    OpenDhtRecord,
+    WithSequencing {
-    CloseDhtRecord,
+        value: String,
-    DeleteDhtRecord,
+    },
-    GetDhtValue,
+    AppCall {
-    SetDhtValue,
+        #[serde(flatten)]
-    WatchDhtValues,
+        #[schemars(with = "ApiResult<String>")]
-    CancelDhtWatch,
+        result: ApiResultWithVecU8,
+    },
+    AppMessage {
+        #[serde(flatten)]
+        result: ApiResult<()>,
+    },
+    CreateDhtRecord {
+        #[serde(flatten)]
+        result: ApiResult<DHTRecordDescriptor>,
+    },
+    OpenDhtRecord {
+        #[serde(flatten)]
+        result: ApiResult<DHTRecordDescriptor>,
+    },
+    CloseDhtRecord {
+        #[serde(flatten)]
+        result: ApiResult<()>,
+    },
+    DeleteDhtRecord {
+        #[serde(flatten)]
+        result: ApiResult<()>,
+    },
+    GetDhtValue {
+        #[serde(flatten)]
+        result: ApiResult<Option<ValueData>>,
+    },
+    SetDhtValue {
+        #[serde(flatten)]
+        result: ApiResult<Option<ValueData>>,
+    },
+    WatchDhtValues {
+        #[serde(flatten)]
+        result: ApiResult<Timestamp>,
+    },
+    CancelDhtWatch {
+        #[serde(flatten)]
+        result: ApiResult<bool>,
+    },
 }

veilid-core/src/veilid_api/mod.rs

@@ -22,6 +22,7 @@ pub use alloc::string::ToString;
 pub use attachment_manager::AttachmentManager;
 pub use core::str::FromStr;
 pub use crypto::*;
+#[cfg(feature = "unstable-blockstore")]
 pub use intf::BlockStore;
 pub use intf::ProtectedStore;
 pub use network_manager::NetworkManager;

veilid-core/src/veilid_api/routing_context.rs

@@ -290,10 +290,12 @@ impl RoutingContext {
     ///////////////////////////////////
     /// Block Store
 
+    #[cfg(feature = "unstable-blockstore")]
     pub async fn find_block(&self, _block_id: PublicKey) -> VeilidAPIResult<Vec<u8>> {
         panic!("unimplemented");
     }
 
+    #[cfg(feature = "unstable-blockstore")]
     pub async fn supply_block(&self, _block_id: PublicKey) -> VeilidAPIResult<bool> {
         panic!("unimplemented");
     }

veilid-core/src/veilid_api/serialize_helpers/serialize_json.rs

@@ -55,6 +55,27 @@ pub mod json_as_base64 {
     }
 }
 
+pub mod opt_json_as_base64 {
+    use data_encoding::BASE64URL_NOPAD;
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+    pub fn serialize<S: Serializer>(v: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
+        let base64 = v.as_ref().map(|x| BASE64URL_NOPAD.encode(&x));
+        Option::<String>::serialize(&base64, s)
+    }
+
+    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Vec<u8>>, D::Error> {
+        let base64 = Option::<String>::deserialize(d)?;
+        base64
+            .map(|x| {
+                BASE64URL_NOPAD
+                    .decode(x.as_bytes())
+                    .map_err(|e| serde::de::Error::custom(e))
+            })
+            .transpose()
+    }
+}
+
 pub mod json_as_string {
     use std::fmt::Display;
     use std::str::FromStr;

veilid-core/src/veilid_api/types/aligned_u64.rs

@@ -24,10 +24,7 @@ use super::*;
 #[archive_attr(repr(C, align(8)), derive(CheckBytes))]
 #[serde(transparent)]
 pub struct AlignedU64(
-    #[serde(
+    #[serde(with = "json_as_string")]
-        serialize_with = "json_as_string::serialize",
-        deserialize_with = "json_as_string::deserialize"
-    )]
     #[schemars(with = "String")]
     u64,
 );

veilid-core/src/veilid_api/types/app_message_call.rs

@@ -16,18 +16,12 @@ use super::*;
 #[archive_attr(repr(C), derive(CheckBytes))]
 pub struct VeilidAppMessage {
     /// Some(sender) if the message was sent directly, None if received via a private/safety route
-    #[serde(
+    #[serde(with = "opt_json_as_string")]
-        serialize_with = "opt_json_as_string::serialize",
-        deserialize_with = "opt_json_as_string::deserialize"
-    )]
     #[schemars(with = "Option<String>")]
     sender: Option<TypedKey>,
 
     /// The content of the message to deliver to the application
-    #[serde(
+    #[serde(with = "json_as_base64")]
-        serialize_with = "json_as_base64::serialize",
-        deserialize_with = "json_as_base64::deserialize"
-    )]
     #[schemars(with = "String")]
     message: Vec<u8>,
 }
@@ -61,26 +55,17 @@ impl VeilidAppMessage {
 #[archive_attr(repr(C), derive(CheckBytes))]
 pub struct VeilidAppCall {
     /// Some(sender) if the request was sent directly, None if received via a private/safety route
-    #[serde(
+    #[serde(with = "opt_json_as_string")]
-        serialize_with = "opt_json_as_string::serialize",
-        deserialize_with = "opt_json_as_string::deserialize"
-    )]
     #[schemars(with = "Option<String>")]
     sender: Option<TypedKey>,
 
     /// The content of the request to deliver to the application
-    #[serde(
+    #[serde(with = "json_as_base64")]
-        serialize_with = "json_as_base64::serialize",
-        deserialize_with = "json_as_base64::deserialize"
-    )]
     #[schemars(with = "String")]
     message: Vec<u8>,
 
     /// The id to reply to
-    #[serde(
+    #[serde(with = "json_as_string")]
-        serialize_with = "json_as_string::serialize",
-        deserialize_with = "json_as_string::deserialize"
-    )]
     #[schemars(with = "String")]
     id: OperationId,
 }

veilid-core/src/veilid_api/types/dht/value_data.rs

@@ -21,10 +21,7 @@ pub struct ValueData {
     seq: ValueSeqNum,
 
     /// The contents of a DHT Record
-    #[serde(
+    #[serde(with = "json_as_base64")]
-        serialize_with = "json_as_base64::serialize",
-        deserialize_with = "json_as_base64::deserialize"
-    )]
     #[schemars(with = "String")]
     data: Vec<u8>,
 

veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs

@@ -18,12 +18,10 @@ use range_set_blaze::*;
     JsonSchema,
 )]
 #[archive_attr(repr(C), derive(CheckBytes))]
+#[serde(transparent)]
 pub struct ValueSubkeyRangeSet {
     #[with(RkyvRangeSetBlaze)]
-    #[serde(
+    #[serde(with = "serialize_range_set_blaze")]
-        serialize_with = "serialize_range_set_blaze::serialize",
-        deserialize_with = "serialize_range_set_blaze::deserialize"
-    )]
     #[schemars(with = "Vec<(u32,u32)>")]
     data: RangeSetBlaze<ValueSubkey>,
 }