Improved WatchValue

Cargo.lock

@@ -691,6 +691,12 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
 [[package]]
 name = "base64ct"
 version = "1.7.3"
@@ -954,6 +960,7 @@ dependencies = [
  "iana-time-zone",
  "js-sys",
  "num-traits",
+ "serde",
  "wasm-bindgen",
  "windows-link",
 ]
@@ -1575,6 +1582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058"
 dependencies = [
  "powerfmt",
+ "serde",
 ]
 
 [[package]]
@@ -2395,7 +2403,7 @@ version = "7.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "byteorder",
  "flate2",
  "nom",
@@ -2877,6 +2885,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
 dependencies = [
  "autocfg",
  "hashbrown 0.12.3",
+ "serde",
 ]
 
 [[package]]
@@ -2887,6 +2896,7 @@ checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
 dependencies = [
  "equivalent",
  "hashbrown 0.15.2",
+ "serde",
 ]
 
 [[package]]
@@ -4639,7 +4649,7 @@ version = "0.11.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "bytes 1.10.1",
  "encoding_rs",
  "futures-core",
@@ -4829,7 +4839,7 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
 dependencies = [
- "base64",
+ "base64 0.21.7",
 ]
 
 [[package]]
@@ -5158,6 +5168,36 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_with"
+version = "3.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa"
+dependencies = [
+ "base64 0.22.1",
+ "chrono",
+ "hex",
+ "indexmap 1.9.3",
+ "indexmap 2.8.0",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "serde_with_macros",
+ "time",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "3.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e"
+dependencies = [
+ "darling 0.20.10",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
 [[package]]
 name = "serde_yaml"
 version = "0.9.34+deprecated"
@@ -5845,7 +5885,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
 dependencies = [
  "async-trait",
  "axum",
- "base64",
+ "base64 0.21.7",
  "bytes 1.10.1",
  "futures-core",
  "futures-util",
@@ -5874,7 +5914,7 @@ dependencies = [
  "async-stream",
  "async-trait",
  "axum",
- "base64",
+ "base64 0.21.7",
  "bytes 1.10.1",
  "h2",
  "http 0.2.12",
@@ -6475,6 +6515,7 @@ dependencies = [
  "serde-wasm-bindgen 0.6.5",
  "serde_bytes",
  "serde_json",
+ "serde_with",
  "serial_test 2.0.0",
  "sha2 0.10.8",
  "shell-words",

veilid-cli/src/client_api_connection.rs

@@ -173,7 +173,7 @@ impl ClientApiConnection {
             let mut inner = this.inner.lock();
             inner.request_sender = None;
         };
-        unord.push(system_boxed(recv_messages_future));
+        unord.push(pin_dyn_future!(recv_messages_future));
 
         // Requests send processor
         let send_requests_future = async move {
@@ -183,7 +183,7 @@ impl ClientApiConnection {
                 }
             }
         };
-        unord.push(system_boxed(send_requests_future));
+        unord.push(pin_dyn_future!(send_requests_future));
 
         // Request initial server state
         let capi = self.clone();

veilid-core/Cargo.toml

@@ -147,6 +147,7 @@ lz4_flex = { version = "0.11.3", default-features = false, features = [
 ] }
 indent = "0.1.1"
 sanitize-filename = "0.5.0"
+serde_with = "3.12.0"
 
 # Dependencies for native builds only
 # Linux, Windows, Mac, iOS, Android

veilid-core/proto/veilid.capnp

@@ -353,8 +353,8 @@ struct OperationSetValueA @0x9378d0732dc95be2 {
 
 struct OperationWatchValueQ @0xf9a5a6c547b9b228 {
     key                     @0  :TypedKey;              # key for value to watch
-    subkeys                 @1  :List(SubkeyRange);     # subkey range to watch (up to 512 subranges), if empty this implies 0..=UINT32_MAX
+    subkeys                 @1  :List(SubkeyRange);     # subkey range to watch (up to 512 subranges). An empty range here should not be specified unless cancelling a watch (count=0).
-    expiration              @2  :UInt64;                # requested timestamp when this watch will expire in usec since epoch (can be return less, 0 for max)
+    expiration              @2  :UInt64;                # requested timestamp when this watch will expire in usec since epoch (watch can return less, 0 for max)
     count                   @3  :UInt32;                # requested number of changes to watch for (0 = cancel, 1 = single shot, 2+ = counter, UINT32_MAX = continuous)
     watchId                 @4  :UInt64;                # if 0, request a new watch. if >0, existing watch id 
     watcher                 @5  :PublicKey;             # the watcher performing the watch, can be the owner or a schema member, or a generated anonymous watch keypair

veilid-core/src/component.rs

@@ -105,6 +105,7 @@ impl VeilidComponentRegistry {
         self.namespace
     }
 
+    #[allow(dead_code)]
     pub fn program_name(&self) -> &'static str {
         self.program_name
     }
@@ -293,7 +294,7 @@ impl VeilidComponentRegistryAccessor for VeilidComponentRegistry {
 ////////////////////////////////////////////////////////////////////
 
 macro_rules! impl_veilid_component_registry_accessor {
-    ($struct_name:ident) => {
+    ($struct_name:ty) => {
         impl VeilidComponentRegistryAccessor for $struct_name {
             fn registry(&self) -> VeilidComponentRegistry {
                 self.registry.clone()
@@ -307,7 +308,7 @@ pub(crate) use impl_veilid_component_registry_accessor;
 /////////////////////////////////////////////////////////////////////
 
 macro_rules! impl_veilid_component {
-    ($component_name:ident) => {
+    ($component_name:ty) => {
         impl_veilid_component_registry_accessor!($component_name);
 
         impl VeilidComponent for $component_name {

veilid-core/src/network_manager/connection_manager.rs

@@ -691,10 +691,7 @@ impl ConnectionManager {
 
     fn spawn_reconnector(&self, dial_info: DialInfo) {
         let this = self.clone();
-        self.arc.reconnection_processor.add(
+        self.arc.reconnection_processor.add_future(
-            Box::pin(futures_util::stream::once(async { dial_info })),
-            move |dial_info| {
-                let this = this.clone();
                 Box::pin(async move {
                     match this.get_or_create_connection(dial_info.clone()).await {
                         Ok(NetworkResult::Value(conn)) => {
@@ -706,15 +703,11 @@ impl ConnectionManager {
                         Err(e) => {
                             veilid_log!(this debug "Reconnection error to {}: {}", dial_info, e);
                         }
-                    }
+                    };
-                    false
+                }));
-                })
-            },
-        );
     }
 
     pub fn debug_print(&self) -> String {
-        //let inner = self.arc.inner.lock();
         format!(
             "Connection Table:\n\n{}",
             self.arc.connection_table.debug_print_table()

veilid-core/src/network_manager/network_connection.rs

@@ -352,7 +352,7 @@ impl NetworkConnection {
             };
             let timer = MutableFuture::new(new_timer());
 
-            unord.push(system_boxed(timer.clone().in_current_span()));
+            unord.push(pin_dyn_future!(timer.clone().in_current_span()));
 
             loop {
                 // Add another message sender future if necessary
@@ -386,7 +386,7 @@ impl NetworkConnection {
                             }
                         }
                     }.in_current_span());
-                    unord.push(system_boxed(sender_fut.in_current_span()));
+                    unord.push(pin_dyn_future!(sender_fut.in_current_span()));
                 }
 
                 // Add another message receiver future if necessary
@@ -445,7 +445,7 @@ impl NetworkConnection {
                             }
                         }.in_current_span());
 
-                    unord.push(system_boxed(receiver_fut.in_current_span()));
+                    unord.push(pin_dyn_future!(receiver_fut.in_current_span()));
                 }
 
                 // Process futures

veilid-core/src/routing_table/node_ref/node_ref_lock.rs

@@ -39,6 +39,7 @@ impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Disp
         }
     }
 
+    #[expect(dead_code)]
     pub fn unlocked(&self) -> N {
         self.nr.clone()
     }

veilid-core/src/routing_table/tasks/ping_validator.rs

@@ -236,7 +236,7 @@ impl RoutingTable {
         }
 
         // Get all the active watches from the storage manager
-        let watch_destinations = self.storage_manager().get_active_watch_nodes().await;
+        let watch_destinations = self.storage_manager().get_outbound_watch_nodes().await;
 
         for watch_destination in watch_destinations {
             let registry = self.registry();

veilid-core/src/rpc_processor/destination.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 /// Where to send an RPC message
 #[derive(Debug, Clone)]
 pub(crate) enum Destination {

veilid-core/src/rpc_processor/fanout/fanout_call.rs

@@ -1,40 +1,63 @@
 use super::*;
 
-struct FanoutContext<R>
+impl_veilid_log_facility!("fanout");
-where
+
-    R: Unpin,
+#[derive(Debug)]
-{
+struct FanoutContext<'a> {
-    fanout_queue: FanoutQueue,
+    fanout_queue: FanoutQueue<'a>,
-    result: Option<Result<R, RPCError>>,
+    result: FanoutResult,
+    done: bool,
 }
 
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, Default)]
 pub enum FanoutResultKind {
-    Partial,
+    #[default]
+    Incomplete,
     Timeout,
-    Finished,
+    Consensus,
     Exhausted,
 }
 impl FanoutResultKind {
-    pub fn is_partial(&self) -> bool {
+    pub fn is_incomplete(&self) -> bool {
-        matches!(self, Self::Partial)
+        matches!(self, Self::Incomplete)
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, Default)]
 pub struct FanoutResult {
+    /// How the fanout completed
     pub kind: FanoutResultKind,
+    /// The set of nodes that counted toward consensus
+    /// (for example, had the most recent value for this subkey)
+    pub consensus_nodes: Vec<NodeRef>,
+    /// Which nodes accepted the request
     pub value_nodes: Vec<NodeRef>,
 }
 
-pub fn debug_fanout_result(result: &FanoutResult) -> String {
+impl fmt::Display for FanoutResult {
-    let kc = match result.kind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        FanoutResultKind::Partial => "P",
+        let kc = match self.kind {
+            FanoutResultKind::Incomplete => "I",
             FanoutResultKind::Timeout => "T",
-        FanoutResultKind::Finished => "F",
+            FanoutResultKind::Consensus => "C",
             FanoutResultKind::Exhausted => "E",
         };
-    format!("{}:{}", kc, result.value_nodes.len())
+        if f.alternate() {
+            write!(
+                f,
+                "{}:{}[{}]",
+                kc,
+                self.consensus_nodes.len(),
+                self.consensus_nodes
+                    .iter()
+                    .map(|x| x.to_string())
+                    .collect::<Vec<_>>()
+                    .join(","),
+            )
+        } else {
+            write!(f, "{}:{}", kc, self.consensus_nodes.len())
+        }
+    }
 }
 
 pub fn debug_fanout_results(results: &[FanoutResult]) -> String {
@@ -45,7 +68,7 @@ pub fn debug_fanout_results(results: &[FanoutResult]) -> String {
         if col == 0 {
             out += "    ";
         }
-        let sr = debug_fanout_result(r);
+        let sr = format!("{}", r);
         out += &sr;
         out += ",";
         col += 1;
@@ -61,11 +84,36 @@ pub fn debug_fanout_results(results: &[FanoutResult]) -> String {
 #[derive(Debug)]
 pub struct FanoutCallOutput {
     pub peer_info_list: Vec<Arc<PeerInfo>>,
+    pub disposition: FanoutCallDisposition,
 }
 
-pub type FanoutCallResult = RPCNetworkResult<FanoutCallOutput>;
+/// The return type of the fanout call routine
+#[derive(Debug)]
+pub enum FanoutCallDisposition {
+    /// The call routine timed out
+    Timeout,
+    /// The call routine returned an invalid result
+    Invalid,
+    /// The called node rejected the rpc request but may have returned more nodes
+    Rejected,
+    /// The called node accepted the rpc request and may have returned more nodes,
+    /// but we don't count the result toward our consensus
+    Stale,
+    /// The called node accepted the rpc request and may have returned more nodes,
+    /// counting the result toward our consensus
+    Accepted,
+    /// The called node accepted the rpc request and may have returned more nodes,
+    /// returning a newer value that indicates we should restart our consensus
+    AcceptedNewerRestart,
+    /// The called node accepted the rpc request and may have returned more nodes,
+    /// returning a newer value that indicates our current consensus is stale and should be ignored,
+    /// and counting the result toward a new consensus
+    AcceptedNewer,
+}
+
+pub type FanoutCallResult = Result<FanoutCallOutput, RPCError>;
 pub type FanoutNodeInfoFilter = Arc<dyn (Fn(&[TypedKey], &NodeInfo) -> bool) + Send + Sync>;
-pub type FanoutCheckDone<R> = Arc<dyn (Fn(&[NodeRef]) -> Option<R>) + Send + Sync>;
+pub type FanoutCheckDone = Arc<dyn (Fn(&FanoutResult) -> bool) + Send + Sync>;
 pub type FanoutCallRoutine =
     Arc<dyn (Fn(NodeRef) -> PinBoxFutureStatic<FanoutCallResult>) + Send + Sync>;
 
@@ -89,52 +137,50 @@ pub fn capability_fanout_node_info_filter(caps: Vec<Capability>) -> FanoutNodeIn
 /// The algorithm is parameterized by:
 ///  * 'node_count' - the number of nodes to keep in the closest_nodes set
 ///  * 'fanout' - the number of concurrent calls being processed at the same time
+///  * 'consensus_count' - the number of nodes in the processed queue that need to be in the
+///                        'Accepted' state before we terminate the fanout early.
 ///
 /// The algorithm returns early if 'check_done' returns some value, or if an error is found during the process.
 /// If the algorithm times out, a Timeout result is returned, however operations will still have been performed and a
 /// timeout is not necessarily indicative of an algorithmic 'failure', just that no definitive stopping condition was found
 /// in the given time
-pub(crate) struct FanoutCall<'a, R>
+pub(crate) struct FanoutCall<'a> {
-where
-    R: Unpin,
-{
     routing_table: &'a RoutingTable,
     node_id: TypedKey,
-    context: Mutex<FanoutContext<R>>,
     node_count: usize,
-    fanout: usize,
+    fanout_tasks: usize,
+    consensus_count: usize,
     timeout_us: TimestampDuration,
     node_info_filter: FanoutNodeInfoFilter,
     call_routine: FanoutCallRoutine,
-    check_done: FanoutCheckDone<R>,
+    check_done: FanoutCheckDone,
 }
 
-impl<'a, R> FanoutCall<'a, R>
+impl VeilidComponentRegistryAccessor for FanoutCall<'_> {
-where
+    fn registry(&self) -> VeilidComponentRegistry {
-    R: Unpin,
+        self.routing_table.registry()
-{
+    }
+}
+
+impl<'a> FanoutCall<'a> {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         routing_table: &'a RoutingTable,
         node_id: TypedKey,
         node_count: usize,
-        fanout: usize,
+        fanout_tasks: usize,
+        consensus_count: usize,
         timeout_us: TimestampDuration,
         node_info_filter: FanoutNodeInfoFilter,
         call_routine: FanoutCallRoutine,
-        check_done: FanoutCheckDone<R>,
+        check_done: FanoutCheckDone,
     ) -> Self {
-        let context = Mutex::new(FanoutContext {
-            fanout_queue: FanoutQueue::new(node_id.kind),
-            result: None,
-        });
-
         Self {
             routing_table,
             node_id,
-            context,
             node_count,
-            fanout,
+            fanout_tasks,
+            consensus_count,
             timeout_us,
             node_info_filter,
             call_routine,
@@ -143,61 +189,104 @@ where
     }
 
     #[instrument(level = "trace", target = "fanout", skip_all)]
-    fn evaluate_done(&self, ctx: &mut FanoutContext<R>) -> bool {
+    fn evaluate_done(&self, ctx: &mut FanoutContext) -> bool {
-        // If we have a result, then we're done
+        // If we already finished, just return
-        if ctx.result.is_some() {
+        if ctx.done {
             return true;
         }
 
-        // Check for a new done result
+        // Calculate fanout result so far
-        ctx.result = (self.check_done)(ctx.fanout_queue.nodes()).map(|o| Ok(o));
+        let fanout_result = ctx.fanout_queue.with_nodes(|nodes, sorted_nodes| {
-        ctx.result.is_some()
+            // Count up nodes we have seen in order and see if our closest nodes have a consensus
+            let mut consensus: Option<bool> = None;
+            let mut consensus_nodes: Vec<NodeRef> = vec![];
+            let mut value_nodes: Vec<NodeRef> = vec![];
+            for sn in sorted_nodes {
+                let node = nodes.get(sn).unwrap();
+                match node.status {
+                    FanoutNodeStatus::Queued | FanoutNodeStatus::InProgress => {
+                        // Still have a closer node to do before reaching consensus,
+                        // or are doing it still, then wait until those are done
+                        if consensus.is_none() {
+                            consensus = Some(false);
+                        }
+                    }
+                    FanoutNodeStatus::Timeout
+                    | FanoutNodeStatus::Rejected
+                    | FanoutNodeStatus::Disqualified => {
+                        // Node does not count toward consensus or value node list
+                    }
+                    FanoutNodeStatus::Stale => {
+                        // Node does not count toward consensus but does count toward value node list
+                        value_nodes.push(node.node_ref.clone());
+                    }
+                    FanoutNodeStatus::Accepted => {
+                        // Node counts toward consensus and value node list
+                        value_nodes.push(node.node_ref.clone());
+
+                        consensus_nodes.push(node.node_ref.clone());
+                        if consensus.is_none() && consensus_nodes.len() >= self.consensus_count {
+                            consensus = Some(true);
+                        }
+                    }
+                }
             }
 
-    #[instrument(level = "trace", target = "fanout", skip_all)]
+            // If we have reached sufficient consensus, return done
-    fn add_to_fanout_queue(&self, new_nodes: &[NodeRef]) {
+            match consensus {
-        event!(target: "fanout", Level::DEBUG,
+                Some(true) => FanoutResult {
-            "FanoutCall::add_to_fanout_queue:\n  new_nodes={{\n{}}}\n",
+                    kind: FanoutResultKind::Consensus,
-            new_nodes
+                    consensus_nodes,
-                .iter()
+                    value_nodes,
-                .map(|x| format!("  {}", x))
+                },
-                .collect::<Vec<String>>()
+                Some(false) => FanoutResult {
-                .join(",\n"),
+                    kind: FanoutResultKind::Incomplete,
-        );
+                    consensus_nodes,
-
+                    value_nodes,
-        let ctx = &mut *self.context.lock();
+                },
-        ctx.fanout_queue.add(new_nodes, |current_nodes| {
+                None => FanoutResult {
-            let mut current_nodes_vec = self
+                    kind: FanoutResultKind::Exhausted,
-                .routing_table
+                    consensus_nodes,
-                .sort_and_clean_closest_noderefs(self.node_id, current_nodes);
+                    value_nodes,
-            current_nodes_vec.truncate(self.node_count);
+                },
-            current_nodes_vec
+            }
         });
+
+        let done = (self.check_done)(&fanout_result);
+        ctx.result = fanout_result;
+        ctx.done = done;
+        done
     }
 
     #[instrument(level = "trace", target = "fanout", skip_all)]
-    async fn fanout_processor(&self) -> bool {
+    async fn fanout_processor<'b>(
+        &self,
+        context: &Mutex<FanoutContext<'b>>,
+    ) -> Result<bool, RPCError> {
+        // Make a work request channel
+        let (work_sender, work_receiver) = flume::bounded(1);
+
         // Loop until we have a result or are done
         loop {
-            // Get the closest node we haven't processed yet if we're not done yet
+            // Put in a work request
-            let next_node = {
+            {
-                let mut ctx = self.context.lock();
+                let mut context_locked = context.lock();
-                if self.evaluate_done(&mut ctx) {
+                context_locked
-                    break true;
+                    .fanout_queue
+                    .request_work(work_sender.clone());
             }
-                ctx.fanout_queue.next()
-            };
 
+            // Wait around for some work to do
+            let Ok(next_node) = work_receiver.recv_async().await else {
                 // If we don't have a node to process, stop fanning out
-            let Some(next_node) = next_node else {
+                break Ok(false);
-                break false;
             };
 
             // Do the call for this node
             match (self.call_routine)(next_node.clone()).await {
-                Ok(NetworkResult::Value(v)) => {
+                Ok(output) => {
                     // Filter returned nodes
-                    let filtered_v: Vec<Arc<PeerInfo>> = v
+                    let filtered_v: Vec<Arc<PeerInfo>> = output
                         .peer_info_list
                         .into_iter()
                         .filter(|pi| {
@@ -217,25 +306,58 @@ where
                     let new_nodes = self
                         .routing_table
                         .register_nodes_with_peer_info_list(filtered_v);
-                    self.add_to_fanout_queue(&new_nodes);
+
+                    // Update queue
+                    {
+                        let mut context_locked = context.lock();
+                        context_locked.fanout_queue.add(&new_nodes);
+
+                        // Process disposition of the output of the fanout call routine
+                        match output.disposition {
+                            FanoutCallDisposition::Timeout => {
+                                context_locked.fanout_queue.timeout(next_node);
+                            }
+                            FanoutCallDisposition::Rejected => {
+                                context_locked.fanout_queue.rejected(next_node);
+                            }
+                            FanoutCallDisposition::Accepted => {
+                                context_locked.fanout_queue.accepted(next_node);
+                            }
+                            FanoutCallDisposition::AcceptedNewerRestart => {
+                                context_locked.fanout_queue.all_accepted_to_queued();
+                                context_locked.fanout_queue.accepted(next_node);
+                            }
+                            FanoutCallDisposition::AcceptedNewer => {
+                                context_locked.fanout_queue.all_accepted_to_stale();
+                                context_locked.fanout_queue.accepted(next_node);
+                            }
+                            FanoutCallDisposition::Invalid => {
+                                // Do nothing with invalid fanout calls
+                            }
+                            FanoutCallDisposition::Stale => {
+                                context_locked.fanout_queue.stale(next_node);
+                            }
+                        }
+
+                        // See if we're done before going back for more processing
+                        if self.evaluate_done(&mut context_locked) {
+                            break Ok(true);
+                        }
+
+                        // We modified the queue so we may have more work to do now,
+                        // tell the queue it should send more work to the workers
+                        context_locked.fanout_queue.send_more_work();
                     }
-                #[allow(unused_variables)]
-                Ok(x) => {
-                    // Call failed, node will not be considered again
-                    event!(target: "fanout", Level::DEBUG,
-                        "Fanout result {}: {:?}", &next_node, x);
                 }
                 Err(e) => {
-                    // Error happened, abort everything and return the error
+                    break Err(e);
-                    self.context.lock().result = Some(Err(e));
-                    break true;
                 }
             };
         }
     }
 
     #[instrument(level = "trace", target = "fanout", skip_all)]
-    fn init_closest_nodes(&self) -> Result<(), RPCError> {
+    fn init_closest_nodes(&self, context: &mut FanoutContext) -> Result<(), RPCError> {
         // Get the 'node_count' closest nodes to the key out of our routing table
         let closest_nodes = {
             let node_info_filter = self.node_info_filter.clone();
@@ -279,36 +401,58 @@ where
                 .find_preferred_closest_nodes(self.node_count, self.node_id, filters, transform)
                 .map_err(RPCError::invalid_format)?
         };
-        self.add_to_fanout_queue(&closest_nodes);
+        context.fanout_queue.add(&closest_nodes);
+
         Ok(())
     }
 
     #[instrument(level = "trace", target = "fanout", skip_all)]
-    pub async fn run(
+    pub async fn run(&self, init_fanout_queue: Vec<NodeRef>) -> Result<FanoutResult, RPCError> {
-        &self,
+        // Create context for this run
-        init_fanout_queue: Vec<NodeRef>,
+        let crypto = self.routing_table.crypto();
-    ) -> TimeoutOr<Result<Option<R>, RPCError>> {
+        let Some(vcrypto) = crypto.get(self.node_id.kind) else {
-        // Get timeout in milliseconds
+            return Err(RPCError::internal(
-        let timeout_ms = match us_to_ms(self.timeout_us.as_u64()).map_err(RPCError::internal) {
+                "should not try this on crypto we don't support",
-            Ok(v) => v,
+            ));
-            Err(e) => {
-                return TimeoutOr::value(Err(e));
-            }
         };
+        let node_sort = Box::new(
+            |a_key: &CryptoTyped<CryptoKey>,
+             b_key: &CryptoTyped<CryptoKey>|
+             -> core::cmp::Ordering {
+                let da = vcrypto.distance(&a_key.value, &self.node_id.value);
+                let db = vcrypto.distance(&b_key.value, &self.node_id.value);
+                da.cmp(&db)
+            },
+        );
+        let context = Arc::new(Mutex::new(FanoutContext {
+            fanout_queue: FanoutQueue::new(
+                self.routing_table.registry(),
+                self.node_id.kind,
+                node_sort,
+                self.consensus_count,
+            ),
+            result: FanoutResult {
+                kind: FanoutResultKind::Incomplete,
+                consensus_nodes: vec![],
+                value_nodes: vec![],
+            },
+            done: false,
+        }));
+
+        // Get timeout in milliseconds
+        let timeout_ms = us_to_ms(self.timeout_us.as_u64()).map_err(RPCError::internal)?;
 
         // Initialize closest nodes list
-        if let Err(e) = self.init_closest_nodes() {
+        {
-            return TimeoutOr::value(Err(e));
+            let context_locked = &mut *context.lock();
-        }
+            self.init_closest_nodes(context_locked)?;
 
             // Ensure we include the most recent nodes
-        self.add_to_fanout_queue(&init_fanout_queue);
+            context_locked.fanout_queue.add(&init_fanout_queue);
 
             // Do a quick check to see if we're already done
-        {
+            if self.evaluate_done(context_locked) {
-            let mut ctx = self.context.lock();
+                return Ok(core::mem::take(&mut context_locked.result));
-            if self.evaluate_done(&mut ctx) {
-                return TimeoutOr::value(ctx.result.take().transpose());
             }
         }
 
@@ -316,28 +460,67 @@ where
         let mut unord = FuturesUnordered::new();
         {
             // Spin up 'fanout' tasks to process the fanout
-            for _ in 0..self.fanout {
+            for _ in 0..self.fanout_tasks {
-                let h = self.fanout_processor();
+                let h = self.fanout_processor(&context);
                 unord.push(h);
             }
         }
         // Wait for them to complete
-        timeout(
+        match timeout(
             timeout_ms,
             async {
-                while let Some(is_done) = unord.next().in_current_span().await {
+                loop {
+                    if let Some(res) = unord.next().in_current_span().await {
+                        match res {
+                            Ok(is_done) => {
                                 if is_done {
-                        break;
+                                    break Ok(());
+                                }
+                            }
+                            Err(e) => {
+                                break Err(e);
+                            }
+                        }
+                    } else {
+                        break Ok(());
                     }
                 }
             }
             .in_current_span(),
         )
         .await
-        .into_timeout_or()
+        {
-        .map(|_| {
+            Ok(Ok(())) => {
-            // Finished, return whatever value we came up with
+                // Finished, either by exhaustion or consensus,
-            self.context.lock().result.take().transpose()
+                // time to return whatever value we came up with
-        })
+                let context_locked = &mut *context.lock();
+                // Print final queue
+                veilid_log!(self debug "Finished FanoutQueue: {}", context_locked.fanout_queue);
+                return Ok(core::mem::take(&mut context_locked.result));
+            }
+            Ok(Err(e)) => {
+                // Fanout died with an error
+                return Err(e);
+            }
+            Err(_) => {
+                // Timeout, do one last evaluate with remaining nodes in timeout state
+                let context_locked = &mut *context.lock();
+                context_locked.fanout_queue.all_unfinished_to_timeout();
+
+                // Print final queue
+                veilid_log!(self debug "Timeout FanoutQueue: {}", context_locked.fanout_queue);
+
+                // Final evaluate
+                if self.evaluate_done(context_locked) {
+                    // Last-chance value returned at timeout
+                    return Ok(core::mem::take(&mut context_locked.result));
+                }
+
+                // We definitely weren't done, so this is just a plain timeout
+                let mut result = core::mem::take(&mut context_locked.result);
+                result.kind = FanoutResultKind::Timeout;
+                return Ok(result);
+            }
+        }
     }
 }

veilid-core/src/rpc_processor/fanout/fanout_queue.rs

@@ -4,93 +4,270 @@
 /// When passing in a 'cleanup' function, if it sorts the queue, the 'first' items in the queue are the 'next' out.
 use super::*;
 
-#[derive(Debug)]
+impl_veilid_log_facility!("fanout");
-pub struct FanoutQueue {
+impl_veilid_component_registry_accessor!(FanoutQueue<'_>);
-    crypto_kind: CryptoKind,
+
-    current_nodes: VecDeque<NodeRef>,
+/// The status of a particular node we fanned out to
-    returned_nodes: HashSet<TypedKey>,
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum FanoutNodeStatus {
+    /// Node that needs processing
+    Queued,
+    /// Node currently being processed
+    InProgress,
+    /// Node that timed out during processing
+    Timeout,
+    /// Node that rejected the query
+    Rejected,
+    /// Node that accepted the query with a current result
+    Accepted,
+    /// Node that accepted the query but had an older result
+    Stale,
+    /// Node that has been disqualified for being too far away from the key
+    Disqualified,
 }
 
-impl FanoutQueue {
+#[derive(Debug, Clone)]
-    // Create a queue for fanout candidates that have a crypto-kind compatible node id
+pub struct FanoutNode {
-    pub fn new(crypto_kind: CryptoKind) -> Self {
+    pub node_ref: NodeRef,
+    pub status: FanoutNodeStatus,
+}
+
+pub type FanoutQueueSort<'a> = Box<dyn Fn(&TypedKey, &TypedKey) -> core::cmp::Ordering + Send + 'a>;
+
+pub struct FanoutQueue<'a> {
+    /// Link back to veilid component registry for logging
+    registry: VeilidComponentRegistry,
+    /// Crypto kind in use for this queue
+    crypto_kind: CryptoKind,
+    /// The status of all the nodes we have added so far
+    nodes: HashMap<TypedKey, FanoutNode>,
+    /// Closer nodes to the record key are at the front of the list
+    sorted_nodes: Vec<TypedKey>,
+    /// The sort function to use for the nodes
+    node_sort: FanoutQueueSort<'a>,
+    /// The channel to receive work requests to process
+    sender: flume::Sender<flume::Sender<NodeRef>>,
+    receiver: flume::Receiver<flume::Sender<NodeRef>>,
+    /// Consensus count to use
+    consensus_count: usize,
+}
+
+impl fmt::Debug for FanoutQueue<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("FanoutQueue")
+            .field("crypto_kind", &self.crypto_kind)
+            .field("nodes", &self.nodes)
+            .field("sorted_nodes", &self.sorted_nodes)
+            // .field("node_sort", &self.node_sort)
+            .field("sender", &self.sender)
+            .field("receiver", &self.receiver)
+            .finish()
+    }
+}
+
+impl fmt::Display for FanoutQueue<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "nodes:\n{}",
+            self.sorted_nodes
+                .iter()
+                .map(|x| format!("{}: {:?}", x, self.nodes.get(x).unwrap().status))
+                .collect::<Vec<_>>()
+                .join("\n")
+        )
+    }
+}
+
+impl<'a> FanoutQueue<'a> {
+    /// Create a queue for fanout candidates that have a crypto-kind compatible node id
+    pub fn new(
+        registry: VeilidComponentRegistry,
+        crypto_kind: CryptoKind,
+        node_sort: FanoutQueueSort<'a>,
+        consensus_count: usize,
+    ) -> Self {
+        let (sender, receiver) = flume::unbounded();
         Self {
+            registry,
             crypto_kind,
-            current_nodes: VecDeque::new(),
+            nodes: HashMap::new(),
-            returned_nodes: HashSet::new(),
+            sorted_nodes: Vec::new(),
+            node_sort,
+            sender,
+            receiver,
+            consensus_count,
         }
     }
 
-    // Add new nodes to list of fanout candidates
+    /// Ask for more work when some is ready
-    // Run a cleanup routine afterwards to trim down the list of candidates so it doesn't grow too large
+    /// When work is ready it will be sent to work_sender so it can be received
-    pub fn add<F: FnOnce(&[NodeRef]) -> Vec<NodeRef>>(
+    /// by the worker
-        &mut self,
+    pub fn request_work(&mut self, work_sender: flume::Sender<NodeRef>) {
-        new_nodes: &[NodeRef],
+        let _ = self.sender.send(work_sender);
-        cleanup: F,
+
-    ) {
+        // Send whatever work is available immediately
-        for nn in new_nodes {
+        self.send_more_work();
+    }
+
+    /// Add new nodes to a filtered and sorted list of fanout candidates
+    pub fn add(&mut self, new_nodes: &[NodeRef]) {
+        for node_ref in new_nodes {
             // Ensure the node has a comparable key with our current crypto kind
-            let Some(key) = nn.node_ids().get(self.crypto_kind) else {
+            let Some(key) = node_ref.node_ids().get(self.crypto_kind) else {
                 continue;
             };
-            // Check if we have already done this node before (only one call per node ever)
+            // Check if we have already seen this node before (only one call per node ever)
-            if self.returned_nodes.contains(&key) {
+            if self.nodes.contains_key(&key) {
                 continue;
             }
-
-            // Make sure the new node isnt already in the list
-            let mut dup = false;
-            for cn in &self.current_nodes {
-                if cn.same_entry(nn) {
-                    dup = true;
-                    break;
-                }
-            }
-            if !dup {
             // Add the new node
-                self.current_nodes.push_back(nn.clone());
+            self.nodes.insert(
-            }
+                key,
+                FanoutNode {
+                    node_ref: node_ref.clone(),
+                    status: FanoutNodeStatus::Queued,
+                },
+            );
+            self.sorted_nodes.push(key);
         }
 
-        // Make sure the deque is a single slice
+        // Sort the node list
-        self.current_nodes.make_contiguous();
+        self.sorted_nodes.sort_by(&self.node_sort);
 
-        // Sort and trim the candidate set
+        // Disqualify any nodes that can be
-        self.current_nodes =
+        self.disqualify();
-            VecDeque::from_iter(cleanup(self.current_nodes.as_slices().0).iter().cloned());
 
-        event!(target: "fanout", Level::DEBUG,
+        veilid_log!(self debug
-            "FanoutQueue::add:\n  current_nodes={{\n{}}}\n  returned_nodes={{\n{}}}\n",
+            "FanoutQueue::add:\n  new_nodes={{\n{}}}\n  nodes={{\n{}}}\n",
-            self.current_nodes
+            new_nodes.iter().map(|x| format!("  {}", x))
-                .iter()
-                .map(|x| format!("  {}", x))
                 .collect::<Vec<String>>()
                 .join(",\n"),
-            self.returned_nodes
+            self.sorted_nodes
                 .iter()
-                .map(|x| format!("  {}", x))
+                .map(|x| format!("  {:?}", self.nodes.get(x).unwrap()))
                 .collect::<Vec<String>>()
                 .join(",\n")
         );
     }
 
-    // Return next fanout candidate
+    /// Send next fanout candidates if available to whatever workers are ready
-    pub fn next(&mut self) -> Option<NodeRef> {
+    pub fn send_more_work(&mut self) {
-        let cn = self.current_nodes.pop_front()?;
+        // Get the next work and send it along
-        self.current_nodes.make_contiguous();
+        let registry = self.registry();
-        let key = cn.node_ids().get(self.crypto_kind).unwrap();
+        for x in &mut self.sorted_nodes {
-
+            // If there are no work receivers left then we should stop trying to send
-        // Ensure we don't return this node again
+            if self.receiver.is_empty() {
-        self.returned_nodes.insert(key);
+                break;
-
-        event!(target: "fanout", Level::DEBUG,
-            "FanoutQueue::next: => {}", cn);
-
-        Some(cn)
             }
 
-    // Get a slice of all the current fanout candidates
+            let node = self.nodes.get_mut(x).unwrap();
-    pub fn nodes(&self) -> &[NodeRef] {
+            if matches!(node.status, FanoutNodeStatus::Queued) {
-        self.current_nodes.as_slices().0
+                // Send node to a work request
+                while let Ok(work_sender) = self.receiver.try_recv() {
+                    let node_ref = node.node_ref.clone();
+                    if work_sender.send(node_ref).is_ok() {
+                        // Queued -> InProgress
+                        node.status = FanoutNodeStatus::InProgress;
+                        veilid_log!(registry debug "FanoutQueue::next: => {}", node.node_ref);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    /// Transition node InProgress -> Timeout
+    pub fn timeout(&mut self, node_ref: NodeRef) {
+        let key = node_ref.node_ids().get(self.crypto_kind).unwrap();
+        let node = self.nodes.get_mut(&key).unwrap();
+        assert_eq!(node.status, FanoutNodeStatus::InProgress);
+        node.status = FanoutNodeStatus::Timeout;
+    }
+
+    /// Transition node InProgress -> Rejected
+    pub fn rejected(&mut self, node_ref: NodeRef) {
+        let key = node_ref.node_ids().get(self.crypto_kind).unwrap();
+        let node = self.nodes.get_mut(&key).unwrap();
+        assert_eq!(node.status, FanoutNodeStatus::InProgress);
+        node.status = FanoutNodeStatus::Rejected;
+
+        self.disqualify();
+    }
+
+    /// Transition node InProgress -> Accepted
+    pub fn accepted(&mut self, node_ref: NodeRef) {
+        let key = node_ref.node_ids().get(self.crypto_kind).unwrap();
+        let node = self.nodes.get_mut(&key).unwrap();
+        assert_eq!(node.status, FanoutNodeStatus::InProgress);
+        node.status = FanoutNodeStatus::Accepted;
+    }
+
+    /// Transition node InProgress -> Stale
+    pub fn stale(&mut self, node_ref: NodeRef) {
+        let key = node_ref.node_ids().get(self.crypto_kind).unwrap();
+        let node = self.nodes.get_mut(&key).unwrap();
+        assert_eq!(node.status, FanoutNodeStatus::InProgress);
+        node.status = FanoutNodeStatus::Stale;
+    }
+
+    /// Transition all Accepted -> Queued, in the event a newer value for consensus is found and we want to try again
+    pub fn all_accepted_to_queued(&mut self) {
+        for node in &mut self.nodes {
+            if matches!(node.1.status, FanoutNodeStatus::Accepted) {
+                node.1.status = FanoutNodeStatus::Queued;
+            }
+        }
+    }
+
+    /// Transition all Accepted -> Stale, in the event a newer value for consensus is found but we don't want to try again
+    pub fn all_accepted_to_stale(&mut self) {
+        for node in &mut self.nodes {
+            if matches!(node.1.status, FanoutNodeStatus::Accepted) {
+                node.1.status = FanoutNodeStatus::Stale;
+            }
+        }
+    }
+
+    /// Transition all Queued | InProgress -> Timeout, in the event that the fanout is being cut short by a timeout
+    pub fn all_unfinished_to_timeout(&mut self) {
+        for node in &mut self.nodes {
+            if matches!(
+                node.1.status,
+                FanoutNodeStatus::Queued | FanoutNodeStatus::InProgress
+            ) {
+                node.1.status = FanoutNodeStatus::Timeout;
+            }
+        }
+    }
+
+    /// Transition Queued -> Disqualified that are too far away from the record key
+    fn disqualify(&mut self) {
+        let mut consecutive_rejections = 0usize;
+        let mut rejected_consensus = false;
+        for node_id in &self.sorted_nodes {
+            let node = self.nodes.get_mut(node_id).unwrap();
+            if !rejected_consensus {
+                if matches!(node.status, FanoutNodeStatus::Rejected) {
+                    consecutive_rejections += 1;
+                    if consecutive_rejections >= self.consensus_count {
+                        rejected_consensus = true;
+                    }
+                    continue;
+                } else {
+                    consecutive_rejections = 0;
+                }
+            } else if matches!(node.status, FanoutNodeStatus::Queued) {
+                node.status = FanoutNodeStatus::Disqualified;
+            }
+        }
+    }
+
+    /// Review the nodes in the queue
+    pub fn with_nodes<R, F: FnOnce(&HashMap<TypedKey, FanoutNode>, &[TypedKey]) -> R>(
+        &self,
+        func: F,
+    ) -> R {
+        func(&self.nodes, &self.sorted_nodes)
     }
 }

veilid-core/src/rpc_processor/fanout/mod.rs

@@ -2,7 +2,6 @@ mod fanout_call;
 mod fanout_queue;
 
 pub(crate) use fanout_call::*;
+pub(crate) use fanout_queue::*;
 
 use super::*;
-
-use fanout_queue::*;

veilid-core/src/rpc_processor/mod.rs

@@ -379,40 +379,57 @@ impl RPCProcessor {
         }
 
         // Routine to call to generate fanout
+        let result = Arc::new(Mutex::new(Option::<NodeRef>::None));
+
         let registry = self.registry();
         let call_routine = Arc::new(move |next_node: NodeRef| {
             let registry = registry.clone();
             Box::pin(async move {
                 let this = registry.rpc_processor();
-                let v = network_result_try!(
+                match this
-                    this.rpc_call_find_node(
+                    .rpc_call_find_node(
                         Destination::direct(next_node.routing_domain_filtered(routing_domain))
                             .with_safety(safety_selection),
                         node_id,
                         vec![],
                     )
                     .await?
-                );
+                {
-                Ok(NetworkResult::value(FanoutCallOutput {
+                    NetworkResult::Timeout => Ok(FanoutCallOutput {
+                        peer_info_list: vec![],
+                        disposition: FanoutCallDisposition::Timeout,
+                    }),
+                    NetworkResult::ServiceUnavailable(_)
+                    | NetworkResult::NoConnection(_)
+                    | NetworkResult::AlreadyExists(_)
+                    | NetworkResult::InvalidMessage(_) => Ok(FanoutCallOutput {
+                        peer_info_list: vec![],
+                        disposition: FanoutCallDisposition::Rejected,
+                    }),
+                    NetworkResult::Value(v) => Ok(FanoutCallOutput {
                         peer_info_list: v.answer,
-                }))
+                        disposition: FanoutCallDisposition::Accepted,
+                    }),
+                }
             }) as PinBoxFuture<FanoutCallResult>
         });
 
         // Routine to call to check if we're done at each step
-        let check_done = Arc::new(move |_: &[NodeRef]| {
+        let result2 = result.clone();
+        let check_done = Arc::new(move |_: &FanoutResult| -> bool {
             let Ok(Some(nr)) = routing_table.lookup_node_ref(node_id) else {
-                return None;
+                return false;
             };
 
             // ensure we have some dial info for the entry already,
             // and that the node is still alive
             // if not, we should keep looking for better info
             if nr.state(Timestamp::now()).is_alive() && nr.has_any_dial_info() {
-                return Some(nr);
+                *result2.lock() = Some(nr);
+                return true;
             }
 
-            None
+            false
         });
 
         // Call the fanout
@@ -422,13 +439,23 @@ impl RPCProcessor {
             node_id,
             count,
             fanout,
+            0,
             timeout_us,
             empty_fanout_node_info_filter(),
             call_routine,
             check_done,
         );
 
-        fanout_call.run(vec![]).await
+        match fanout_call.run(vec![]).await {
+            Ok(fanout_result) => {
+                if matches!(fanout_result.kind, FanoutResultKind::Timeout) {
+                    TimeoutOr::timeout()
+                } else {
+                    TimeoutOr::value(Ok(result.lock().take()))
+                }
+            }
+            Err(e) => TimeoutOr::value(Err(e)),
+        }
     }
 
     /// Search the DHT for a specific node corresponding to a key unless we

veilid-core/src/rpc_processor/rpc_app_call.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     // Sends a high level app request and wait for response
     // Can be sent via all methods including relays and routes

veilid-core/src/rpc_processor/rpc_app_message.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     // Sends a high level app message
     // Can be sent via all methods including relays and routes

veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
     pub(super) async fn process_cancel_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

veilid-core/src/rpc_processor/rpc_complete_tunnel.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
     pub(super) async fn process_complete_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

veilid-core/src/rpc_processor/rpc_find_block.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
     pub(super) async fn process_find_block_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

veilid-core/src/rpc_processor/rpc_find_node.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     /// Send FindNodeQ RPC request, receive FindNodeA answer
     /// Can be sent via all methods including relays

veilid-core/src/rpc_processor/rpc_get_value.rs

@@ -1,6 +1,8 @@
 use super::*;
 use crate::storage_manager::{SignedValueData, SignedValueDescriptor};
 
+impl_veilid_log_facility!("rpc");
+
 #[derive(Clone, Debug)]
 pub struct GetValueAnswer {
     pub value: Option<SignedValueData>,
@@ -78,7 +80,7 @@ impl RPCProcessor {
             crypto_kind: vcrypto.kind(),
         });
 
-        veilid_log!(self debug "{}", debug_string);
+        veilid_log!(self debug target: "dht", "{}", debug_string);
 
         let waitable_reply = network_result_try!(
             self.question(dest.clone(), question, Some(question_context))
@@ -128,13 +130,13 @@ impl RPCProcessor {
                 dest
             );
 
-            veilid_log!(self debug "{}", debug_string_answer);
+            veilid_log!(self debug target: "dht", "{}", debug_string_answer);
 
             let peer_ids: Vec<String> = peers
                 .iter()
                 .filter_map(|p| p.node_ids().get(key.kind).map(|k| k.to_string()))
                 .collect();
-            veilid_log!(self debug "Peers: {:#?}", peer_ids);
+            veilid_log!(self debug target: "dht", "Peers: {:#?}", peer_ids);
         }
 
         // Validate peers returned are, in fact, closer to the key than the node we sent this to
@@ -228,7 +230,7 @@ impl RPCProcessor {
                 msg.header.direct_sender_node_id()
             );
 
-            veilid_log!(self debug "{}", debug_string);
+            veilid_log!(self debug target: "dht", "{}", debug_string);
         }
 
         // See if we would have accepted this as a set
@@ -278,7 +280,7 @@ impl RPCProcessor {
                 msg.header.direct_sender_node_id()
             );
 
-            veilid_log!(self debug "{}", debug_string_answer);
+            veilid_log!(self debug target: "dht", "{}", debug_string_answer);
         }
 
         // Make GetValue answer

veilid-core/src/rpc_processor/rpc_inspect_value.rs

@@ -1,6 +1,8 @@
 use super::*;
 use crate::storage_manager::SignedValueDescriptor;
 
+impl_veilid_log_facility!("rpc");
+
 #[derive(Clone, Debug)]
 pub struct InspectValueAnswer {
     pub seqs: Vec<ValueSeqNum>,
@@ -81,7 +83,7 @@ impl RPCProcessor {
             crypto_kind: vcrypto.kind(),
         });
 
-        veilid_log!(self debug "{}", debug_string);
+        veilid_log!(self debug target: "dht", "{}", debug_string);
 
         let waitable_reply = network_result_try!(
             self.question(dest.clone(), question, Some(question_context))
@@ -118,13 +120,13 @@ impl RPCProcessor {
                 debug_seqs(&seqs)
             );
 
-            veilid_log!(self debug "{}", debug_string_answer);
+            veilid_log!(self debug target: "dht", "{}", debug_string_answer);
 
             let peer_ids: Vec<String> = peers
                 .iter()
                 .filter_map(|p| p.node_ids().get(key.kind).map(|k| k.to_string()))
                 .collect();
-            veilid_log!(self debug "Peers: {:#?}", peer_ids);
+            veilid_log!(self debug target: "dht", "Peers: {:#?}", peer_ids);
         }
 
         // Validate peers returned are, in fact, closer to the key than the node we sent this to
@@ -209,7 +211,7 @@ impl RPCProcessor {
                 msg.header.direct_sender_node_id()
             );
 
-            veilid_log!(self debug "{}", debug_string);
+            veilid_log!(self debug target: "dht", "{}", debug_string);
         }
 
         // See if we would have accepted this as a set
@@ -247,7 +249,7 @@ impl RPCProcessor {
                 msg.header.direct_sender_node_id()
             );
 
-            veilid_log!(self debug "{}", debug_string_answer);
+            veilid_log!(self debug target: "dht", "{}", debug_string_answer);
         }
 
         // Make InspectValue answer

veilid-core/src/rpc_processor/rpc_return_receipt.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     // Sends a unidirectional in-band return receipt
     // Can be sent via all methods including relays and routes

veilid-core/src/rpc_processor/rpc_route.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     #[instrument(level = "trace", target = "rpc", skip_all, err)]
     async fn process_route_safety_route_hop(

veilid-core/src/rpc_processor/rpc_set_value.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 #[derive(Clone, Debug)]
 pub struct SetValueAnswer {
     pub set: bool,
@@ -58,10 +60,11 @@ impl RPCProcessor {
         };
 
         let debug_string = format!(
-            "OUT ==> SetValueQ({} #{} len={} writer={}{}) => {}",
+            "OUT ==> SetValueQ({} #{} len={} seq={} writer={}{}) => {}",
             key,
             subkey,
             value.value_data().data().len(),
+            value.value_data().seq(),
             value.value_data().writer(),
             if send_descriptor { " +senddesc" } else { "" },
             dest
@@ -89,7 +92,7 @@ impl RPCProcessor {
         });
 
         if debug_target_enabled!("dht") {
-            veilid_log!(self debug "{}", debug_string);
+            veilid_log!(self debug target: "dht", "{}", debug_string);
         }
 
         let waitable_reply = network_result_try!(
@@ -123,8 +126,9 @@ impl RPCProcessor {
                 .as_ref()
                 .map(|v| {
                     format!(
-                        " len={} writer={}",
+                        " len={} seq={} writer={}",
                         v.value_data().data().len(),
+                        v.value_data().seq(),
                         v.value_data().writer(),
                     )
                 })
@@ -140,13 +144,13 @@ impl RPCProcessor {
                 dest,
             );
 
-            veilid_log!(self debug "{}", debug_string_answer);
+            veilid_log!(self debug target: "dht", "{}", debug_string_answer);
 
             let peer_ids: Vec<String> = peers
                 .iter()
                 .filter_map(|p| p.node_ids().get(key.kind).map(|k| k.to_string()))
                 .collect();
-            veilid_log!(self debug "Peers: {:#?}", peer_ids);
+            veilid_log!(self debug target: "dht", "Peers: {:#?}", peer_ids);
         }
 
         // Validate peers returned are, in fact, closer to the key than the node we sent this to
@@ -244,7 +248,7 @@ impl RPCProcessor {
             msg.header.direct_sender_node_id()
         );
 
-        veilid_log!(self debug "{}", debug_string);
+        veilid_log!(self debug target: "dht", "{}", debug_string);
 
         // If there are less than 'set_value_count' peers that are closer, then store here too
         let set_value_count = self
@@ -296,7 +300,7 @@ impl RPCProcessor {
                 msg.header.direct_sender_node_id()
             );
 
-            veilid_log!(self debug "{}", debug_string_answer);
+            veilid_log!(self debug target: "dht", "{}", debug_string_answer);
         }
 
         // Make SetValue answer

veilid-core/src/rpc_processor/rpc_signal.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     // Sends a unidirectional signal to a node
     // Can be sent via relays but not routes. For routed 'signal' like capabilities, use AppMessage.

veilid-core/src/rpc_processor/rpc_start_tunnel.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
     pub(super) async fn process_start_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

veilid-core/src/rpc_processor/rpc_status.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Default)]
 pub struct StatusResult {
     pub opt_sender_info: Option<SenderInfo>,

veilid-core/src/rpc_processor/rpc_supply_block.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
     pub(super) async fn process_supply_block_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

veilid-core/src/rpc_processor/rpc_validate_dial_info.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     // Can only be sent directly, not via relays or routes
     #[instrument(level = "trace", target = "rpc", skip(self), ret, err)]

veilid-core/src/rpc_processor/rpc_value_changed.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 impl RPCProcessor {
     // Sends a dht value change notification
     // Can be sent via all methods including relays and routes but never over a safety route
@@ -88,7 +90,7 @@ impl RPCProcessor {
                 msg.header.direct_sender_node_id(),
             );
 
-            veilid_log!(self debug "{}", debug_string_stmt);
+            veilid_log!(self debug target: "dht", "{}", debug_string_stmt);
         }
 
         // Save the subkey, creating a new record if necessary

veilid-core/src/rpc_processor/rpc_watch_value.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 #[derive(Clone, Debug)]
 pub struct WatchValueAnswer {
     pub accepted: bool,
@@ -85,7 +87,7 @@ impl RPCProcessor {
             RPCQuestionDetail::WatchValueQ(Box::new(watch_value_q)),
         );
 
-        veilid_log!(self debug "{}", debug_string);
+        veilid_log!(self debug target: "dht", "{}", debug_string);
 
         let waitable_reply =
             network_result_try!(self.question(dest.clone(), question, None).await?);
@@ -122,13 +124,13 @@ impl RPCProcessor {
                 dest
             );
 
-            veilid_log!(self debug "{}", debug_string_answer);
+            veilid_log!(self debug target: "dht", "{}", debug_string_answer);
 
             let peer_ids: Vec<String> = peers
                 .iter()
                 .filter_map(|p| p.node_ids().get(key.kind).map(|k| k.to_string()))
                 .collect();
-            veilid_log!(self debug "Peers: {:#?}", peer_ids);
+            veilid_log!(self debug target: "dht", "Peers: {:#?}", peer_ids);
         }
 
         // Validate accepted requests
@@ -249,7 +251,7 @@ impl RPCProcessor {
                 watcher
             );
 
-            veilid_log!(self debug "{}", debug_string);
+            veilid_log!(self debug target: "dht", "{}", debug_string);
         }
 
         // Get the nodes that we know about that are closer to the the key than our own node
@@ -268,8 +270,7 @@ impl RPCProcessor {
                 (false, 0, watch_id.unwrap_or_default())
             } else {
                 // Accepted, lets try to watch or cancel it
-
+                let params = InboundWatchParameters {
-                let params = WatchParameters {
                     subkeys: subkeys.clone(),
                     expiration: Timestamp::new(expiration),
                     count,
@@ -280,19 +281,19 @@ impl RPCProcessor {
                 // See if we have this record ourselves, if so, accept the watch
                 let storage_manager = self.storage_manager();
                 let watch_result = network_result_try!(storage_manager
-                    .inbound_watch_value(key, params, watch_id,)
+                    .inbound_watch_value(key, params, watch_id)
                     .await
                     .map_err(RPCError::internal)?);
 
                 // Encode the watch result
                 // Rejections and cancellations are treated the same way by clients
                 let (ret_expiration, ret_watch_id) = match watch_result {
-                    WatchResult::Created { id, expiration } => (expiration.as_u64(), id),
+                    InboundWatchResult::Created { id, expiration } => (expiration.as_u64(), id),
-                    WatchResult::Changed { expiration } => {
+                    InboundWatchResult::Changed { expiration } => {
                         (expiration.as_u64(), watch_id.unwrap_or_default())
                     }
-                    WatchResult::Cancelled => (0, watch_id.unwrap_or_default()),
+                    InboundWatchResult::Cancelled => (0, watch_id.unwrap_or_default()),
-                    WatchResult::Rejected => (0, watch_id.unwrap_or_default()),
+                    InboundWatchResult::Rejected => (0, watch_id.unwrap_or_default()),
                 };
                 (true, ret_expiration, ret_watch_id)
             };
@@ -309,7 +310,7 @@ impl RPCProcessor {
                 msg.header.direct_sender_node_id()
             );
 
-            veilid_log!(self debug "{}", debug_string_answer);
+            veilid_log!(self debug target: "dht", "{}", debug_string_answer);
         }
 
         // Make WatchValue answer

veilid-core/src/rpc_processor/rpc_worker.rs

@@ -3,6 +3,8 @@ use stop_token::future::FutureExt as _;
 
 use super::*;
 
+impl_veilid_log_facility!("rpc");
+
 #[derive(Debug)]
 pub(super) enum RPCWorkerRequestKind {
     Message { message_encoded: MessageEncoded },

veilid-core/src/storage_manager/debug.rs

@@ -24,16 +24,14 @@ impl StorageManager {
             } else {
                 "".to_owned()
             };
-            let watch = if let Some(w) = v.active_watch() {
+            out += &format!("  {} {}\n", k, writer);
-                format!("  watch: {:?}\n", w)
-            } else {
-                "".to_owned()
-            };
-            out += &format!("  {} {}{}\n", k, writer, watch);
         }
         format!("{}]\n", out)
     }
-
+    pub async fn debug_watched_records(&self) -> String {
+        let inner = self.inner.lock().await;
+        inner.outbound_watch_manager.to_string()
+    }
     pub async fn debug_offline_records(&self) -> String {
         let inner = self.inner.lock().await;
         let Some(local_record_store) = &inner.local_record_store else {
@@ -53,6 +51,9 @@ impl StorageManager {
 
     pub async fn purge_local_records(&self, reclaim: Option<usize>) -> String {
         let mut inner = self.inner.lock().await;
+        if !inner.opened_records.is_empty() {
+            return "records still opened".to_owned();
+        }
         let Some(local_record_store) = &mut inner.local_record_store else {
             return "not initialized".to_owned();
         };
@@ -64,6 +65,9 @@ impl StorageManager {
     }
     pub async fn purge_remote_records(&self, reclaim: Option<usize>) -> String {
         let mut inner = self.inner.lock().await;
+        if !inner.opened_records.is_empty() {
+            return "records still opened".to_owned();
+        }
         let Some(remote_record_store) = &mut inner.remote_record_store else {
             return "not initialized".to_owned();
         };
@@ -72,6 +76,7 @@ impl StorageManager {
             .await;
         format!("Remote records purged: reclaimed {} bytes", reclaimed)
     }
+
     pub async fn debug_local_record_subkey_info(
         &self,
         key: TypedKey,

veilid-core/src/storage_manager/get_value.rs

@@ -6,8 +6,6 @@ impl_veilid_log_facility!("stor");
 struct OutboundGetValueContext {
     /// The latest value of the subkey, may be the value passed in
     pub value: Option<Arc<SignedValueData>>,
-    /// The nodes that have returned the value so far (up to the consensus count)
-    pub value_nodes: Vec<NodeRef>,
     /// The descriptor if we got a fresh one or empty if no descriptor was needed
     pub descriptor: Option<Arc<SignedValueDescriptor>>,
     /// The parsed schema from the descriptor if we have one
@@ -17,7 +15,7 @@ struct OutboundGetValueContext {
 }
 
 /// The result of the outbound_get_value operation
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub(super) struct OutboundGetValueResult {
     /// Fanout result
     pub fanout_result: FanoutResult,
@@ -74,23 +72,23 @@ impl StorageManager {
         // Make do-get-value answer context
         let context = Arc::new(Mutex::new(OutboundGetValueContext {
             value: last_get_result.opt_value,
-            value_nodes: vec![],
             descriptor: last_get_result.opt_descriptor.clone(),
             schema,
-            send_partial_update: false,
+            send_partial_update: true,
         }));
 
         // Routine to call to generate fanout
         let call_routine = {
             let context = context.clone();
             let registry = self.registry();
-            Arc::new(move |next_node: NodeRef| {
+            Arc::new(
+                move |next_node: NodeRef| -> PinBoxFutureStatic<FanoutCallResult> {
                     let context = context.clone();
                     let registry = registry.clone();
                     let last_descriptor = last_get_result.opt_descriptor.clone();
                     Box::pin(async move {
                         let rpc_processor = registry.rpc_processor();
-                    let gva = network_result_try!(
+                        let gva = match
                             rpc_processor
                                 .rpc_call_get_value(
                                     Destination::direct(next_node.routing_domain_filtered(routing_domain))
@@ -99,8 +97,18 @@ impl StorageManager {
                                     subkey,
                                     last_descriptor.map(|x| (*x).clone()),
                                 )
-                            .await?
+                                .await? {
-                    );
+                                    NetworkResult::Timeout => {
+                                        return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Timeout});
+                                    }
+                                    NetworkResult::ServiceUnavailable(_) |
+                                    NetworkResult::NoConnection(_)  |
+                                    NetworkResult::AlreadyExists(_) |
+                                    NetworkResult::InvalidMessage(_) => {
+                                        return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
+                                    }
+                                    NetworkResult::Value(v) => v
+                                };
                         let mut ctx = context.lock();
 
                         // Keep the descriptor if we got one. If we had a last_descriptor it will
@@ -110,7 +118,8 @@ impl StorageManager {
                                 let schema = match descriptor.schema() {
                                     Ok(v) => v,
                                     Err(e) => {
-                                    return Ok(NetworkResult::invalid_message(e));
+                                        veilid_log!(registry debug target:"network_result", "GetValue returned an invalid descriptor: {}", e);
+                                        return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
                                     }
                                 };
                                 ctx.schema = Some(schema);
@@ -122,8 +131,7 @@ impl StorageManager {
                         let Some(value) = gva.answer.value else {
                             // Return peers if we have some
                             veilid_log!(registry debug target:"network_result", "GetValue returned no value, fanout call returned peers {}", gva.answer.peers.len());
-
+                            return Ok(FanoutCallOutput{peer_info_list: gva.answer.peers, disposition: FanoutCallDisposition::Rejected});
-                        return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list: gva.answer.peers}))
                         };
 
                         veilid_log!(registry debug "GetValue got value back: len={} seq={}", value.value_data().data().len(), value.value_data().seq());
@@ -133,9 +141,7 @@ impl StorageManager {
                         else {
                             // Got a value but no descriptor for it
                             // Move to the next node
-                        return Ok(NetworkResult::invalid_message(
+                            return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
-                            "Got value with no descriptor",
-                        ));
                         };
 
                         // Validate with schema
@@ -146,51 +152,52 @@ impl StorageManager {
                         ) {
                             // Validation failed, ignore this value
                             // Move to the next node
-                        return Ok(NetworkResult::invalid_message(format!(
+                            return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
-                            "Schema validation failed on subkey {}",
-                            subkey
-                        )));
                         }
 
                         // If we have a prior value, see if this is a newer sequence number
-                    if let Some(prior_value) = &ctx.value {
+                        let disposition = if let Some(prior_value) = &ctx.value {
                             let prior_seq = prior_value.value_data().seq();
                             let new_seq = value.value_data().seq();
 
                             if new_seq == prior_seq {
                                 // If sequence number is the same, the data should be the same
                                 if prior_value.value_data() != value.value_data() {
-                                // Move to the next node
+                                    // Value data mismatch means skip this node
-                                return Ok(NetworkResult::invalid_message(
+                                    // This is okay because even the conflicting value is signed,
-                                    "value data mismatch",
+                                    // so the application just needs to push a newer value
-                                ));
+                                    FanoutCallDisposition::Stale
-                            }
+                                } else {
                                     // Increase the consensus count for the existing value
-                            ctx.value_nodes.push(next_node);
+                                    FanoutCallDisposition::Accepted
+                                }
                             } else if new_seq > prior_seq {
                                 // If the sequence number is greater, start over with the new value
                                 ctx.value = Some(Arc::new(value));
-                            // One node has shown us this value so far
-                            ctx.value_nodes = vec![next_node];
                                 // Send an update since the value changed
                                 ctx.send_partial_update = true;
+
+                                // Restart the consensus since we have a new value, but
+                                // don't retry nodes we've already seen because they will return
+                                // the same answer
+                                FanoutCallDisposition::AcceptedNewer
                             } else {
                                 // If the sequence number is older, ignore it
+                                FanoutCallDisposition::Stale
                             }
                         } else {
                             // If we have no prior value, keep it
                             ctx.value = Some(Arc::new(value));
-                        // One node has shown us this value so far
+                            // No value was returned
-                        ctx.value_nodes = vec![next_node];
+                            FanoutCallDisposition::Accepted
-                        // Send an update since the value changed
+                        };
-                        ctx.send_partial_update = true;
-                    }
                         // Return peers if we have some
                         veilid_log!(registry debug target:"network_result", "GetValue fanout call returned peers {}", gva.answer.peers.len());
 
-                    Ok(NetworkResult::value(FanoutCallOutput{peer_info_list: gva.answer.peers}))
+                        Ok(FanoutCallOutput{peer_info_list: gva.answer.peers, disposition})
                     }.instrument(tracing::trace_span!("outbound_get_value fanout routine"))) as PinBoxFuture<FanoutCallResult>
-            })
+                },
+            )
         };
 
         // Routine to call to check if we're done at each step
@@ -198,37 +205,44 @@ impl StorageManager {
             let context = context.clone();
             let out_tx = out_tx.clone();
             let registry = self.registry();
-            Arc::new(move |_closest_nodes: &[NodeRef]| {
+            Arc::new(move |fanout_result: &FanoutResult| -> bool {
                 let mut ctx = context.lock();
 
-                // send partial update if desired
+                match fanout_result.kind {
-                if ctx.send_partial_update {
+                    FanoutResultKind::Incomplete => {
+                        // Send partial update if desired, if we've gotten at least one consensus node
+                        if ctx.send_partial_update && !fanout_result.consensus_nodes.is_empty() {
                             ctx.send_partial_update = false;
 
-                    // return partial result
+                            // Return partial result
-                    let fanout_result = FanoutResult {
+                            let out = OutboundGetValueResult {
-                        kind: FanoutResultKind::Partial,
+                                fanout_result: fanout_result.clone(),
-                        value_nodes: ctx.value_nodes.clone(),
-                    };
-                    if let Err(e) = out_tx.send(Ok(OutboundGetValueResult {
-                        fanout_result,
                                 get_result: GetResult {
                                     opt_value: ctx.value.clone(),
                                     opt_descriptor: ctx.descriptor.clone(),
                                 },
-                    })) {
+                            };
+                            veilid_log!(registry debug "Sending partial GetValue result: {:?}", out);
+                            if let Err(e) = out_tx.send(Ok(out)) {
                                 veilid_log!(registry debug "Sending partial GetValue result failed: {}", e);
                             }
                         }
-
+                        // Keep going
-                // If we have reached sufficient consensus, return done
+                        false
-                if ctx.value.is_some()
+                    }
-                    && ctx.descriptor.is_some()
+                    FanoutResultKind::Timeout | FanoutResultKind::Exhausted => {
-                    && ctx.value_nodes.len() >= consensus_count
+                        // Signal we're done
-                {
+                        true
-                    return Some(());
+                    }
+                    FanoutResultKind::Consensus => {
+                        assert!(
+                            ctx.value.is_some() && ctx.descriptor.is_some(),
+                            "should have gotten a value if we got consensus"
+                        );
+                        // Signal we're done
+                        true
+                    }
                 }
-                None
             })
         };
 
@@ -244,21 +258,16 @@ impl StorageManager {
                         key,
                         key_count,
                         fanout,
+                        consensus_count,
                         timeout_us,
                         capability_fanout_node_info_filter(vec![CAP_DHT]),
                         call_routine,
                         check_done,
                     );
 
-                    let kind = match fanout_call.run(init_fanout_queue).await {
+                    let fanout_result = match fanout_call.run(init_fanout_queue).await {
-                        // If we don't finish in the timeout (too much time passed checking for consensus)
+                        Ok(v) => v,
-                        TimeoutOr::Timeout => FanoutResultKind::Timeout,
+                        Err(e) => {
-                        // If we finished with or without consensus (enough nodes returning the same value)
-                        TimeoutOr::Value(Ok(Some(()))) => FanoutResultKind::Finished,
-                        // If we ran out of nodes before getting consensus)
-                        TimeoutOr::Value(Ok(None)) => FanoutResultKind::Exhausted,
-                        // Failed
-                        TimeoutOr::Value(Err(e)) => {
                             // If we finished with an error, return that
                             veilid_log!(registry debug "GetValue fanout error: {}", e);
                             if let Err(e) = out_tx.send(Err(e.into())) {
@@ -268,20 +277,20 @@ impl StorageManager {
                         }
                     };
 
-                    let ctx = context.lock();
+                    veilid_log!(registry debug "GetValue Fanout: {:#}", fanout_result);
-                    let fanout_result = FanoutResult {
-                        kind,
-                        value_nodes: ctx.value_nodes.clone(),
-                    };
-                    veilid_log!(registry debug "GetValue Fanout: {:?}", fanout_result);
 
-                    if let Err(e) = out_tx.send(Ok(OutboundGetValueResult {
+                    let out = {
+                        let ctx = context.lock();
+                        OutboundGetValueResult {
                             fanout_result,
                             get_result: GetResult {
                                 opt_value: ctx.value.clone(),
                                 opt_descriptor: ctx.descriptor.clone(),
                             },
-                    })) {
+                        }
+                    };
+
+                    if let Err(e) = out_tx.send(Ok(out)) {
                         veilid_log!(registry debug "Sending GetValue result failed: {}", e);
                     }
                 }
@@ -316,18 +325,18 @@ impl StorageManager {
                                 return false;
                             }
                         };
-                        let is_partial = result.fanout_result.kind.is_partial();
+                        let is_incomplete = result.fanout_result.kind.is_incomplete();
                         let value_data = match this.process_outbound_get_value_result(key, subkey, Some(last_seq), result).await {
                             Ok(Some(v)) => v,
                             Ok(None) => {
-                                return is_partial;
+                                return is_incomplete;
                             }
                             Err(e) => {
                                 veilid_log!(this debug "Deferred fanout error: {}", e);
                                 return false;
                             }
                         };
-                        if is_partial {
+                        if is_incomplete {
                             // If more partial results show up, don't send an update until we're done
                             return true;
                         }
@@ -349,7 +358,7 @@ impl StorageManager {
     #[instrument(level = "trace", target = "dht", skip_all)]
     pub(super) async fn process_outbound_get_value_result(
         &self,
-        key: TypedKey,
+        record_key: TypedKey,
         subkey: ValueSubkey,
         opt_last_seq: Option<u32>,
         result: get_value::OutboundGetValueResult,
@@ -360,13 +369,20 @@ impl StorageManager {
             return Ok(None);
         };
 
+        // Get cryptosystem
+        let crypto = self.crypto();
+        let Some(vcrypto) = crypto.get(record_key.kind) else {
+            apibail_generic!("unsupported cryptosystem");
+        };
+
         // Keep the list of nodes that returned a value for later reference
         let mut inner = self.inner.lock().await;
 
         Self::process_fanout_results_inner(
             &mut inner,
-            key,
+            &vcrypto,
-            core::iter::once((subkey, &result.fanout_result)),
+            record_key,
+            core::iter::once((ValueSubkeyRangeSet::single(subkey), result.fanout_result)),
             false,
             self.config()
                 .with(|c| c.network.dht.set_value_count as usize),
@@ -376,10 +392,10 @@ impl StorageManager {
         if Some(get_result_value.value_data().seq()) != opt_last_seq {
             Self::handle_set_local_value_inner(
                 &mut inner,
-                key,
+                record_key,
                 subkey,
                 get_result_value.clone(),
-                WatchUpdateMode::UpdateAll,
+                InboundWatchUpdateMode::UpdateAll,
             )
             .await?;
         }

veilid-core/src/storage_manager/inspect_value.rs

@@ -29,7 +29,9 @@ impl DescriptorInfo {
 struct SubkeySeqCount {
     /// The newest sequence number found for a subkey
     pub seq: ValueSeqNum,
-    /// The nodes that have returned the value so far (up to the consensus count)
+    /// The set of nodes that had the most recent value for this subkey
+    pub consensus_nodes: Vec<NodeRef>,
+    /// The set of nodes that had any value for this subkey
     pub value_nodes: Vec<NodeRef>,
 }
 
@@ -44,7 +46,7 @@ struct OutboundInspectValueContext {
 /// The result of the outbound_get_value operation
 pub(super) struct OutboundInspectValueResult {
     /// Fanout results for each subkey
-    pub fanout_results: Vec<FanoutResult>,
+    pub subkey_fanout_results: Vec<FanoutResult>,
     /// The inspection that was retrieved
     pub inspect_result: InspectResult,
 }
@@ -110,6 +112,7 @@ impl StorageManager {
                 .iter()
                 .map(|s| SubkeySeqCount {
                     seq: *s,
+                    consensus_nodes: vec![],
                     value_nodes: vec![],
                 })
                 .collect(),
@@ -120,7 +123,8 @@ impl StorageManager {
         let call_routine = {
             let context = context.clone();
             let registry = self.registry();
-            Arc::new(move |next_node: NodeRef| {
+            Arc::new(
+                move |next_node: NodeRef| -> PinBoxFutureStatic<FanoutCallResult> {
                     let context = context.clone();
                     let registry = registry.clone();
                     let opt_descriptor = local_inspect_result.opt_descriptor.clone();
@@ -128,7 +132,7 @@ impl StorageManager {
                     Box::pin(async move {
                         let rpc_processor = registry.rpc_processor();
 
-                    let iva = network_result_try!(
+                        let iva = match
                             rpc_processor
                                 .rpc_call_inspect_value(
                                     Destination::direct(next_node.routing_domain_filtered(routing_domain)).with_safety(safety_selection),
@@ -136,8 +140,19 @@ impl StorageManager {
                                     subkeys.clone(),
                                     opt_descriptor.map(|x| (*x).clone()),
                                 )
-                            .await?
+                                .await? {
-                    );
+                            NetworkResult::Timeout => {
+                                return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Timeout});
+                            }
+                            NetworkResult::ServiceUnavailable(_) |
+                            NetworkResult::NoConnection(_)  |
+                            NetworkResult::AlreadyExists(_) |
+                            NetworkResult::InvalidMessage(_) => {
+                                return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
+                            }
+                            NetworkResult::Value(v) => v
+                        };
+
                         let answer = iva.answer;
 
                         // Keep the descriptor if we got one. If we had a last_descriptor it will
@@ -150,7 +165,8 @@ impl StorageManager {
                                     match DescriptorInfo::new(Arc::new(descriptor.clone()), &subkeys) {
                                         Ok(v) => v,
                                         Err(e) => {
-                                        return Ok(NetworkResult::invalid_message(e));
+                                            veilid_log!(registry debug target:"network_result", "InspectValue returned an invalid descriptor: {}", e);
+                                            return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
                                         }
                                     };
                                 ctx.opt_descriptor_info = Some(descriptor_info);
@@ -158,17 +174,20 @@ impl StorageManager {
                         }
 
                         // Keep the value if we got one and it is newer and it passes schema validation
-                    if !answer.seqs.is_empty() {
+                        if answer.seqs.is_empty() {
-                        veilid_log!(registry debug "Got seqs back: len={}", answer.seqs.len());
+                            veilid_log!(registry debug target:"network_result", "InspectValue returned no seq, fanout call returned peers {}", answer.peers.len());
+                            return Ok(FanoutCallOutput{peer_info_list: answer.peers, disposition: FanoutCallDisposition::Rejected});
+                        }
+
+                        veilid_log!(registry debug target:"network_result", "Got seqs back: len={}", answer.seqs.len());
                         let mut ctx = context.lock();
 
                         // Ensure we have a schema and descriptor etc
                         let Some(descriptor_info) = &ctx.opt_descriptor_info else {
                             // Got a value but no descriptor for it
                             // Move to the next node
-                            return Ok(NetworkResult::invalid_message(
+                            veilid_log!(registry debug target:"network_result", "InspectValue returned a value with no descriptor invalid descriptor");
-                                "Got inspection with no descriptor",
+                            return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
-                            ));
                         };
 
                         // Get number of subkeys from schema and ensure we are getting the
@@ -177,11 +196,10 @@ impl StorageManager {
                         if answer.seqs.len() as u64 != descriptor_info.subkeys.len() as u64 {
                             // Not the right number of sequence numbers
                             // Move to the next node
-                            return Ok(NetworkResult::invalid_message(format!(
+                            veilid_log!(registry debug target:"network_result", "wrong number of seqs returned {} (wanted {})",
-                                "wrong number of seqs returned {} (wanted {})",
                                 answer.seqs.len(),
-                                descriptor_info.subkeys.len()
+                                descriptor_info.subkeys.len());
-                            )));
+                            return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
                         }
 
                         // If we have a prior seqs list, merge in the new seqs
@@ -192,18 +210,17 @@ impl StorageManager {
                                 .map(|s| SubkeySeqCount {
                                     seq: *s,
                                     // One node has shown us the newest sequence numbers so far
-                                    value_nodes: if *s == ValueSeqNum::MAX {
+                                    consensus_nodes: vec![next_node.clone()],
-                                        vec![]
+                                    value_nodes: vec![next_node.clone()],
-                                    } else {
-                                        vec![next_node.clone()]
-                                    },
                                 })
                                 .collect();
                         } else {
                             if ctx.seqcounts.len() != answer.seqs.len() {
-                                return Err(RPCError::internal(
+                                veilid_log!(registry debug target:"network_result", "seqs list length should always be equal by now: {} (wanted {})",
-                                    "seqs list length should always be equal by now",
+                                    answer.seqs.len(),
-                                ));
+                                    ctx.seqcounts.len());
+                                return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
+
                             }
                             for pair in ctx.seqcounts.iter_mut().zip(answer.seqs.iter()) {
                                 let ctx_seqcnt = pair.0;
@@ -212,7 +229,7 @@ impl StorageManager {
                                 // If we already have consensus for this subkey, don't bother updating it any more
                                 // While we may find a better sequence number if we keep looking, this does not mimic the behavior
                                 // of get and set unless we stop here
-                                if ctx_seqcnt.value_nodes.len() >= consensus_count {
+                                if ctx_seqcnt.consensus_nodes.len() >= consensus_count {
                                     continue;
                                 }
 
@@ -225,42 +242,45 @@ impl StorageManager {
                                     {
                                         // One node has shown us the latest sequence numbers so far
                                         ctx_seqcnt.seq = answer_seq;
-                                        ctx_seqcnt.value_nodes = vec![next_node.clone()];
+                                        ctx_seqcnt.consensus_nodes = vec![next_node.clone()];
                                     } else if answer_seq == ctx_seqcnt.seq {
                                         // Keep the nodes that showed us the latest values
+                                        ctx_seqcnt.consensus_nodes.push(next_node.clone());
+                                    }
+                                }
                                 ctx_seqcnt.value_nodes.push(next_node.clone());
                             }
                         }
-                            }
+
-                        }
-                    }
 
                         // Return peers if we have some
                         veilid_log!(registry debug target:"network_result", "InspectValue fanout call returned peers {}", answer.peers.len());
 
-                    Ok(NetworkResult::value(FanoutCallOutput { peer_info_list: answer.peers}))
+                        // Inspect doesn't actually use the fanout queue consensus tracker
+                        Ok(FanoutCallOutput { peer_info_list: answer.peers, disposition: FanoutCallDisposition::Accepted})
                     }.instrument(tracing::trace_span!("outbound_inspect_value fanout call"))) as PinBoxFuture<FanoutCallResult>
-            })
+                },
+            )
         };
 
         // Routine to call to check if we're done at each step
-
+        // For inspect, we are tracking consensus externally from the FanoutCall,
+        // for each subkey, rather than a single consensus, so the single fanoutresult
+        // that is passed in here is ignored in favor of our own per-subkey tracking
         let check_done = {
             let context = context.clone();
-            Arc::new(move |_closest_nodes: &[NodeRef]| {
+            Arc::new(move |_: &FanoutResult| {
                 // If we have reached sufficient consensus on all subkeys, return done
                 let ctx = context.lock();
                 let mut has_consensus = true;
                 for cs in ctx.seqcounts.iter() {
-                    if cs.value_nodes.len() < consensus_count {
+                    if cs.consensus_nodes.len() < consensus_count {
                         has_consensus = false;
                         break;
                     }
                 }
-                if !ctx.seqcounts.is_empty() && ctx.opt_descriptor_info.is_some() && has_consensus {
+
-                    return Some(());
+                !ctx.seqcounts.is_empty() && ctx.opt_descriptor_info.is_some() && has_consensus
-                }
-                None
             })
         };
 
@@ -271,46 +291,39 @@ impl StorageManager {
             key,
             key_count,
             fanout,
+            consensus_count,
             timeout_us,
             capability_fanout_node_info_filter(vec![CAP_DHT]),
             call_routine,
             check_done,
         );
 
-        let kind = match fanout_call.run(init_fanout_queue).await {
+        let fanout_result = fanout_call.run(init_fanout_queue).await?;
-            // If we don't finish in the timeout (too much time passed checking for consensus)
-            TimeoutOr::Timeout => FanoutResultKind::Timeout,
-            // If we finished with or without consensus (enough nodes returning the same value)
-            TimeoutOr::Value(Ok(Some(()))) => FanoutResultKind::Finished,
-            // If we ran out of nodes before getting consensus)
-            TimeoutOr::Value(Ok(None)) => FanoutResultKind::Exhausted,
-            // Failed
-            TimeoutOr::Value(Err(e)) => {
-                // If we finished with an error, return that
-                veilid_log!(self debug "InspectValue Fanout Error: {}", e);
-                return Err(e.into());
-            }
-        };
 
         let ctx = context.lock();
-        let mut fanout_results = vec![];
+        let mut subkey_fanout_results = vec![];
         for cs in &ctx.seqcounts {
-            let has_consensus = cs.value_nodes.len() >= consensus_count;
+            let has_consensus = cs.consensus_nodes.len() >= consensus_count;
-            let fanout_result = FanoutResult {
+            let subkey_fanout_result = FanoutResult {
                 kind: if has_consensus {
-                    FanoutResultKind::Finished
+                    FanoutResultKind::Consensus
                 } else {
-                    kind
+                    fanout_result.kind
                 },
+                consensus_nodes: cs.consensus_nodes.clone(),
                 value_nodes: cs.value_nodes.clone(),
             };
-            fanout_results.push(fanout_result);
+            subkey_fanout_results.push(subkey_fanout_result);
         }
 
-        veilid_log!(self debug "InspectValue Fanout ({:?}):\n{}", kind, debug_fanout_results(&fanout_results));
+        if subkey_fanout_results.len() == 1 {
+            veilid_log!(self debug "InspectValue Fanout: {:#}\n{:#}", fanout_result, subkey_fanout_results.first().unwrap());
+        } else {
+            veilid_log!(self debug "InspectValue Fanout: {:#}:\n{}", fanout_result, debug_fanout_results(&subkey_fanout_results));
+        }
 
         Ok(OutboundInspectValueResult {
-            fanout_results,
+            subkey_fanout_results,
             inspect_result: InspectResult {
                 subkeys: ctx
                     .opt_descriptor_info

veilid-core/src/storage_manager/mod.rs

@@ -1,6 +1,7 @@
 mod debug;
 mod get_value;
 mod inspect_value;
+mod outbound_watch_manager;
 mod record_store;
 mod set_value;
 mod tasks;
@@ -8,11 +9,12 @@ mod types;
 mod watch_value;
 
 use super::*;
+use outbound_watch_manager::*;
 use record_store::*;
 use routing_table::*;
 use rpc_processor::*;
 
-pub use record_store::{WatchParameters, WatchResult};
+pub use record_store::{InboundWatchParameters, InboundWatchResult};
 
 pub use types::*;
 
@@ -24,18 +26,26 @@ pub(crate) const MAX_SUBKEY_SIZE: usize = ValueData::MAX_LEN;
 pub(crate) const MAX_RECORD_DATA_SIZE: usize = 1_048_576;
 /// Frequency to flush record stores to disk
 const FLUSH_RECORD_STORES_INTERVAL_SECS: u32 = 1;
+/// Frequency to save metadata to disk
+const SAVE_METADATA_INTERVAL_SECS: u32 = 30;
 /// Frequency to check for offline subkeys writes to send to the network
 const OFFLINE_SUBKEY_WRITES_INTERVAL_SECS: u32 = 5;
 /// Frequency to send ValueChanged notifications to the network
 const SEND_VALUE_CHANGES_INTERVAL_SECS: u32 = 1;
-/// Frequency to check for dead nodes and routes for client-side active watches
+/// Frequency to check for dead nodes and routes for client-side outbound watches
-const CHECK_ACTIVE_WATCHES_INTERVAL_SECS: u32 = 1;
+const CHECK_OUTBOUND_WATCHES_INTERVAL_SECS: u32 = 1;
+/// Frequency to retry reconciliation of watches that are not at consensus
+const RECONCILE_OUTBOUND_WATCHES_INTERVAL_SECS: u32 = 300;
+/// How long before expiration to try to renew per-node watches
+const RENEW_OUTBOUND_WATCHES_DURATION_SECS: u32 = 30;
 /// Frequency to check for expired server-side watched records
 const CHECK_WATCHED_RECORDS_INTERVAL_SECS: u32 = 1;
 /// Table store table for storage manager metadata
 const STORAGE_MANAGER_METADATA: &str = "storage_manager_metadata";
 /// Storage manager metadata key name for offline subkey write persistence
 const OFFLINE_SUBKEY_WRITES: &[u8] = b"offline_subkey_writes";
+/// Outbound watch manager metadata key name for watch persistence
+const OUTBOUND_WATCH_MANAGER: &[u8] = b"outbound_watch_manager";
 
 #[derive(Debug, Clone)]
 /// A single 'value changed' message to send
@@ -61,6 +71,8 @@ struct StorageManagerInner {
     pub offline_subkey_writes: HashMap<TypedKey, tasks::offline_subkey_writes::OfflineSubkeyWrite>,
     /// Record subkeys that are currently being written to in the foreground
     pub active_subkey_writes: HashMap<TypedKey, ValueSubkeyRangeSet>,
+    /// State management for outbound watches
+    pub outbound_watch_manager: OutboundWatchManager,
     /// Storage manager metadata that is persistent, including copy of offline subkey writes
     pub metadata_db: Option<TableDB>,
     /// Background processing task (not part of attachment manager tick tree so it happens when detached too)
@@ -76,6 +88,7 @@ impl fmt::Debug for StorageManagerInner {
             .field("remote_record_store", &self.remote_record_store)
             .field("offline_subkey_writes", &self.offline_subkey_writes)
             .field("active_subkey_writes", &self.active_subkey_writes)
+            .field("outbound_watch_manager", &self.outbound_watch_manager)
             //.field("metadata_db", &self.metadata_db)
             //.field("tick_future", &self.tick_future)
             .finish()
@@ -87,17 +100,24 @@ pub(crate) struct StorageManager {
     inner: AsyncMutex<StorageManagerInner>,
 
     // Background processes
+    save_metadata_task: TickTask<EyreReport>,
     flush_record_stores_task: TickTask<EyreReport>,
     offline_subkey_writes_task: TickTask<EyreReport>,
     send_value_changes_task: TickTask<EyreReport>,
-    check_active_watches_task: TickTask<EyreReport>,
+    check_outbound_watches_task: TickTask<EyreReport>,
-    check_watched_records_task: TickTask<EyreReport>,
+    check_inbound_watches_task: TickTask<EyreReport>,
 
     // Anonymous watch keys
     anonymous_watch_keys: TypedKeyPairGroup,
 
-    /// Deferred result processor
+    // Outbound watch operation lock
-    deferred_result_processor: DeferredStreamProcessor,
+    // Keeps changes to watches to one-at-a-time per record
+    outbound_watch_lock_table: AsyncTagLockTable<TypedKey>,
+
+    // Background operation processor
+    // for offline subkey writes, watch changes, and any other
+    // background operations the storage manager wants to perform
+    background_operation_processor: DeferredStreamProcessor,
 }
 
 impl fmt::Debug for StorageManager {
@@ -116,7 +136,11 @@ impl fmt::Debug for StorageManager {
             //     "check_watched_records_task",
             //     &self.check_watched_records_task,
             // )
-            .field("deferred_result_processor", &self.deferred_result_processor)
+            .field("outbound_watch_lock_table", &self.outbound_watch_lock_table)
+            .field(
+                "background_operation_processor",
+                &self.background_operation_processor,
+            )
             .field("anonymous_watch_keys", &self.anonymous_watch_keys)
             .finish()
     }
@@ -145,6 +169,7 @@ impl StorageManager {
             registry,
             inner: AsyncMutex::new(inner),
 
+            save_metadata_task: TickTask::new("save_metadata_task", SAVE_METADATA_INTERVAL_SECS),
             flush_record_stores_task: TickTask::new(
                 "flush_record_stores_task",
                 FLUSH_RECORD_STORES_INTERVAL_SECS,
@@ -157,17 +182,17 @@ impl StorageManager {
                 "send_value_changes_task",
                 SEND_VALUE_CHANGES_INTERVAL_SECS,
             ),
-            check_active_watches_task: TickTask::new(
+            check_outbound_watches_task: TickTask::new(
                 "check_active_watches_task",
-                CHECK_ACTIVE_WATCHES_INTERVAL_SECS,
+                CHECK_OUTBOUND_WATCHES_INTERVAL_SECS,
             ),
-            check_watched_records_task: TickTask::new(
+            check_inbound_watches_task: TickTask::new(
                 "check_watched_records_task",
                 CHECK_WATCHED_RECORDS_INTERVAL_SECS,
             ),
-
+            outbound_watch_lock_table: AsyncTagLockTable::new(),
             anonymous_watch_keys,
-            deferred_result_processor: DeferredStreamProcessor::new(),
+            background_operation_processor: DeferredStreamProcessor::new(),
         };
 
         this.setup_tasks();
@@ -240,7 +265,7 @@ impl StorageManager {
         }
 
         // Start deferred results processors
-        self.deferred_result_processor.init();
+        self.background_operation_processor.init();
 
         Ok(())
     }
@@ -249,6 +274,9 @@ impl StorageManager {
     async fn post_init_async(&self) -> EyreResult<()> {
         let mut inner = self.inner.lock().await;
 
+        // Resolve outbound watch manager noderefs
+        inner.outbound_watch_manager.prepare(self.routing_table());
+
         // Schedule tick
         let registry = self.registry();
         let tick_future = interval("storage manager tick", 1000, move || {
@@ -286,7 +314,7 @@ impl StorageManager {
         veilid_log!(self debug "starting storage manager shutdown");
 
         // Stop deferred result processor
-        self.deferred_result_processor.terminate().await;
+        self.background_operation_processor.terminate().await;
 
         // Terminate and release the storage manager
         {
@@ -320,6 +348,7 @@ impl StorageManager {
         if let Some(metadata_db) = &inner.metadata_db {
             let tx = metadata_db.transact();
             tx.store_json(0, OFFLINE_SUBKEY_WRITES, &inner.offline_subkey_writes)?;
+            tx.store_json(0, OUTBOUND_WATCH_MANAGER, &inner.outbound_watch_manager)?;
             tx.commit().await.wrap_err("failed to commit")?
         }
         Ok(())
@@ -338,7 +367,19 @@ impl StorageManager {
                     }
                     Default::default()
                 }
+            };
+            inner.outbound_watch_manager = match metadata_db
+                .load_json(0, OUTBOUND_WATCH_MANAGER)
+                .await
+            {
+                Ok(v) => v.unwrap_or_default(),
+                Err(_) => {
+                    if let Err(e) = metadata_db.delete(0, OUTBOUND_WATCH_MANAGER).await {
+                        veilid_log!(self debug "outbound_watch_manager format changed, clearing: {}", e);
                     }
+                    Default::default()
+                }
+            };
         }
         Ok(())
     }
@@ -362,21 +403,39 @@ impl StorageManager {
     }
 
     /// Get the set of nodes in our active watches
-    pub async fn get_active_watch_nodes(&self) -> Vec<Destination> {
+    pub async fn get_outbound_watch_nodes(&self) -> Vec<Destination> {
         let inner = self.inner.lock().await;
-        inner
+
-            .opened_records
+        let mut out = vec![];
-            .values()
+        let mut node_set = HashSet::new();
-            .filter_map(|v| {
+        for v in inner.outbound_watch_manager.outbound_watches.values() {
-                v.active_watch().map(|aw| {
+            if let Some(current) = v.state() {
+                let node_refs =
+                    current.watch_node_refs(&inner.outbound_watch_manager.per_node_states);
+                for node_ref in &node_refs {
+                    let mut found = false;
+                    for nid in node_ref.node_ids().iter() {
+                        if node_set.contains(nid) {
+                            found = true;
+                            break;
+                        }
+                    }
+                    if found {
+                        continue;
+                    }
+
+                    node_set.insert(node_ref.best_node_id());
+                    out.push(
                         Destination::direct(
-                        aw.watch_node
+                            node_ref.routing_domain_filtered(RoutingDomain::PublicInternet),
-                            .routing_domain_filtered(RoutingDomain::PublicInternet),
                         )
-                    .with_safety(v.safety_selection())
+                        .with_safety(current.params().safety_selection),
-                })
+                    )
-            })
+                }
-            .collect()
+            }
+        }
+
+        out
     }
 
     /// Builds the record key for a given schema and owner
@@ -514,53 +573,19 @@ impl StorageManager {
     #[instrument(level = "trace", target = "stor", skip_all)]
     pub async fn close_record(&self, key: TypedKey) -> VeilidAPIResult<()> {
         // Attempt to close the record, returning the opened record if it wasn't already closed
-        let opened_record = {
         let mut inner = self.inner.lock().await;
-            let Some(opened_record) = Self::close_record_inner(&mut inner, key)? else {
+        Self::close_record_inner(&mut inner, key)?;
-                return Ok(());
+        Ok(())
-            };
+    }
-            opened_record
-        };
 
-        // See if we have an active watch on the closed record
+    /// Close all opened records
-        let Some(active_watch) = opened_record.active_watch() else {
+    #[instrument(level = "trace", target = "stor", skip_all)]
-            return Ok(());
+    pub async fn close_all_records(&self) -> VeilidAPIResult<()> {
-        };
+        // Attempt to close the record, returning the opened record if it wasn't already closed
-
+        let mut inner = self.inner.lock().await;
-        // Send a one-time cancel request for the watch if we have one and we're online
+        let keys = inner.opened_records.keys().copied().collect::<Vec<_>>();
-        if !self.dht_is_online() {
+        for key in keys {
-            veilid_log!(self debug "skipping last-ditch watch cancel because we are offline");
+            Self::close_record_inner(&mut inner, key)?;
-            return Ok(());
-        }
-        // Use the safety selection we opened the record with
-        // Use the writer we opened with as the 'watcher' as well
-        let opt_owvresult = match self
-            .outbound_watch_value_cancel(
-                key,
-                ValueSubkeyRangeSet::full(),
-                opened_record.safety_selection(),
-                opened_record.writer().cloned(),
-                active_watch.id,
-                active_watch.watch_node,
-            )
-            .await
-        {
-            Ok(v) => v,
-            Err(e) => {
-                veilid_log!(self debug
-                    "close record watch cancel failed: {}", e
-                );
-                None
-            }
-        };
-        if let Some(owvresult) = opt_owvresult {
-            if owvresult.expiration_ts.as_u64() != 0 {
-                veilid_log!(self debug
-                    "close record watch cancel should have zero expiration"
-                );
-            }
-        } else {
-            veilid_log!(self debug "close record watch cancel unsuccessful");
         }
 
         Ok(())
@@ -570,10 +595,10 @@ impl StorageManager {
     #[instrument(level = "trace", target = "stor", skip_all)]
     pub async fn delete_record(&self, key: TypedKey) -> VeilidAPIResult<()> {
         // Ensure the record is closed
-        self.close_record(key).await?;
+        let mut inner = self.inner.lock().await;
+        Self::close_record_inner(&mut inner, key)?;
 
         // Get record from the local store
-        let mut inner = self.inner.lock().await;
         let Some(local_record_store) = inner.local_record_store.as_mut() else {
             apibail_not_initialized!();
         };
@@ -636,7 +661,7 @@ impl StorageManager {
             apibail_internal!("failed to receive results");
         };
         let result = result?;
-        let partial = result.fanout_result.kind.is_partial();
+        let partial = result.fanout_result.kind.is_incomplete();
 
         // Process the returned result
         let out = self
@@ -735,7 +760,7 @@ impl StorageManager {
             key,
             subkey,
             signed_value_data.clone(),
-            WatchUpdateMode::NoUpdate,
+            InboundWatchUpdateMode::NoUpdate,
         )
         .await?;
 
@@ -800,7 +825,7 @@ impl StorageManager {
                 apibail_internal!("failed to receive results");
             };
             let result = result?;
-            let partial = result.fanout_result.kind.is_partial();
+            let partial = result.fanout_result.kind.is_incomplete();
 
             // Process the returned result
             let out = self
@@ -845,7 +870,7 @@ impl StorageManager {
         out
     }
 
-    /// Create,update or cancel an outbound watch to a DHT value
+    /// Create, update or cancel an outbound watch to a DHT value
     #[instrument(level = "trace", target = "stor", skip_all)]
     pub async fn watch_values(
         &self,
@@ -853,20 +878,37 @@ impl StorageManager {
         subkeys: ValueSubkeyRangeSet,
         expiration: Timestamp,
         count: u32,
-    ) -> VeilidAPIResult<Timestamp> {
+    ) -> VeilidAPIResult<bool> {
-        let inner = self.inner.lock().await;
+        // Obtain the watch change lock
+        // (may need to wait for background operations to complete on the watch)
+        let watch_lock = self.outbound_watch_lock_table.lock_tag(key).await;
+
+        self.watch_values_inner(watch_lock, subkeys, expiration, count)
+            .await
+    }
+
+    #[instrument(level = "trace", target = "stor", skip_all)]
+    async fn watch_values_inner(
+        &self,
+        watch_lock: AsyncTagLockGuard<TypedKey>,
+        subkeys: ValueSubkeyRangeSet,
+        expiration: Timestamp,
+        count: u32,
+    ) -> VeilidAPIResult<bool> {
+        let key = watch_lock.tag();
+
+        // Obtain the inner state lock
+        let mut inner = self.inner.lock().await;
 
         // Get the safety selection and the writer we opened this record
-        // and whatever active watch id and watch node we may have in case this is a watch update
+        let (safety_selection, opt_watcher) = {
-        let (safety_selection, opt_writer, opt_watch_id, opt_watch_node) = {
             let Some(opened_record) = inner.opened_records.get(&key) else {
+                // Record must be opened already to change watch
                 apibail_generic!("record not open");
             };
             (
                 opened_record.safety_selection(),
                 opened_record.writer().cloned(),
-                opened_record.active_watch().map(|aw| aw.id),
-                opened_record.active_watch().map(|aw| aw.watch_node.clone()),
             )
         };
 
@@ -888,86 +930,67 @@ impl StorageManager {
         };
         let subkeys = schema.truncate_subkeys(&subkeys, None);
 
-        // Get rpc processor and drop mutex so we don't block while requesting the watch from the network
+        // Calculate desired watch parameters
-        if !self.dht_is_online() {
+        let desired_params = if count == 0 {
-            apibail_try_again!("offline, try again later");
+            // Cancel
+            None
+        } else {
+            // Get the minimum expiration timestamp we will accept
+            let rpc_timeout_us = self
+                .config()
+                .with(|c| TimestampDuration::from(ms_to_us(c.network.rpc.timeout_ms)));
+            let cur_ts = get_timestamp();
+            let min_expiration_ts = Timestamp::new(cur_ts + rpc_timeout_us.as_u64());
+            let expiration_ts = if expiration.as_u64() == 0 {
+                expiration
+            } else if expiration < min_expiration_ts {
+                apibail_invalid_argument!("expiration is too soon", "expiration", expiration);
+            } else {
+                expiration
             };
 
+            // Create or modify
+            Some(OutboundWatchParameters {
+                expiration_ts,
+                count,
+                subkeys,
+                opt_watcher,
+                safety_selection,
+            })
+        };
+
+        // Modify the 'desired' state of the watch or add one if it does not exist
+        inner
+            .outbound_watch_manager
+            .set_desired_watch(key, desired_params);
+
         // Drop the lock for network access
         drop(inner);
 
-        // Use the safety selection we opened the record with
+        // Process this watch's state machine operations until we are done
-        // Use the writer we opened with as the 'watcher' as well
+        loop {
-        let opt_owvresult = self
+            let opt_op_fut = {
-            .outbound_watch_value(
-                key,
-                subkeys.clone(),
-                expiration,
-                count,
-                safety_selection,
-                opt_writer,
-                opt_watch_id,
-                opt_watch_node,
-            )
-            .await?;
-        // If we did not get a valid response assume nothing changed
-        let Some(owvresult) = opt_owvresult else {
-            apibail_try_again!("did not get a valid response");
-        };
-
-        // Clear any existing watch if the watch succeeded or got cancelled
                 let mut inner = self.inner.lock().await;
-        let Some(opened_record) = inner.opened_records.get_mut(&key) else {
+                let Some(outbound_watch) =
-            apibail_generic!("record not open");
+                    inner.outbound_watch_manager.outbound_watches.get_mut(&key)
+                else {
+                    // Watch is gone
+                    return Ok(false);
                 };
-        opened_record.clear_active_watch();
+                self.get_next_outbound_watch_operation(
-
+                    key,
-        // Get the minimum expiration timestamp we will accept
+                    Some(watch_lock.clone()),
-        let (rpc_timeout_us, max_watch_expiration_us) = self.config().with(|c| {
+                    Timestamp::now(),
-            (
+                    outbound_watch,
-                TimestampDuration::from(ms_to_us(c.network.rpc.timeout_ms)),
-                TimestampDuration::from(ms_to_us(c.network.dht.max_watch_expiration_ms)),
                 )
-        });
-        let cur_ts = get_timestamp();
-        let min_expiration_ts = cur_ts + rpc_timeout_us.as_u64();
-        let max_expiration_ts = if expiration.as_u64() == 0 {
-            cur_ts + max_watch_expiration_us.as_u64()
-        } else {
-            expiration.as_u64()
             };
-
+            let Some(op_fut) = opt_op_fut else {
-        // If the expiration time is less than our minimum expiration time (or zero) consider this watch inactive
+                break;
-        let mut expiration_ts = owvresult.expiration_ts;
+            };
-        if expiration_ts.as_u64() < min_expiration_ts {
+            op_fut.await;
-            return Ok(Timestamp::new(0));
         }
 
-        // If the expiration time is greater than our maximum expiration time, clamp our local watch so we ignore extra valuechanged messages
+        Ok(true)
-        if expiration_ts.as_u64() > max_expiration_ts {
-            expiration_ts = Timestamp::new(max_expiration_ts);
-        }
-
-        // If we requested a cancellation, then consider this watch cancelled
-        if count == 0 {
-            // Expiration returned should be zero if we requested a cancellation
-            if expiration_ts.as_u64() != 0 {
-                veilid_log!(self debug "got active watch despite asking for a cancellation");
-            }
-            return Ok(Timestamp::new(0));
-        }
-
-        // Keep a record of the watch
-        opened_record.set_active_watch(ActiveWatch {
-            id: owvresult.watch_id,
-            expiration_ts,
-            watch_node: owvresult.watch_node,
-            opt_value_changed_route: owvresult.opt_value_changed_route,
-            subkeys,
-            count,
-        });
-
-        Ok(owvresult.expiration_ts)
     }
 
     #[instrument(level = "trace", target = "stor", skip_all)]
@@ -976,18 +999,31 @@ impl StorageManager {
         key: TypedKey,
         subkeys: ValueSubkeyRangeSet,
     ) -> VeilidAPIResult<bool> {
-        let (subkeys, active_watch) = {
+        // Obtain the watch change lock
+        // (may need to wait for background operations to complete on the watch)
+        let watch_lock = self.outbound_watch_lock_table.lock_tag(key).await;
+
+        // Calculate change to existing watch
+        let (subkeys, count, expiration_ts) = {
             let inner = self.inner.lock().await;
-            let Some(opened_record) = inner.opened_records.get(&key) else {
+            let Some(_opened_record) = inner.opened_records.get(&key) else {
                 apibail_generic!("record not open");
             };
 
             // See what watch we have currently if any
-            let Some(active_watch) = opened_record.active_watch() else {
+            let Some(outbound_watch) = inner.outbound_watch_manager.outbound_watches.get(&key)
+            else {
                 // If we didn't have an active watch, then we can just return false because there's nothing to do here
                 return Ok(false);
             };
 
+            // Ensure we have a 'desired' watch state
+            let Some(desired) = outbound_watch.desired() else {
+                // If we didn't have a desired watch, then we're already cancelling
+                let still_active = outbound_watch.state().is_some();
+                return Ok(still_active);
+            };
+
             // Rewrite subkey range if empty to full
             let subkeys = if subkeys.is_empty() {
                 ValueSubkeyRangeSet::full()
@@ -996,32 +1032,29 @@ impl StorageManager {
             };
 
             // Reduce the subkey range
-            let new_subkeys = active_watch.subkeys.difference(&subkeys);
+            let new_subkeys = desired.subkeys.difference(&subkeys);
 
-            (new_subkeys, active_watch)
+            // If no change is happening return false
-        };
+            if new_subkeys == desired.subkeys {
-
-        // If we have no subkeys left, then set the count to zero to indicate a full cancellation
-        let count = if subkeys.is_empty() {
-            0
-        } else {
-            active_watch.count
-        };
-
-        // Update the watch. This just calls through to the above watch_values() function
-        // This will update the active_watch so we don't need to do that in this routine.
-        let expiration_ts =
-            pin_future!(self.watch_values(key, subkeys, active_watch.expiration_ts, count)).await?;
-
-        // A zero expiration time returned from watch_value() means the watch is done
-        // or no subkeys are left, and the watch is no longer active
-        if expiration_ts.as_u64() == 0 {
-            // Return false indicating the watch is completely gone
                 return Ok(false);
             }
 
-        // Return true because the the watch was changed
+            // If we have no subkeys left, then set the count to zero to indicate a full cancellation
-        Ok(true)
+            let count = if new_subkeys.is_empty() {
+                0
+            } else if let Some(state) = outbound_watch.state() {
+                state.remaining_count()
+            } else {
+                desired.count
+            };
+
+            (new_subkeys, count, desired.expiration_ts)
+        };
+
+        // Update the watch. This just calls through to the above watch_values_inner() function
+        // This will update the active_watch so we don't need to do that in this routine.
+        self.watch_values_inner(watch_lock, subkeys, expiration_ts, count)
+            .await
     }
 
     /// Inspect an opened DHT record for its subkey sequence numbers
@@ -1038,6 +1071,12 @@ impl StorageManager {
             subkeys
         };
 
+        // Get cryptosystem
+        let crypto = self.crypto();
+        let Some(vcrypto) = crypto.get(key.kind) else {
+            apibail_generic!("unsupported cryptosystem");
+        };
+
         let mut inner = self.inner.lock().await;
         let safety_selection = {
             let Some(opened_record) = inner.opened_records.get(&key) else {
@@ -1122,7 +1161,7 @@ impl StorageManager {
         {
             assert_eq!(
                 result.inspect_result.subkeys.len() as u64,
-                result.fanout_results.len() as u64,
+                result.subkey_fanout_results.len() as u64,
                 "mismatch between subkeys returned and fanout results returned"
             );
         }
@@ -1140,10 +1179,12 @@ impl StorageManager {
             .inspect_result
             .subkeys
             .iter()
-            .zip(result.fanout_results.iter());
+            .map(ValueSubkeyRangeSet::single)
+            .zip(result.subkey_fanout_results.into_iter());
 
         Self::process_fanout_results_inner(
             &mut inner,
+            &vcrypto,
             key,
             results_iter,
             false,
@@ -1210,12 +1251,12 @@ impl StorageManager {
         fanout_result: &FanoutResult,
     ) -> bool {
         match fanout_result.kind {
-            FanoutResultKind::Partial => false,
+            FanoutResultKind::Incomplete => false,
             FanoutResultKind::Timeout => {
                 let get_consensus = self
                     .config()
                     .with(|c| c.network.dht.get_value_count as usize);
-                let value_node_count = fanout_result.value_nodes.len();
+                let value_node_count = fanout_result.consensus_nodes.len();
                 if value_node_count < get_consensus {
                     veilid_log!(self debug "timeout with insufficient consensus ({}<{}), adding offline subkey: {}:{}",
                         value_node_count, get_consensus,
@@ -1232,7 +1273,7 @@ impl StorageManager {
                 let get_consensus = self
                     .config()
                     .with(|c| c.network.dht.get_value_count as usize);
-                let value_node_count = fanout_result.value_nodes.len();
+                let value_node_count = fanout_result.consensus_nodes.len();
                 if value_node_count < get_consensus {
                     veilid_log!(self debug "exhausted with insufficient consensus ({}<{}), adding offline subkey: {}:{}",
                         value_node_count, get_consensus,
@@ -1245,7 +1286,7 @@ impl StorageManager {
                     false
                 }
             }
-            FanoutResultKind::Finished => false,
+            FanoutResultKind::Consensus => false,
         }
     }
 
@@ -1361,7 +1402,7 @@ impl StorageManager {
                 continue;
             };
             local_record_store
-                .set_subkey(key, subkey, subkey_data, WatchUpdateMode::NoUpdate)
+                .set_subkey(key, subkey, subkey_data, InboundWatchUpdateMode::NoUpdate)
                 .await?;
         }
 
@@ -1495,7 +1536,12 @@ impl StorageManager {
         if let Some(signed_value_data) = get_result.opt_value {
             // Write subkey to local store
             local_record_store
-                .set_subkey(key, subkey, signed_value_data, WatchUpdateMode::NoUpdate)
+                .set_subkey(
+                    key,
+                    subkey,
+                    signed_value_data,
+                    InboundWatchUpdateMode::NoUpdate,
+                )
                 .await?;
         }
 
@@ -1539,11 +1585,11 @@ impl StorageManager {
 
     #[instrument(level = "trace", target = "stor", skip_all)]
     pub(super) fn process_fanout_results_inner<
-        'a,
+        I: IntoIterator<Item = (ValueSubkeyRangeSet, FanoutResult)>,
-        I: IntoIterator<Item = (ValueSubkey, &'a FanoutResult)>,
     >(
         inner: &mut StorageManagerInner,
-        key: TypedKey,
+        vcrypto: &CryptoSystemGuard<'_>,
+        record_key: TypedKey,
         subkey_results_iter: I,
         is_set: bool,
         consensus_count: usize,
@@ -1552,21 +1598,21 @@ impl StorageManager {
         let local_record_store = inner.local_record_store.as_mut().unwrap();
 
         let cur_ts = Timestamp::now();
-        local_record_store.with_record_mut(key, |r| {
+        local_record_store.with_record_mut(record_key, |r| {
             let d = r.detail_mut();
 
-            for (subkey, fanout_result) in subkey_results_iter {
+            for (subkeys, fanout_result) in subkey_results_iter {
                 for node_id in fanout_result
                     .value_nodes
                     .iter()
-                    .filter_map(|x| x.node_ids().get(key.kind).map(|k| k.value))
+                    .filter_map(|x| x.node_ids().get(record_key.kind).map(|k| k.value))
                 {
                     let pnd = d.nodes.entry(node_id).or_default();
                     if is_set || pnd.last_set == Timestamp::default() {
                         pnd.last_set = cur_ts;
                     }
                     pnd.last_seen = cur_ts;
-                    pnd.subkeys.insert(subkey);
+                    pnd.subkeys = pnd.subkeys.union(&subkeys);
                 }
             }
 
@@ -1576,7 +1622,17 @@ impl StorageManager {
                 .iter()
                 .map(|kv| (*kv.0, kv.1.last_seen))
                 .collect::<Vec<_>>();
-            nodes_ts.sort_by(|a, b| b.1.cmp(&a.1));
+            nodes_ts.sort_by(|a, b| {
+                // Timestamp is first metric
+                let res = b.1.cmp(&a.1);
+                if res != cmp::Ordering::Equal {
+                    return res;
+                }
+                // Distance is the next metric, closer nodes first
+                let da = vcrypto.distance(&a.0, &record_key.value);
+                let db = vcrypto.distance(&b.0, &record_key.value);
+                da.cmp(&db)
+            });
 
             for dead_node_key in nodes_ts.iter().skip(consensus_count) {
                 d.nodes.remove(&dead_node_key.0);
@@ -1584,18 +1640,21 @@ impl StorageManager {
         });
     }
 
-    fn close_record_inner(
+    fn close_record_inner(inner: &mut StorageManagerInner, key: TypedKey) -> VeilidAPIResult<()> {
-        inner: &mut StorageManagerInner,
-        key: TypedKey,
-    ) -> VeilidAPIResult<Option<OpenedRecord>> {
         let Some(local_record_store) = inner.local_record_store.as_mut() else {
             apibail_not_initialized!();
         };
         if local_record_store.peek_record(key, |_| {}).is_none() {
-            return Err(VeilidAPIError::key_not_found(key));
+            apibail_key_not_found!(key);
         }
 
-        Ok(inner.opened_records.remove(&key))
+        if inner.opened_records.remove(&key).is_some() {
+            // Set the watch to cancelled if we have one
+            // Will process cancellation in the background
+            inner.outbound_watch_manager.set_desired_watch(key, None);
+        }
+
+        Ok(())
     }
 
     #[instrument(level = "trace", target = "stor", skip_all, err)]
@@ -1628,7 +1687,7 @@ impl StorageManager {
         key: TypedKey,
         subkey: ValueSubkey,
         signed_value_data: Arc<SignedValueData>,
-        watch_update_mode: WatchUpdateMode,
+        watch_update_mode: InboundWatchUpdateMode,
     ) -> VeilidAPIResult<()> {
         // See if it's in the local record store
         let Some(local_record_store) = inner.local_record_store.as_mut() else {
@@ -1699,7 +1758,7 @@ impl StorageManager {
         subkey: ValueSubkey,
         signed_value_data: Arc<SignedValueData>,
         signed_value_descriptor: Arc<SignedValueDescriptor>,
-        watch_update_mode: WatchUpdateMode,
+        watch_update_mode: InboundWatchUpdateMode,
     ) -> VeilidAPIResult<()> {
         // See if it's in the remote record store
         let Some(remote_record_store) = inner.remote_record_store.as_mut() else {
@@ -1791,7 +1850,7 @@ impl StorageManager {
         receiver: flume::Receiver<T>,
         handler: impl FnMut(T) -> PinBoxFutureStatic<bool> + Send + 'static,
     ) -> bool {
-        self.deferred_result_processor
+        self.background_operation_processor
-            .add(receiver.into_stream(), handler)
+            .add_stream(receiver.into_stream(), handler)
     }
 }

veilid-core/src/storage_manager/outbound_watch_manager/mod.rs

@@ -0,0 +1,213 @@
+mod outbound_watch;
+mod outbound_watch_parameters;
+mod outbound_watch_state;
+mod per_node_state;
+
+pub(in crate::storage_manager) use outbound_watch::*;
+pub(in crate::storage_manager) use outbound_watch_parameters::*;
+pub(in crate::storage_manager) use outbound_watch_state::*;
+pub(in crate::storage_manager) use per_node_state::*;
+
+use super::*;
+
+use serde_with::serde_as;
+
+impl_veilid_log_facility!("stor");
+
+#[serde_as]
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub(in crate::storage_manager) struct OutboundWatchManager {
+    /// Each watch per record key
+    #[serde(skip)]
+    pub outbound_watches: HashMap<TypedKey, OutboundWatch>,
+    /// Last known active watch per node+record
+    #[serde_as(as = "Vec<(_, _)>")]
+    pub per_node_states: HashMap<PerNodeKey, PerNodeState>,
+    /// Value changed updates that need inpection to determine if they should be reported
+    #[serde(skip)]
+    pub needs_change_inspection: HashMap<TypedKey, ValueSubkeyRangeSet>,
+}
+
+impl fmt::Display for OutboundWatchManager {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut out = format!("outbound_watches({}): [\n", self.outbound_watches.len());
+        {
+            let mut keys = self.outbound_watches.keys().copied().collect::<Vec<_>>();
+            keys.sort();
+
+            for k in keys {
+                let v = self.outbound_watches.get(&k).unwrap();
+                out += &format!("  {}:\n{}\n", k, indent_all_by(4, v.to_string()));
+            }
+        }
+        out += "]\n";
+        out += &format!("per_node_states({}): [\n", self.per_node_states.len());
+        {
+            let mut keys = self.per_node_states.keys().copied().collect::<Vec<_>>();
+            keys.sort();
+
+            for k in keys {
+                let v = self.per_node_states.get(&k).unwrap();
+                out += &format!("  {}:\n{}\n", k, indent_all_by(4, v.to_string()));
+            }
+        }
+        out += "]\n";
+        out += &format!(
+            "needs_change_inspection({}): [\n",
+            self.needs_change_inspection.len()
+        );
+        {
+            let mut keys = self
+                .needs_change_inspection
+                .keys()
+                .copied()
+                .collect::<Vec<_>>();
+            keys.sort();
+
+            for k in keys {
+                let v = self.needs_change_inspection.get(&k).unwrap();
+                out += &format!("  {}: {}\n", k, v);
+            }
+        }
+        out += "]\n";
+
+        write!(f, "{}", out)
+    }
+}
+
+impl Default for OutboundWatchManager {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl OutboundWatchManager {
+    pub fn new() -> Self {
+        Self {
+            outbound_watches: HashMap::new(),
+            per_node_states: HashMap::new(),
+            needs_change_inspection: HashMap::new(),
+        }
+    }
+
+    pub fn prepare(&mut self, routing_table: VeilidComponentGuard<'_, RoutingTable>) {
+        for (pnk, pns) in &mut self.per_node_states {
+            pns.watch_node_ref = match routing_table.lookup_node_ref(pnk.node_id) {
+                Ok(v) => v,
+                Err(e) => {
+                    veilid_log!(routing_table debug "Error looking up outbound watch node ref: {}", e);
+                    None
+                }
+            };
+        }
+        self.per_node_states
+            .retain(|_, v| v.watch_node_ref.is_some());
+
+        let keys = self.per_node_states.keys().copied().collect::<HashSet<_>>();
+
+        for v in self.outbound_watches.values_mut() {
+            if let Some(state) = v.state_mut() {
+                state.edit(&self.per_node_states, |editor| {
+                    editor.retain_nodes(|n| keys.contains(n));
+                })
+            }
+        }
+    }
+
+    pub fn set_desired_watch(
+        &mut self,
+        record_key: TypedKey,
+        desired_watch: Option<OutboundWatchParameters>,
+    ) {
+        match self.outbound_watches.get_mut(&record_key) {
+            Some(w) => {
+                // Replace desired watch
+                w.set_desired(desired_watch);
+
+                // Remove if the watch is done (shortcut the dead state)
+                if w.state().is_none() && w.state().is_none() {
+                    self.outbound_watches.remove(&record_key);
+                }
+            }
+            None => {
+                // Watch does not exist, add one if that's what is desired
+                if let Some(desired) = desired_watch {
+                    self.outbound_watches
+                        .insert(record_key, OutboundWatch::new(desired));
+                }
+            }
+        }
+    }
+
+    pub fn set_next_reconcile_ts(&mut self, record_key: TypedKey, next_ts: Timestamp) {
+        if let Some(outbound_watch) = self.outbound_watches.get_mut(&record_key) {
+            if let Some(state) = outbound_watch.state_mut() {
+                state.edit(&self.per_node_states, |editor| {
+                    editor.set_next_reconcile_ts(next_ts);
+                });
+            }
+        }
+    }
+
+    /// Iterate all per-node watches and remove ones with dead nodes from outbound watches
+    /// This may trigger reconciliation to increase the number of active per-node watches
+    /// for an outbound watch that is still alive
+    pub fn update_per_node_states(&mut self, cur_ts: Timestamp) {
+        // Node is unreachable
+        let mut dead_pnks = HashSet::new();
+        // Per-node expiration reached
+        let mut expired_pnks = HashSet::new();
+        // Count reached
+        let mut finished_pnks = HashSet::new();
+
+        for (pnk, pns) in &self.per_node_states {
+            if pns.count == 0 {
+                // If per-node watch is done, add to finished list
+                finished_pnks.insert(*pnk);
+            } else if !pns
+                .watch_node_ref
+                .as_ref()
+                .unwrap()
+                .state(cur_ts)
+                .is_alive()
+            {
+                // If node is unreachable add to dead list
+                dead_pnks.insert(*pnk);
+            } else if cur_ts >= pns.expiration_ts {
+                // If per-node watch has expired add to expired list
+                expired_pnks.insert(*pnk);
+            }
+        }
+
+        // Go through and remove nodes that are dead or finished from active states
+        // If an expired per-node watch is still referenced, it may be renewable
+        // so remove it from the expired list
+        for v in self.outbound_watches.values_mut() {
+            let Some(current) = v.state_mut() else {
+                continue;
+            };
+
+            // Don't drop expired per-node watches that could be renewed (still referenced by this watch)
+            for node in current.nodes() {
+                expired_pnks.remove(node);
+            }
+
+            // Remove dead and finished per-node watch nodes from this outbound watch
+            current.edit(&self.per_node_states, |editor| {
+                editor.retain_nodes(|x| !dead_pnks.contains(x) && !finished_pnks.contains(x));
+            });
+        }
+
+        // Drop finished per-node watches and unreferenced expired per-node watches
+        self.per_node_states
+            .retain(|k, _| !finished_pnks.contains(k) && !expired_pnks.contains(k));
+    }
+
+    /// Set a record up to be inspected for changed subkeys
+    pub fn enqueue_change_inspect(&mut self, record_key: TypedKey, subkeys: ValueSubkeyRangeSet) {
+        self.needs_change_inspection
+            .entry(record_key)
+            .and_modify(|x| *x = x.union(&subkeys))
+            .or_insert(subkeys);
+    }
+}

veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch.rs

@@ -0,0 +1,253 @@
+use super::*;
+
+impl_veilid_log_facility!("stor");
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub(in crate::storage_manager) struct OutboundWatch {
+    /// Current state
+    /// None means inactive/cancelled
+    state: Option<OutboundWatchState>,
+
+    /// Desired parameters
+    /// None means cancelled
+    desired: Option<OutboundWatchParameters>,
+}
+
+impl fmt::Display for OutboundWatch {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "Desired: {}\nState:\n{}\n",
+            if let Some(desired) = &self.desired {
+                desired.to_string()
+            } else {
+                "None".to_owned()
+            },
+            if let Some(state) = &self.state {
+                indent_all_by(4, state.to_string())
+            } else {
+                "None".to_owned()
+            },
+        )
+    }
+}
+
+impl OutboundWatch {
+    /// Create new outbound watch with desired parameters
+    pub fn new(desired: OutboundWatchParameters) -> Self {
+        Self {
+            state: None,
+            desired: Some(desired),
+        }
+    }
+    /// Get current watch state if it exists
+    pub fn state(&self) -> Option<&OutboundWatchState> {
+        self.state.as_ref()
+    }
+
+    /// Get mutable current watch state if it exists
+    pub fn state_mut(&mut self) -> Option<&mut OutboundWatchState> {
+        self.state.as_mut()
+    }
+
+    /// Clear current watch state
+    pub fn clear_state(&mut self) {
+        self.state = None;
+    }
+
+    /// Get or create current watch state
+    pub fn state_mut_or_create<F: FnOnce() -> OutboundWatchParameters>(
+        &mut self,
+        make_parameters: F,
+    ) -> &mut OutboundWatchState {
+        if self.state.is_none() {
+            self.state = Some(OutboundWatchState::new(make_parameters()));
+        }
+        self.state.as_mut().unwrap()
+    }
+
+    /// Get desired watch parameters if it exists
+    pub fn desired(&self) -> Option<OutboundWatchParameters> {
+        self.desired.clone()
+    }
+
+    /// Set desired watch parameters
+    pub fn set_desired(&mut self, desired: Option<OutboundWatchParameters>) {
+        self.desired = desired;
+    }
+
+    /// Check for desired state changes
+    pub fn try_expire_desired_state(&mut self, cur_ts: Timestamp) {
+        let Some(desired) = self.desired.as_ref() else {
+            // No desired parameters means this is already done
+            return;
+        };
+
+        // Check if desired parameters have expired
+        if desired.expiration_ts.as_u64() != 0 && desired.expiration_ts <= cur_ts {
+            // Expired
+            self.set_desired(None);
+            return;
+        }
+
+        // Check if the existing state has no remaining count
+        if let Some(state) = self.state.as_ref() {
+            if state.remaining_count() == 0 {
+                // No remaining count
+                self.set_desired(None);
+            }
+        }
+    }
+
+    /// Returns true if this outbound watch can be removed from the table
+    pub fn is_dead(&self) -> bool {
+        self.desired.is_none() && self.state.is_none()
+    }
+
+    /// Returns true if this outbound watch needs to be cancelled
+    pub fn needs_cancel(&self, registry: &VeilidComponentRegistry) -> bool {
+        if self.is_dead() {
+            veilid_log!(registry warn "should have checked for is_dead first");
+            return false;
+        }
+
+        // If there is no current watch then there is nothing to cancel
+        let Some(_state) = self.state.as_ref() else {
+            return false;
+        };
+
+        // If the desired parameters is None then cancel
+        let Some(_desired) = self.desired.as_ref() else {
+            return true;
+        };
+
+        false
+    }
+
+    /// Returns true if this outbound watch can be renewed
+    pub fn needs_renew(
+        &self,
+        registry: &VeilidComponentRegistry,
+        consensus_count: usize,
+        cur_ts: Timestamp,
+    ) -> bool {
+        if self.is_dead() || self.needs_cancel(registry) {
+            veilid_log!(registry warn "should have checked for is_dead and needs_cancel first");
+            return false;
+        }
+
+        // If there is no current watch then there is nothing to renew
+        let Some(state) = self.state.as_ref() else {
+            return false;
+        };
+
+        // Should have desired parameters here
+        let Some(desired) = self.desired.as_ref() else {
+            veilid_log!(registry warn "needs_cancel should have returned true");
+            return false;
+        };
+
+        // If we have a consensus, we can avoid fanout by renewing rather than reconciling
+        // but if we don't have a consensus, we should defer to fanout to try to improve it
+        if state.nodes().len() < consensus_count {
+            return false;
+        }
+
+        // If we have a consensus but need to renew because some per-node watches
+        // either expired or had their routes die, do it
+        if self.wants_per_node_watch_update(registry, state, cur_ts) {
+            return true;
+        }
+
+        // If the desired parameters have changed, then we should renew with them
+        state.params() != desired
+    }
+
+    /// Returns true if there is work to be done on getting the outbound
+    /// watch to its desired state
+    pub fn needs_reconcile(
+        &self,
+        registry: &VeilidComponentRegistry,
+        consensus_count: usize,
+        cur_ts: Timestamp,
+    ) -> bool {
+        if self.is_dead()
+            || self.needs_cancel(registry)
+            || self.needs_renew(registry, consensus_count, cur_ts)
+        {
+            veilid_log!(registry warn "should have checked for is_dead, needs_cancel, needs_renew first");
+            return false;
+        }
+
+        // If desired is none, then is_dead() or needs_cancel() should have been true
+        let Some(desired) = self.desired.as_ref() else {
+            veilid_log!(registry warn "is_dead() or needs_cancel() should have been true");
+            return false;
+        };
+
+        // If there is a desired watch but no current state, then reconcile
+        let Some(state) = self.state() else {
+            return true;
+        };
+
+        // If we are still working on getting the 'current' state to match
+        // the 'desired' state, then do the reconcile if we are within the timeframe for it
+        if state.nodes().len() < consensus_count
+            && cur_ts >= state.next_reconcile_ts().unwrap_or_default()
+        {
+            return true;
+        }
+
+        // Try to reconcile if our number of nodes currently is less than what we got from
+        // the previous reconciliation attempt
+        if let Some(last_consensus_node_count) = state.last_consensus_node_count() {
+            if state.nodes().len() < last_consensus_node_count {
+                return true;
+            }
+        }
+
+        // If we have a consensus, or are not attempting consensus at this time,
+        // but need to reconcile because some per-node watches either expired or had their routes die, do it
+        if self.wants_per_node_watch_update(registry, state, cur_ts) {
+            return true;
+        }
+
+        // If the desired parameters have changed, then we should reconcile with them
+        state.params() != desired
+    }
+
+    /// Returns true if we need to update our per-node watches due to expiration,
+    /// or if they are all dead because the route died and needs to be updated
+    fn wants_per_node_watch_update(
+        &self,
+        registry: &VeilidComponentRegistry,
+        state: &OutboundWatchState,
+        cur_ts: Timestamp,
+    ) -> bool {
+        // If the watch has per node watches that have expired, but we can extend our watch then renew.
+        // Do this only within RENEW_OUTBOUND_WATCHES_DURATION_SECS of the actual expiration.
+        // If we're looking at this after the actual expiration, don't try because the watch id will have died.
+        let renew_ts = cur_ts + TimestampDuration::new_secs(RENEW_OUTBOUND_WATCHES_DURATION_SECS);
+        if renew_ts >= state.min_expiration_ts()
+            && cur_ts < state.min_expiration_ts()
+            && (state.params().expiration_ts.as_u64() == 0
+                || renew_ts < state.params().expiration_ts)
+        {
+            return true;
+        }
+
+        let routing_table = registry.routing_table();
+        let rss = routing_table.route_spec_store();
+
+        // See if any of our per node watches have a dead value changed route
+        // if so, speculatively renew them
+        for vcr in state.value_changed_routes() {
+            if rss.get_route_id_for_key(vcr).is_none() {
+                // Route we would receive value changes on is dead
+                return true;
+            }
+        }
+
+        false
+    }
+}

veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch_parameters.rs

@@ -0,0 +1,37 @@
+use super::*;
+
+impl_veilid_log_facility!("stor");
+
+/// Requested parameters for watch
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+pub struct OutboundWatchParameters {
+    /// Requested expiration timestamp. A zero timestamp here indicates
+    /// that the watch it to be renewed indefinitely
+    pub expiration_ts: Timestamp,
+    /// How many notifications the requestor asked for
+    pub count: u32,
+    /// Subkeys requested for this watch
+    pub subkeys: ValueSubkeyRangeSet,
+    /// What key to use to perform the watch
+    pub opt_watcher: Option<KeyPair>,
+    /// What safety selection to use on the network
+    pub safety_selection: SafetySelection,
+}
+
+impl fmt::Display for OutboundWatchParameters {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "{{ expiration={}, count={}, subkeys={}, opt_watcher={}, safety_selection={:?} }}",
+            self.expiration_ts,
+            self.count,
+            self.subkeys,
+            if let Some(watcher) = &self.opt_watcher {
+                watcher.to_string()
+            } else {
+                "None".to_owned()
+            },
+            self.safety_selection
+        )
+    }
+}

veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch_state.rs

@@ -0,0 +1,188 @@
+use super::*;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub(in crate::storage_manager) struct OutboundWatchState {
+    /// Requested parameters
+    params: OutboundWatchParameters,
+    /// Nodes that have an active watch on our behalf
+    nodes: Vec<PerNodeKey>,
+    /// How many value change updates remain
+    remaining_count: u32,
+    /// The next earliest time we are willing to try to reconcile and improve the watch
+    opt_next_reconcile_ts: Option<Timestamp>,
+    /// The number of nodes we got at our last reconciliation
+    opt_last_consensus_node_count: Option<usize>,
+    /// Calculated field: minimum expiration time for all our nodes
+    min_expiration_ts: Timestamp,
+    /// Calculated field: the set of value changed routes for this watch from all per node watches
+    value_changed_routes: BTreeSet<PublicKey>,
+}
+
+impl fmt::Display for OutboundWatchState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut value_changed_routes = self
+            .value_changed_routes
+            .iter()
+            .map(|x| x.to_string())
+            .collect::<Vec<_>>();
+        value_changed_routes.sort();
+
+        write!(
+            f,
+            r#"params: {}
+nodes: [{}]
+remaining_count: {}
+opt_next_reconcile_ts: {}
+opt_consensus_node_count: {}
+min_expiration_ts: {}
+value_changed_routes: [{}]"#,
+            self.params,
+            self.nodes
+                .iter()
+                .map(|x| x.node_id.to_string())
+                .collect::<Vec<_>>()
+                .join(","),
+            self.remaining_count,
+            if let Some(next_reconcile_ts) = &self.opt_next_reconcile_ts {
+                next_reconcile_ts.to_string()
+            } else {
+                "None".to_owned()
+            },
+            if let Some(consensus_node_count) = &self.opt_last_consensus_node_count {
+                consensus_node_count.to_string()
+            } else {
+                "None".to_owned()
+            },
+            self.min_expiration_ts,
+            value_changed_routes.join(","),
+        )
+    }
+}
+
+pub(in crate::storage_manager) struct OutboundWatchStateEditor<'a> {
+    state: &'a mut OutboundWatchState,
+}
+
+impl OutboundWatchStateEditor<'_> {
+    pub fn set_params(&mut self, params: OutboundWatchParameters) {
+        self.state.params = params;
+    }
+    pub fn add_nodes<I: IntoIterator<Item = PerNodeKey>>(&mut self, nodes: I) {
+        for node in nodes {
+            if !self.state.nodes.contains(&node) {
+                self.state.nodes.push(node);
+            }
+        }
+    }
+    pub fn retain_nodes<F: FnMut(&PerNodeKey) -> bool>(&mut self, f: F) {
+        self.state.nodes.retain(f);
+    }
+    pub fn set_remaining_count(&mut self, remaining_count: u32) {
+        self.state.remaining_count = remaining_count;
+    }
+    pub fn set_next_reconcile_ts(&mut self, next_reconcile_ts: Timestamp) {
+        self.state.opt_next_reconcile_ts = Some(next_reconcile_ts);
+    }
+    pub fn update_last_consensus_node_count(&mut self) {
+        self.state.opt_last_consensus_node_count = Some(self.state.nodes().len());
+    }
+}
+
+impl OutboundWatchState {
+    pub fn new(params: OutboundWatchParameters) -> Self {
+        let remaining_count = params.count;
+        let min_expiration_ts = params.expiration_ts;
+
+        Self {
+            params,
+            nodes: vec![],
+            remaining_count,
+            opt_next_reconcile_ts: None,
+            opt_last_consensus_node_count: None,
+            min_expiration_ts,
+            value_changed_routes: BTreeSet::new(),
+        }
+    }
+
+    pub fn params(&self) -> &OutboundWatchParameters {
+        &self.params
+    }
+    pub fn nodes(&self) -> &Vec<PerNodeKey> {
+        &self.nodes
+    }
+    pub fn remaining_count(&self) -> u32 {
+        self.remaining_count
+    }
+    pub fn next_reconcile_ts(&self) -> Option<Timestamp> {
+        self.opt_next_reconcile_ts
+    }
+    pub fn last_consensus_node_count(&self) -> Option<usize> {
+        self.opt_last_consensus_node_count
+    }
+    pub fn min_expiration_ts(&self) -> Timestamp {
+        self.min_expiration_ts
+    }
+    pub fn value_changed_routes(&self) -> &BTreeSet<PublicKey> {
+        &self.value_changed_routes
+    }
+
+    /// Get the parameters we use if we're updating this state's per node watches
+    pub fn get_per_node_params(
+        &self,
+        desired: &OutboundWatchParameters,
+    ) -> OutboundWatchParameters {
+        // Change the params to update count
+        if self.params() != desired {
+            // If parameters are changing, just use the desired parameters
+            desired.clone()
+        } else {
+            // If this is a renewal of the same parameters,
+            // use the current remaining update count for the rpc
+            let mut renew_params = desired.clone();
+            renew_params.count = self.remaining_count();
+            renew_params
+        }
+    }
+
+    pub fn edit<R, F: FnOnce(&mut OutboundWatchStateEditor) -> R>(
+        &mut self,
+        per_node_state: &HashMap<PerNodeKey, PerNodeState>,
+        closure: F,
+    ) -> R {
+        let mut editor = OutboundWatchStateEditor { state: self };
+        let res = closure(&mut editor);
+
+        // Update calculated fields
+        self.min_expiration_ts = self
+            .nodes
+            .iter()
+            .map(|x| per_node_state.get(x).unwrap().expiration_ts)
+            .reduce(|a, b| a.min(b))
+            .unwrap_or(self.params.expiration_ts);
+
+        self.value_changed_routes = self
+            .nodes
+            .iter()
+            .filter_map(|x| per_node_state.get(x).unwrap().opt_value_changed_route)
+            .collect();
+
+        res
+    }
+
+    pub fn watch_node_refs(
+        &self,
+        per_node_state: &HashMap<PerNodeKey, PerNodeState>,
+    ) -> Vec<NodeRef> {
+        self.nodes
+            .iter()
+            .map(|x| {
+                per_node_state
+                    .get(x)
+                    .unwrap()
+                    .watch_node_ref
+                    .clone()
+                    .unwrap()
+            })
+            .collect()
+    }
+}

veilid-core/src/storage_manager/outbound_watch_manager/per_node_state.rs

@@ -0,0 +1,75 @@
+use super::*;
+
+impl_veilid_log_facility!("stor");
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub(in crate::storage_manager) struct PerNodeKey {
+    /// Watched record key
+    pub record_key: TypedKey,
+    /// Watching node id
+    pub node_id: TypedKey,
+}
+
+impl fmt::Display for PerNodeKey {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}@{}", self.record_key, self.node_id)
+    }
+}
+
+impl FromStr for PerNodeKey {
+    type Err = VeilidAPIError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let (rkey, nid) = s
+            .split_once('@')
+            .ok_or_else(|| VeilidAPIError::parse_error("invalid per-node key", s))?;
+        Ok(PerNodeKey {
+            record_key: TypedKey::from_str(rkey)?,
+            node_id: TypedKey::from_str(nid)?,
+        })
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub(in crate::storage_manager) struct PerNodeState {
+    /// Watch Id
+    pub watch_id: u64,
+    /// SafetySelection used to contact the node
+    pub safety_selection: SafetySelection,
+    /// What key was used to perform the watch
+    pub opt_watcher: Option<KeyPair>,
+    /// The expiration of a successful watch
+    pub expiration_ts: Timestamp,
+    /// How many value change notifications are left
+    pub count: u32,
+    /// Resolved watch node reference
+    #[serde(skip)]
+    pub watch_node_ref: Option<NodeRef>,
+    /// Which private route is responsible for receiving ValueChanged notifications
+    pub opt_value_changed_route: Option<PublicKey>,
+}
+
+impl fmt::Display for PerNodeState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{{ watch_id={}, safety_selection={:?}, opt_watcher={}, expiration_ts={}, count={}, watch_node_ref={}, opt_value_changed_route={} }}",
+            self.watch_id,
+            self.safety_selection,
+            if let Some(watcher) = &self.opt_watcher {
+                watcher.to_string()
+            } else {
+                "None".to_owned()
+            },
+            self.expiration_ts,
+            self.count,
+            if let Some(watch_node_ref) = &self.watch_node_ref {
+                watch_node_ref.to_string()
+            } else {
+                "None".to_string()
+            },
+            if let Some(value_changed_route)= &self.opt_value_changed_route {
+                value_changed_route.to_string()
+            } else {
+                "None".to_string()
+            }
+        )
+    }
+}

veilid-core/src/storage_manager/record_store/inbound_watch.rs

@@ -2,7 +2,7 @@ use super::*;
 
 /// Watch parameters used to configure a watch
 #[derive(Debug, Clone)]
-pub struct WatchParameters {
+pub struct InboundWatchParameters {
     /// The range of subkeys being watched, empty meaning full
     pub subkeys: ValueSubkeyRangeSet,
     /// When this watch will expire
@@ -18,7 +18,7 @@ pub struct WatchParameters {
 /// Watch result to return with answer
 /// Default result is cancelled/expired/inactive/rejected
 #[derive(Debug, Clone)]
-pub enum WatchResult {
+pub enum InboundWatchResult {
     /// A new watch was created
     Created {
         /// The new id of the watch
@@ -39,9 +39,9 @@ pub enum WatchResult {
 
 /// An individual watch
 #[derive(Debug, Clone)]
-pub struct Watch {
+pub struct InboundWatch {
     /// The configuration of the watch
-    pub params: WatchParameters,
+    pub params: InboundWatchParameters,
     /// A unique id per record assigned at watch creation time. Used to disambiguate a client's version of a watch
     pub id: u64,
     /// What has changed since the last update
@@ -50,13 +50,13 @@ pub struct Watch {
 
 #[derive(Debug, Default, Clone)]
 /// A record being watched for changes
-pub struct WatchList {
+pub struct InboundWatchList {
     /// The list of active watches
-    pub watches: Vec<Watch>,
+    pub watches: Vec<InboundWatch>,
 }
 
 /// How a watch gets updated when a value changes
-pub enum WatchUpdateMode {
+pub enum InboundWatchUpdateMode {
     /// Update no watchers
     NoUpdate,
     /// Update all watchers

veilid-core/src/storage_manager/record_store/limited_size.rs

@@ -125,7 +125,7 @@ impl<T: PrimInt + Unsigned + fmt::Display + fmt::Debug> LimitedSize<T> {
                     return Err(LimitError::OverLimit);
                 }
             }
-            veilid_log!(self debug "Commit ({}): {} => {}", self.description, self.value, uncommitted_value);
+            veilid_log!(self trace "Commit ({}): {} => {}", self.description, self.value, uncommitted_value);
             self.uncommitted_value = None;
             self.value = uncommitted_value;
         }
@@ -134,7 +134,7 @@ impl<T: PrimInt + Unsigned + fmt::Display + fmt::Debug> LimitedSize<T> {
 
     pub fn rollback(&mut self) -> T {
         if let Some(uv) = self.uncommitted_value.take() {
-            veilid_log!(self debug "Rollback ({}): {} (drop {})", self.description, self.value, uv);
+            veilid_log!(self trace "Rollback ({}): {} (drop {})", self.description, self.value, uv);
         }
         self.value
     }

veilid-core/src/storage_manager/record_store/mod.rs

@@ -4,6 +4,7 @@
 /// This store does not perform any validation on the schema, and all ValueRecordData passed in must have been previously validated.
 /// Uses an in-memory store for the records, backed by the TableStore. Subkey data is LRU cached and rotated out by a limits policy,
 /// and backed to the TableStore for persistence.
+mod inbound_watch;
 mod inspect_cache;
 mod keys;
 mod limited_size;
@@ -13,8 +14,9 @@ mod record;
 mod record_data;
 mod record_store_limits;
 mod remote_record_detail;
-mod watch;
 
+pub(super) use inbound_watch::*;
+pub use inbound_watch::{InboundWatchParameters, InboundWatchResult};
 pub(super) use inspect_cache::*;
 pub(super) use keys::*;
 pub(super) use limited_size::*;
@@ -23,8 +25,6 @@ pub(super) use opened_record::*;
 pub(super) use record::*;
 pub(super) use record_store_limits::*;
 pub(super) use remote_record_detail::*;
-pub(super) use watch::*;
-pub use watch::{WatchParameters, WatchResult};
 
 use super::*;
 use record_data::*;
@@ -75,7 +75,7 @@ where
     /// The list of records that have changed since last flush to disk (optimization for batched writes)
     changed_records: HashSet<RecordTableKey>,
     /// The list of records being watched for changes
-    watched_records: HashMap<RecordTableKey, WatchList>,
+    watched_records: HashMap<RecordTableKey, InboundWatchList>,
     /// The list of watched records that have changed values since last notification
     changed_watched_values: HashSet<RecordTableKey>,
     /// A mutex to ensure we handle this concurrently
@@ -680,12 +680,12 @@ where
         &mut self,
         key: TypedKey,
         subkey: ValueSubkey,
-        watch_update_mode: WatchUpdateMode,
+        watch_update_mode: InboundWatchUpdateMode,
     ) {
         let (do_update, opt_ignore_target) = match watch_update_mode {
-            WatchUpdateMode::NoUpdate => (false, None),
+            InboundWatchUpdateMode::NoUpdate => (false, None),
-            WatchUpdateMode::UpdateAll => (true, None),
+            InboundWatchUpdateMode::UpdateAll => (true, None),
-            WatchUpdateMode::ExcludeTarget(target) => (true, Some(target)),
+            InboundWatchUpdateMode::ExcludeTarget(target) => (true, Some(target)),
         };
         if !do_update {
             return;
@@ -720,7 +720,7 @@ where
         key: TypedKey,
         subkey: ValueSubkey,
         signed_value_data: Arc<SignedValueData>,
-        watch_update_mode: WatchUpdateMode,
+        watch_update_mode: InboundWatchUpdateMode,
     ) -> VeilidAPIResult<()> {
         // Check size limit for data
         if signed_value_data.value_data().data().len() > self.limits.max_subkey_size {
@@ -902,9 +902,9 @@ where
     pub async fn _change_existing_watch(
         &mut self,
         key: TypedKey,
-        params: WatchParameters,
+        params: InboundWatchParameters,
         watch_id: u64,
-    ) -> VeilidAPIResult<WatchResult> {
+    ) -> VeilidAPIResult<InboundWatchResult> {
         if params.count == 0 {
             apibail_internal!("cancel watch should not have gotten here");
         }
@@ -915,7 +915,7 @@ where
         let rtk = RecordTableKey { key };
         let Some(watch_list) = self.watched_records.get_mut(&rtk) else {
             // No watches, nothing to change
-            return Ok(WatchResult::Rejected);
+            return Ok(InboundWatchResult::Rejected);
         };
 
         // Check each watch to see if we have an exact match for the id to change
@@ -925,23 +925,23 @@ where
             if w.id == watch_id && w.params.watcher == params.watcher {
                 // Updating an existing watch
                 w.params = params;
-                return Ok(WatchResult::Changed {
+                return Ok(InboundWatchResult::Changed {
                     expiration: w.params.expiration,
                 });
             }
         }
 
         // No existing watch found
-        Ok(WatchResult::Rejected)
+        Ok(InboundWatchResult::Rejected)
     }
 
     #[instrument(level = "trace", target = "stor", skip_all, err)]
     pub async fn _create_new_watch(
         &mut self,
         key: TypedKey,
-        params: WatchParameters,
+        params: InboundWatchParameters,
         member_check: Box<dyn Fn(PublicKey) -> bool + Send>,
-    ) -> VeilidAPIResult<WatchResult> {
+    ) -> VeilidAPIResult<InboundWatchResult> {
         // Generate a record-unique watch id > 0
         let rtk = RecordTableKey { key };
         let mut id = 0;
@@ -1001,7 +1001,7 @@ where
         // For anonymous, no more than one watch per target per record
         if target_watch_count > 0 {
             // Too many watches
-            return Ok(WatchResult::Rejected);
+            return Ok(InboundWatchResult::Rejected);
         }
 
         // Check watch table for limits
@@ -1011,18 +1011,18 @@ where
             self.limits.public_watch_limit
         };
         if watch_count >= watch_limit {
-            return Ok(WatchResult::Rejected);
+            return Ok(InboundWatchResult::Rejected);
         }
 
         // Ok this is an acceptable new watch, add it
         let watch_list = self.watched_records.entry(rtk).or_default();
         let expiration = params.expiration;
-        watch_list.watches.push(Watch {
+        watch_list.watches.push(InboundWatch {
             params,
             id,
             changed: ValueSubkeyRangeSet::new(),
         });
-        Ok(WatchResult::Created { id, expiration })
+        Ok(InboundWatchResult::Created { id, expiration })
     }
 
     /// Add or update an inbound record watch for changes
@@ -1030,17 +1030,17 @@ where
     pub async fn watch_record(
         &mut self,
         key: TypedKey,
-        mut params: WatchParameters,
+        mut params: InboundWatchParameters,
         opt_watch_id: Option<u64>,
-    ) -> VeilidAPIResult<WatchResult> {
+    ) -> VeilidAPIResult<InboundWatchResult> {
         // If count is zero then we're cancelling a watch completely
         if params.count == 0 {
             if let Some(watch_id) = opt_watch_id {
                 let cancelled = self.cancel_watch(key, watch_id, params.watcher).await?;
                 if cancelled {
-                    return Ok(WatchResult::Cancelled);
+                    return Ok(InboundWatchResult::Cancelled);
                 }
-                return Ok(WatchResult::Rejected);
+                return Ok(InboundWatchResult::Rejected);
             }
             apibail_internal!("shouldn't have let a None watch id get here");
         }
@@ -1058,10 +1058,10 @@ where
             if let Some(watch_id) = opt_watch_id {
                 let cancelled = self.cancel_watch(key, watch_id, params.watcher).await?;
                 if cancelled {
-                    return Ok(WatchResult::Cancelled);
+                    return Ok(InboundWatchResult::Cancelled);
                 }
             }
-            return Ok(WatchResult::Rejected);
+            return Ok(InboundWatchResult::Rejected);
         }
 
         // Make a closure to check for member vs anonymous
@@ -1071,7 +1071,7 @@ where
             Box::new(move |watcher| owner == watcher || schema.is_member(&watcher))
         }) else {
             // Record not found
-            return Ok(WatchResult::Rejected);
+            return Ok(InboundWatchResult::Rejected);
         };
 
         // Create or update depending on if a watch id is specified or not
@@ -1128,8 +1128,8 @@ where
     pub fn move_watches(
         &mut self,
         key: TypedKey,
-        in_watch: Option<(WatchList, bool)>,
+        in_watch: Option<(InboundWatchList, bool)>,
-    ) -> Option<(WatchList, bool)> {
+    ) -> Option<(InboundWatchList, bool)> {
         let rtk = RecordTableKey { key };
         let out = self.watched_records.remove(&rtk);
         if let Some(in_watch) = in_watch {

veilid-core/src/storage_manager/record_store/opened_record.rs

@@ -1,21 +1,5 @@
 use super::*;
 
-#[derive(Clone, Debug)]
-pub(in crate::storage_manager) struct ActiveWatch {
-    /// The watch id returned from the watch node
-    pub id: u64,
-    /// The expiration of a successful watch
-    pub expiration_ts: Timestamp,
-    /// Which node accepted the watch
-    pub watch_node: NodeRef,
-    /// Which private route is responsible for receiving ValueChanged notifications
-    pub opt_value_changed_route: Option<PublicKey>,
-    /// Which subkeys we are watching
-    pub subkeys: ValueSubkeyRangeSet,
-    /// How many notifications are left
-    pub count: u32,
-}
-
 /// The state associated with a local record when it is opened
 /// This is not serialized to storage as it is ephemeral for the lifetime of the opened record
 #[derive(Clone, Debug, Default)]
@@ -27,9 +11,6 @@ pub(in crate::storage_manager) struct OpenedRecord {
 
     /// The safety selection in current use
     safety_selection: SafetySelection,
-
-    /// Active watch we have on this record
-    active_watch: Option<ActiveWatch>,
 }
 
 impl OpenedRecord {
@@ -37,7 +18,6 @@ impl OpenedRecord {
         Self {
             writer,
             safety_selection,
-            active_watch: None,
         }
     }
 
@@ -54,16 +34,4 @@ impl OpenedRecord {
     pub fn set_safety_selection(&mut self, safety_selection: SafetySelection) {
         self.safety_selection = safety_selection;
     }
-
-    pub fn set_active_watch(&mut self, active_watch: ActiveWatch) {
-        self.active_watch = Some(active_watch);
-    }
-
-    pub fn clear_active_watch(&mut self) {
-        self.active_watch = None;
-    }
-
-    pub fn active_watch(&self) -> Option<ActiveWatch> {
-        self.active_watch.clone()
-    }
 }

veilid-core/src/storage_manager/set_value.rs

@@ -6,8 +6,6 @@ impl_veilid_log_facility!("stor");
 struct OutboundSetValueContext {
     /// The latest value of the subkey, may be the value passed in
     pub value: Arc<SignedValueData>,
-    /// The nodes that have set the value so far (up to the consensus count)
-    pub value_nodes: Vec<NodeRef>,
     /// The number of non-sets since the last set we have received
     pub missed_since_last_set: usize,
     /// The parsed schema from the descriptor if we have one
@@ -39,11 +37,9 @@ impl StorageManager {
         let routing_domain = RoutingDomain::PublicInternet;
 
         // Get the DHT parameters for 'SetValue'
-        let (key_count, get_consensus_count, set_consensus_count, fanout, timeout_us) =
+        let (key_count, consensus_count, fanout, timeout_us) = self.config().with(|c| {
-            self.config().with(|c| {
             (
                 c.network.dht.max_find_node_count as usize,
-                    c.network.dht.get_value_count as usize,
                 c.network.dht.set_value_count as usize,
                 c.network.dht.set_value_fanout as usize,
                 TimestampDuration::from(ms_to_us(c.network.dht.set_value_timeout_ms)),
@@ -71,10 +67,9 @@ impl StorageManager {
         let schema = descriptor.schema()?;
         let context = Arc::new(Mutex::new(OutboundSetValueContext {
             value,
-            value_nodes: vec![],
             missed_since_last_set: 0,
             schema,
-            send_partial_update: false,
+            send_partial_update: true,
         }));
 
         // Routine to call to generate fanout
@@ -82,7 +77,8 @@ impl StorageManager {
             let context = context.clone();
             let registry = self.registry();
 
-            Arc::new(move |next_node: NodeRef| {
+            Arc::new(
+                move |next_node: NodeRef| -> PinBoxFutureStatic<FanoutCallResult> {
                     let registry = registry.clone();
                     let context = context.clone();
                     let descriptor = descriptor.clone();
@@ -98,7 +94,7 @@ impl StorageManager {
                         };
 
                         // send across the wire
-                    let sva = network_result_try!(
+                        let sva = match
                             rpc_processor
                                 .rpc_call_set_value(
                                     Destination::direct(next_node.routing_domain_filtered(routing_domain))
@@ -109,8 +105,18 @@ impl StorageManager {
                                     (*descriptor).clone(),
                                     send_descriptor,
                                 )
-                            .await?
+                                .await? {
-                    );
+                            NetworkResult::Timeout => {
+                                return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Timeout});
+                            }
+                            NetworkResult::ServiceUnavailable(_) |
+                            NetworkResult::NoConnection(_)  |
+                            NetworkResult::AlreadyExists(_) |
+                            NetworkResult::InvalidMessage(_) => {
+                                return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
+                            }
+                            NetworkResult::Value(v) => v
+                        };
 
                         // If the node was close enough to possibly set the value
                         let mut ctx = context.lock();
@@ -119,22 +125,17 @@ impl StorageManager {
 
                             // Return peers if we have some
                             veilid_log!(registry debug target:"network_result", "SetValue missed: {}, fanout call returned peers {}", ctx.missed_since_last_set, sva.answer.peers.len());
-                        return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers}));
+                            return Ok(FanoutCallOutput{peer_info_list:sva.answer.peers, disposition: FanoutCallDisposition::Rejected});
                         }
 
-                    // See if we got a value back
+                        // See if we got a newer value back
                         let Some(value) = sva.answer.value else {
                             // No newer value was found and returned, so increase our consensus count
-                        ctx.value_nodes.push(next_node);
                             ctx.missed_since_last_set = 0;
-                        // Send an update since it was set
-                        if ctx.value_nodes.len() == 1 {
-                            ctx.send_partial_update = true;
-                        }
 
                             // Return peers if we have some
                             veilid_log!(registry debug target:"network_result", "SetValue returned no value, fanout call returned peers {}", sva.answer.peers.len());
-                        return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers}));
+                            return Ok(FanoutCallOutput{peer_info_list:sva.answer.peers, disposition: FanoutCallDisposition::Accepted});
                         };
 
                         // Keep the value if we got one and it is newer and it passes schema validation
@@ -147,24 +148,12 @@ impl StorageManager {
                             value.value_data(),
                         ) {
                             // Validation failed, ignore this value and pretend we never saw this node
-                        return Ok(NetworkResult::invalid_message(format!(
+                            return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
-                            "Schema validation failed on subkey {}",
-                            subkey
-                        )));
                         }
 
                         // If we got a value back it should be different than the one we are setting
-                    // But in the case of a benign bug, we can just move to the next node
                         if ctx.value.value_data() == value.value_data() {
-                        ctx.value_nodes.push(next_node);
+                            return Ok(FanoutCallOutput{peer_info_list:sva.answer.peers, disposition: FanoutCallDisposition::Invalid});
-                        ctx.missed_since_last_set = 0;
-
-                        // Send an update since it was set
-                        if ctx.value_nodes.len() == 1 {
-                            ctx.send_partial_update = true;
-                        }
-
-                        return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers}));
                         }
 
                         // We have a prior value, ensure this is a newer sequence number
@@ -174,21 +163,21 @@ impl StorageManager {
                             // If the sequence number is older node should have not returned a value here.
                             // Skip this node and its closer list because it is misbehaving
                             // Ignore this value and pretend we never saw this node
-                        return Ok(NetworkResult::invalid_message("Sequence number is older"));
+                            return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
                         }
 
                         // If the sequence number is greater or equal, keep it
                         // even if the sequence number is the same, accept all conflicts in an attempt to resolve them
                         ctx.value = Arc::new(value);
                         // One node has shown us this value so far
-                    ctx.value_nodes = vec![next_node];
                         ctx.missed_since_last_set = 0;
                         // Send an update since the value changed
                         ctx.send_partial_update = true;
 
-                    Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers}))
+                        Ok(FanoutCallOutput{peer_info_list:sva.answer.peers, disposition: FanoutCallDisposition::AcceptedNewerRestart})
                     }.instrument(tracing::trace_span!("fanout call_routine"))) as PinBoxFuture<FanoutCallResult>
-            })
+                },
+            )
         };
 
         // Routine to call to check if we're done at each step
@@ -196,43 +185,37 @@ impl StorageManager {
             let context = context.clone();
             let out_tx = out_tx.clone();
             let registry = self.registry();
-            Arc::new(move |_closest_nodes: &[NodeRef]| {
+            Arc::new(move |fanout_result: &FanoutResult| -> bool {
                 let mut ctx = context.lock();
 
-                // send partial update if desired
+                match fanout_result.kind {
-                if ctx.send_partial_update {
+                    FanoutResultKind::Incomplete => {
+                        // Send partial update if desired, if we've gotten at least consensus node
+                        if ctx.send_partial_update && !fanout_result.consensus_nodes.is_empty() {
                             ctx.send_partial_update = false;
 
-                    // return partial result
+                            // Return partial result
-                    let fanout_result = FanoutResult {
-                        kind: FanoutResultKind::Partial,
-                        value_nodes: ctx.value_nodes.clone(),
-                    };
                             let out = OutboundSetValueResult {
-                        fanout_result,
+                                fanout_result: fanout_result.clone(),
                                 signed_value_data: ctx.value.clone(),
                             };
                             veilid_log!(registry debug "Sending partial SetValue result: {:?}", out);
-
                             if let Err(e) = out_tx.send(Ok(out)) {
                                 veilid_log!(registry debug "Sending partial SetValue result failed: {}", e);
                             }
                         }
-
+                        // Keep going
-                // If we have reached set consensus (the max consensus we care about), return done
+                        false
-                if ctx.value_nodes.len() >= set_consensus_count {
+                    }
-                    return Some(());
+                    FanoutResultKind::Timeout | FanoutResultKind::Exhausted => {
+                        // Signal we're done
+                        true
+                    }
+                    FanoutResultKind::Consensus => {
+                        // Signal we're done
+                        true
                     }
-
-                // If we have missed get_consensus count (the minimum consensus we care about) or more since our last set, return done
-                // This keeps the traversal from searching too many nodes when we aren't converging
-                // Only do this if we have gotten at least the get_consensus (the minimum consensus we care about)
-                if ctx.value_nodes.len() >= get_consensus_count
-                    && ctx.missed_since_last_set >= get_consensus_count
-                {
-                    return Some(());
                 }
-                None
             })
         };
 
@@ -248,21 +231,16 @@ impl StorageManager {
                         key,
                         key_count,
                         fanout,
+                        consensus_count,
                         timeout_us,
                         capability_fanout_node_info_filter(vec![CAP_DHT]),
                         call_routine,
                         check_done,
                     );
 
-                    let kind = match fanout_call.run(init_fanout_queue).await {
+                    let fanout_result = match fanout_call.run(init_fanout_queue).await {
-                        // If we don't finish in the timeout (too much time passed checking for consensus)
+                        Ok(v) => v,
-                        TimeoutOr::Timeout => FanoutResultKind::Timeout,
+                        Err(e) => {
-                        // If we finished with or without consensus (enough nodes returning the same value)
-                        TimeoutOr::Value(Ok(Some(()))) => FanoutResultKind::Finished,
-                        // If we ran out of nodes before getting consensus)
-                        TimeoutOr::Value(Ok(None)) => FanoutResultKind::Exhausted,
-                        // Failed
-                        TimeoutOr::Value(Err(e)) => {
                             // If we finished with an error, return that
                             veilid_log!(registry debug "SetValue fanout error: {}", e);
                             if let Err(e) = out_tx.send(Err(e.into())) {
@@ -272,19 +250,20 @@ impl StorageManager {
                         }
                     };
 
-                    let ctx = context.lock();
+                    veilid_log!(registry debug "SetValue Fanout: {:#}", fanout_result);
-                    let fanout_result = FanoutResult {
-                        kind,
-                        value_nodes: ctx.value_nodes.clone(),
-                    };
-                    veilid_log!(registry debug "SetValue Fanout: {:?}", fanout_result);
 
-                    if let Err(e) = out_tx.send(Ok(OutboundSetValueResult {
+                    let out = {
+                        let ctx = context.lock();
+                        OutboundSetValueResult {
                             fanout_result,
                             signed_value_data: ctx.value.clone(),
-                    })) {
+                        }
+                    };
+
+                    if let Err(e) = out_tx.send(Ok(out)) {
                         veilid_log!(registry debug "Sending SetValue result failed: {}", e);
                     }
+
                 }
                 .instrument(tracing::trace_span!("outbound_set_value fanout routine")),
             ),
@@ -321,19 +300,19 @@ impl StorageManager {
                                 return false;
                             }
                         };
-                        let is_partial = result.fanout_result.kind.is_partial();
+                        let is_incomplete = result.fanout_result.kind.is_incomplete();
                         let lvd = last_value_data.lock().clone();
                         let value_data = match this.process_outbound_set_value_result(key, subkey, lvd, safety_selection, result).await {
                             Ok(Some(v)) => v,
                             Ok(None) => {
-                                return is_partial;
+                                return is_incomplete;
                             }
                             Err(e) => {
                                 veilid_log!(registry debug "Deferred fanout error: {}", e);
                                 return false;
                             }
                         };
-                        if is_partial {
+                        if is_incomplete {
                             // If more partial results show up, don't send an update until we're done
                             return true;
                         }
@@ -364,27 +343,34 @@ impl StorageManager {
     #[instrument(level = "trace", target = "stor", skip_all, err)]
     pub(super) async fn process_outbound_set_value_result(
         &self,
-        key: TypedKey,
+        record_key: TypedKey,
         subkey: ValueSubkey,
         last_value_data: ValueData,
         safety_selection: SafetySelection,
         result: set_value::OutboundSetValueResult,
     ) -> Result<Option<ValueData>, VeilidAPIError> {
+        // Get cryptosystem
+        let crypto = self.crypto();
+        let Some(vcrypto) = crypto.get(record_key.kind) else {
+            apibail_generic!("unsupported cryptosystem");
+        };
+
         // Regain the lock after network access
         let mut inner = self.inner.lock().await;
 
         // Report on fanout result offline
-        let was_offline = self.check_fanout_set_offline(key, subkey, &result.fanout_result);
+        let was_offline = self.check_fanout_set_offline(record_key, subkey, &result.fanout_result);
         if was_offline {
             // Failed to write, try again later
-            Self::add_offline_subkey_write_inner(&mut inner, key, subkey, safety_selection);
+            Self::add_offline_subkey_write_inner(&mut inner, record_key, subkey, safety_selection);
         }
 
         // Keep the list of nodes that returned a value for later reference
         Self::process_fanout_results_inner(
             &mut inner,
-            key,
+            &vcrypto,
-            core::iter::once((subkey, &result.fanout_result)),
+            record_key,
+            core::iter::once((ValueSubkeyRangeSet::single(subkey), result.fanout_result)),
             true,
             self.config()
                 .with(|c| c.network.dht.set_value_count as usize),
@@ -396,10 +382,10 @@ impl StorageManager {
 
             Self::handle_set_local_value_inner(
                 &mut inner,
-                key,
+                record_key,
                 subkey,
                 result.signed_value_data.clone(),
-                WatchUpdateMode::UpdateAll,
+                InboundWatchUpdateMode::UpdateAll,
             )
             .await?;
 
@@ -500,7 +486,7 @@ impl StorageManager {
                 key,
                 subkey,
                 value,
-                WatchUpdateMode::ExcludeTarget(target),
+                InboundWatchUpdateMode::ExcludeTarget(target),
             )
             .await
         } else {
@@ -510,7 +496,7 @@ impl StorageManager {
                 subkey,
                 value,
                 actual_descriptor,
-                WatchUpdateMode::ExcludeTarget(target),
+                InboundWatchUpdateMode::ExcludeTarget(target),
             )
             .await
         };

veilid-core/src/storage_manager/tasks/check_active_watches.rs

@@ -1,67 +0,0 @@
-use super::*;
-
-impl StorageManager {
-    // Check if client-side watches on opened records either have dead nodes or if the watch has expired
-    #[instrument(level = "trace", target = "stor", skip_all, err)]
-    pub(super) async fn check_active_watches_task_routine(
-        &self,
-        _stop_token: StopToken,
-        _last_ts: Timestamp,
-        _cur_ts: Timestamp,
-    ) -> EyreResult<()> {
-        {
-            let mut inner = self.inner.lock().await;
-
-            let routing_table = self.routing_table();
-            let update_callback = self.update_callback();
-
-            let cur_ts = Timestamp::now();
-            for (k, v) in inner.opened_records.iter_mut() {
-                // If no active watch, then skip this
-                let Some(active_watch) = v.active_watch() else {
-                    continue;
-                };
-
-                // See if the active watch's node is dead
-                let mut is_dead = false;
-                if !active_watch.watch_node.state(cur_ts).is_alive() {
-                    // Watched node is dead
-                    is_dead = true;
-                }
-
-                // See if the private route we're using is dead
-                if !is_dead {
-                    if let Some(value_changed_route) = active_watch.opt_value_changed_route {
-                        if routing_table
-                            .route_spec_store()
-                            .get_route_id_for_key(&value_changed_route)
-                            .is_none()
-                        {
-                            // Route we would receive value changes on is dead
-                            is_dead = true;
-                        }
-                    }
-                }
-                // See if the watch is expired
-                if !is_dead && active_watch.expiration_ts <= cur_ts {
-                    // Watch has expired
-                    is_dead = true;
-                }
-
-                if is_dead {
-                    v.clear_active_watch();
-
-                    // Send valuechange with dead count and no subkeys
-                    update_callback(VeilidUpdate::ValueChange(Box::new(VeilidValueChange {
-                        key: *k,
-                        subkeys: ValueSubkeyRangeSet::new(),
-                        count: 0,
-                        value: None,
-                    })));
-                }
-            }
-        }
-
-        Ok(())
-    }
-}

veilid-core/src/storage_manager/tasks/check_inbound_watches.rs

@@ -3,7 +3,7 @@ use super::*;
 impl StorageManager {
     // Check if server-side watches have expired
     #[instrument(level = "trace", target = "stor", skip_all, err)]
-    pub(super) async fn check_watched_records_task_routine(
+    pub(super) async fn check_inbound_watches_task_routine(
         &self,
         _stop_token: StopToken,
         _last_ts: Timestamp,

veilid-core/src/storage_manager/tasks/check_outbound_watches.rs

@@ -0,0 +1,37 @@
+use super::*;
+
+impl StorageManager {
+    // Check if client-side watches on opened records either have dead nodes or if the watch has expired
+    //#[instrument(level = "trace", target = "stor", skip_all, err)]
+    pub(super) async fn check_outbound_watches_task_routine(
+        &self,
+        _stop_token: StopToken,
+        _last_ts: Timestamp,
+        _cur_ts: Timestamp,
+    ) -> EyreResult<()> {
+        let inner = &mut *self.inner.lock().await;
+
+        let cur_ts = Timestamp::now();
+
+        // Update per-node watch states
+        // Desired state updates are performed by get_next_outbound_watch_operation
+        inner.outbound_watch_manager.update_per_node_states(cur_ts);
+
+        // Iterate all outbound watches and determine what work needs doing if any
+        for (k, v) in &mut inner.outbound_watch_manager.outbound_watches {
+            // Get next work on watch and queue it if we have something to do
+            if let Some(op_fut) = self.get_next_outbound_watch_operation(*k, None, cur_ts, v) {
+                self.background_operation_processor.add_future(op_fut);
+            };
+        }
+
+        // Iterate all queued change inspections and do them
+        for (k, v) in inner.outbound_watch_manager.needs_change_inspection.drain() {
+            // Get next work on watch and queue it if we have something to do
+            let op_fut = self.get_change_inspection_operation(k, v);
+            self.background_operation_processor.add_future(op_fut);
+        }
+
+        Ok(())
+    }
+}

veilid-core/src/storage_manager/tasks/mod.rs

@@ -1,7 +1,8 @@
-pub mod check_active_watches;
+pub mod check_inbound_watches;
-pub mod check_watched_records;
+pub mod check_outbound_watches;
 pub mod flush_record_stores;
 pub mod offline_subkey_writes;
+pub mod save_metadata;
 pub mod send_value_changes;
 
 use super::*;
@@ -16,7 +17,9 @@ impl StorageManager {
             flush_record_stores_task,
             flush_record_stores_task_routine
         );
-
+        // Set save metadata task
+        veilid_log!(self debug "starting save metadata task");
+        impl_setup_task!(self, Self, save_metadata_task, save_metadata_task_routine);
         // Set offline subkey writes tick task
         veilid_log!(self debug "starting offline subkey writes task");
         impl_setup_task!(
@@ -40,8 +43,8 @@ impl StorageManager {
         impl_setup_task!(
             self,
             Self,
-            check_active_watches_task,
+            check_outbound_watches_task,
-            check_active_watches_task_routine
+            check_outbound_watches_task_routine
         );
 
         // Set check watched records tick task
@@ -49,8 +52,8 @@ impl StorageManager {
         impl_setup_task!(
             self,
             Self,
-            check_watched_records_task,
+            check_inbound_watches_task,
-            check_watched_records_task_routine
+            check_inbound_watches_task_routine
         );
     }
 
@@ -59,11 +62,14 @@ impl StorageManager {
         // Run the flush stores task
         self.flush_record_stores_task.tick().await?;
 
+        // Run the flush stores task
+        self.save_metadata_task.tick().await?;
+
         // Check active watches
-        self.check_active_watches_task.tick().await?;
+        self.check_outbound_watches_task.tick().await?;
 
         // Check watched records
-        self.check_watched_records_task.tick().await?;
+        self.check_inbound_watches_task.tick().await?;
 
         // Run online-only tasks
         if self.dht_is_online() {
@@ -81,11 +87,11 @@ impl StorageManager {
     #[instrument(level = "trace", target = "stor", skip_all)]
     pub(super) async fn cancel_tasks(&self) {
         veilid_log!(self debug "stopping check watched records task");
-        if let Err(e) = self.check_watched_records_task.stop().await {
+        if let Err(e) = self.check_inbound_watches_task.stop().await {
             veilid_log!(self warn "check_watched_records_task not stopped: {}", e);
         }
         veilid_log!(self debug "stopping check active watches task");
-        if let Err(e) = self.check_active_watches_task.stop().await {
+        if let Err(e) = self.check_outbound_watches_task.stop().await {
             veilid_log!(self warn "check_active_watches_task not stopped: {}", e);
         }
         veilid_log!(self debug "stopping send value changes task");

veilid-core/src/storage_manager/tasks/offline_subkey_writes.rs

@@ -28,9 +28,9 @@ struct WorkItem {
 
 #[derive(Debug)]
 struct WorkItemResult {
-    key: TypedKey,
+    record_key: TypedKey,
     written_subkeys: ValueSubkeyRangeSet,
-    fanout_results: Vec<(ValueSubkey, FanoutResult)>,
+    fanout_results: Vec<(ValueSubkeyRangeSet, FanoutResult)>,
 }
 
 impl StorageManager {
@@ -74,7 +74,7 @@ impl StorageManager {
                 while let Ok(Ok(res)) = res_rx.recv_async().timeout_at(stop_token.clone()).await {
                     match res {
                         Ok(result) => {
-                            let partial = result.fanout_result.kind.is_partial();
+                            let partial = result.fanout_result.kind.is_incomplete();
                             // Skip partial results in offline subkey write mode
                             if partial {
                                 continue;
@@ -90,7 +90,7 @@ impl StorageManager {
                                     key,
                                     subkey,
                                     result.signed_value_data.clone(),
-                                    WatchUpdateMode::UpdateAll,
+                                    InboundWatchUpdateMode::UpdateAll,
                                 )
                                 .await?;
                             }
@@ -121,7 +121,7 @@ impl StorageManager {
         work_item: WorkItem,
     ) -> EyreResult<WorkItemResult> {
         let mut written_subkeys = ValueSubkeyRangeSet::new();
-        let mut fanout_results = Vec::<(ValueSubkey, FanoutResult)>::new();
+        let mut fanout_results = Vec::<(ValueSubkeyRangeSet, FanoutResult)>::new();
 
         for subkey in work_item.subkeys.iter() {
             if poll!(stop_token.clone()).is_ready() {
@@ -155,11 +155,11 @@ impl StorageManager {
             if !was_offline {
                 written_subkeys.insert(subkey);
             }
-            fanout_results.push((subkey, result.fanout_result));
+            fanout_results.push((ValueSubkeyRangeSet::single(subkey), result.fanout_result));
         }
 
         Ok(WorkItemResult {
-            key: work_item.key,
+            record_key: work_item.key,
             written_subkeys,
             fanout_results,
         })
@@ -192,7 +192,7 @@ impl StorageManager {
         veilid_log!(self debug "Offline write result: {:?}", result);
 
         // Get the offline subkey write record
-        match inner.offline_subkey_writes.entry(result.key) {
+        match inner.offline_subkey_writes.entry(result.record_key) {
             std::collections::hash_map::Entry::Occupied(mut o) => {
                 let finished = {
                     let osw = o.get_mut();
@@ -208,20 +208,24 @@ impl StorageManager {
                     osw.subkeys.is_empty()
                 };
                 if finished {
-                    veilid_log!(self debug "Offline write finished key {}", result.key);
+                    veilid_log!(self debug "Offline write finished key {}", result.record_key);
                     o.remove();
                 }
             }
             std::collections::hash_map::Entry::Vacant(_) => {
-                veilid_log!(self warn "offline write work items should always be on offline_subkey_writes entries that exist: ignoring key {}", result.key);
+                veilid_log!(self warn "offline write work items should always be on offline_subkey_writes entries that exist: ignoring key {}", result.record_key);
             }
         }
 
         // Keep the list of nodes that returned a value for later reference
+        let crypto = self.crypto();
+        let vcrypto = crypto.get(result.record_key.kind).unwrap();
+
         Self::process_fanout_results_inner(
             &mut inner,
-            result.key,
+            &vcrypto,
-            result.fanout_results.iter().map(|x| (x.0, &x.1)),
+            result.record_key,
+            result.fanout_results.into_iter().map(|x| (x.0, x.1)),
             true,
             consensus_count,
         );

veilid-core/src/storage_manager/tasks/save_metadata.rs

@@ -0,0 +1,16 @@
+use super::*;
+
+impl StorageManager {
+    // Save metadata to disk
+    #[instrument(level = "trace", target = "stor", skip_all, err)]
+    pub(super) async fn save_metadata_task_routine(
+        &self,
+        _stop_token: StopToken,
+        _last_ts: Timestamp,
+        _cur_ts: Timestamp,
+    ) -> EyreResult<()> {
+        let mut inner = self.inner.lock().await;
+        self.save_metadata_inner(&mut inner).await?;
+        Ok(())
+    }
+}

veilid-core/src/veilid_api/debug.rs

@@ -1474,6 +1474,11 @@ impl VeilidAPI {
                 out += &storage_manager.debug_opened_records().await;
                 out
             }
+            "watched" => {
+                let mut out = "Watched Records:\n".to_string();
+                out += &storage_manager.debug_watched_records().await;
+                out
+            }
             "offline" => {
                 let mut out = "Offline Records:\n".to_string();
                 out += &storage_manager.debug_offline_records().await;
@@ -1489,6 +1494,11 @@ impl VeilidAPI {
         let registry = self.core_context()?.registry();
         let storage_manager = registry.storage_manager();
 
+        self.with_debug_cache(|dc| {
+            dc.opened_record_contexts.clear();
+        });
+        storage_manager.close_all_records().await?;
+
         let scope = get_debug_argument_at(&args, 1, "debug_record_purge", "scope", get_string)?;
         let bytes = get_debug_argument_at(&args, 2, "debug_record_purge", "bytes", get_number).ok();
         let out = match scope.as_str() {
@@ -1786,13 +1796,14 @@ impl VeilidAPI {
             get_subkeys,
         )
         .ok()
+        .map(Some)
         .unwrap_or_else(|| {
             rest_defaults = true;
-            Default::default()
+            None
         });
 
-        let expiration = if rest_defaults {
+        let opt_expiration = if rest_defaults {
-            Default::default()
+            None
         } else {
             get_debug_argument_at(
                 &args,
@@ -1802,14 +1813,20 @@ impl VeilidAPI {
                 parse_duration,
             )
             .ok()
-            .map(|dur| dur + get_timestamp())
+            .map(|dur| {
+                if dur == 0 {
+                    None
+                } else {
+                    Some(Timestamp::new(dur + get_timestamp()))
+                }
+            })
             .unwrap_or_else(|| {
                 rest_defaults = true;
-                Default::default()
+                None
             })
         };
         let count = if rest_defaults {
-            u32::MAX
+            None
         } else {
             get_debug_argument_at(
                 &args,
@@ -1819,15 +1836,16 @@ impl VeilidAPI {
                 get_number,
             )
             .ok()
+            .map(Some)
             .unwrap_or_else(|| {
                 rest_defaults = true;
-                u32::MAX
+                Some(u32::MAX)
             })
         };
 
         // Do a record watch
-        let ts = match rc
+        let active = match rc
-            .watch_dht_values(key, subkeys, Timestamp::new(expiration), count)
+            .watch_dht_values(key, subkeys, opt_expiration, count)
             .await
         {
             Err(e) => {
@@ -1835,10 +1853,10 @@ impl VeilidAPI {
             }
             Ok(v) => v,
         };
-        if ts.as_u64() == 0 {
+        if !active {
             return Ok("Failed to watch value".to_owned());
         }
-        Ok(format!("Success: expiration={:?}", display_ts(ts.as_u64())))
+        Ok("Success".to_owned())
     }
 
     async fn debug_record_cancel(&self, args: Vec<String>) -> VeilidAPIResult<String> {
@@ -1858,8 +1876,7 @@ impl VeilidAPI {
             "subkeys",
             get_subkeys,
         )
-        .ok()
+        .ok();
-        .unwrap_or_default();
 
         // Do a record watch cancel
         let still_active = match rc.cancel_dht_watch(key, subkeys).await {
@@ -1906,7 +1923,10 @@ impl VeilidAPI {
             })
         };
 
-        let subkeys = get_debug_argument_at(
+        let subkeys = if rest_defaults {
+            None
+        } else {
+            get_debug_argument_at(
                 &args,
                 2 + opt_arg_add,
                 "debug_record_inspect",
@@ -1914,10 +1934,7 @@ impl VeilidAPI {
                 get_subkeys,
             )
             .ok()
-        .unwrap_or_else(|| {
+        };
-            rest_defaults = true;
-            Default::default()
-        });
 
         // Do a record inspect
         let report = match rc.inspect_dht_record(key, subkeys, scope).await {
@@ -2115,7 +2132,7 @@ RPC Operations:
     appreply [#id] <data> - Reply to an 'App Call' RPC received by this node
 
 DHT Operations:
-    record list <local|remote|opened|offline> - display the dht records in the store
+    record list <local|remote|opened|offline|watched> - display the dht records in the store
            purge <local|remote> [bytes] - clear all dht records optionally down to some total size
            create <dhtschema> [<cryptokind> [<safety>]] - create a new dht record
            open <key>[+<safety>] [<writer>] - open an existing dht record

veilid-core/src/veilid_api/json_api/routing_context.rs

@@ -78,19 +78,20 @@ pub enum RoutingContextRequestOp {
     WatchDhtValues {
         #[schemars(with = "String")]
         key: TypedKey,
-        subkeys: ValueSubkeyRangeSet,
+        subkeys: Option<ValueSubkeyRangeSet>,
-        expiration: Timestamp,
+        expiration: Option<Timestamp>,
-        count: u32,
+        count: Option<u32>,
     },
     CancelDhtWatch {
         #[schemars(with = "String")]
         key: TypedKey,
-        subkeys: ValueSubkeyRangeSet,
+        subkeys: Option<ValueSubkeyRangeSet>,
     },
     InspectDhtRecord {
         #[schemars(with = "String")]
         key: TypedKey,
-        subkeys: ValueSubkeyRangeSet,
+        subkeys: Option<ValueSubkeyRangeSet>,
+        #[schemars(default)]
         scope: DHTReportScope,
     },
 }
@@ -149,7 +150,7 @@ pub enum RoutingContextResponseOp {
     },
     WatchDhtValues {
         #[serde(flatten)]
-        result: ApiResult<Timestamp>,
+        result: ApiResult<bool>,
     },
     CancelDhtWatch {
         #[serde(flatten)]

veilid-core/src/veilid_api/routing_context.rs

@@ -398,13 +398,18 @@ impl RoutingContext {
     ///
     /// There is only one watch permitted per record. If a change to a watch is desired, the previous one will be overwritten.
     /// * `key` is the record key to watch. it must first be opened for reading or writing.
-    /// * `subkeys` is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
+    /// * `subkeys`:
-    /// * `expiration` is the desired timestamp of when to automatically terminate the watch, in microseconds. If this value is less than `network.rpc.timeout_ms` milliseconds in the future, this function will return an error immediately.
+    ///   - None: specifies watching the entire range of subkeys.
-    /// * `count` is the number of times the watch will be sent, maximum. A zero value here is equivalent to a cancellation.
+    ///   - Some(range): is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
+    /// * `expiration`:
+    ///   - None: specifies a watch with no expiration
+    ///   - Some(timestamp): the desired timestamp of when to automatically terminate the watch, in microseconds. If this value is less than `network.rpc.timeout_ms` milliseconds in the future, this function will return an error immediately.
+    /// * `count:
+    ///   - None: specifies a watch count of u32::MAX
+    ///   - Some(count): is the number of times the watch will be sent, maximum. A zero value here is equivalent to a cancellation.
     ///
-    /// Returns a timestamp of when the watch will expire. All watches are guaranteed to expire at some point in the future,
+    /// Returns Ok(true) if a watch is active for this record.
-    /// and the returned timestamp will be no later than the requested expiration, but -may- be before the requested expiration.
+    /// Returns Ok(false) if the entire watch has been cancelled.
-    /// If the returned timestamp is zero it indicates that the watch creation or update has failed. In the case of a faild update, the watch is considered cancelled.
     ///
     /// DHT watches are accepted with the following conditions:
     /// * First-come first-served basis for arbitrary unauthenticated readers, up to network.dht.public_watch_limit per record.
@@ -415,12 +420,15 @@ impl RoutingContext {
     pub async fn watch_dht_values(
         &self,
         key: TypedKey,
-        subkeys: ValueSubkeyRangeSet,
+        subkeys: Option<ValueSubkeyRangeSet>,
-        expiration: Timestamp,
+        expiration: Option<Timestamp>,
-        count: u32,
+        count: Option<u32>,
-    ) -> VeilidAPIResult<Timestamp> {
+    ) -> VeilidAPIResult<bool> {
         veilid_log!(self debug
-            "RoutingContext::watch_dht_values(self: {:?}, key: {:?}, subkeys: {:?}, expiration: {}, count: {})", self, key, subkeys, expiration, count);
+            "RoutingContext::watch_dht_values(self: {:?}, key: {:?}, subkeys: {:?}, expiration: {:?}, count: {:?})", self, key, subkeys, expiration, count);
+        let subkeys = subkeys.unwrap_or_default();
+        let expiration = expiration.unwrap_or_default();
+        let count = count.unwrap_or(u32::MAX);
 
         Crypto::validate_crypto_kind(key.kind)?;
 
@@ -431,20 +439,24 @@ impl RoutingContext {
     /// Cancels a watch early.
     ///
     /// This is a convenience function that cancels watching all subkeys in a range. The subkeys specified here
-    /// are subtracted from the watched subkey range. If no range is specified, this is equivalent to cancelling the entire range of subkeys.
+    /// are subtracted from the currently-watched subkey range.
+    /// * `subkeys`:
+    ///   - None: specifies watching the entire range of subkeys.
+    ///   - Some(range): is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
     /// Only the subkey range is changed, the expiration and count remain the same.
     /// If no subkeys remain, the watch is entirely cancelled and will receive no more updates.
     ///
-    /// Returns Ok(true) if there is any remaining watch for this record.
+    /// Returns Ok(true) if a watch is active for this record.
     /// Returns Ok(false) if the entire watch has been cancelled.
     #[instrument(target = "veilid_api", level = "debug", fields(__VEILID_LOG_KEY = self.log_key()), ret, err)]
     pub async fn cancel_dht_watch(
         &self,
         key: TypedKey,
-        subkeys: ValueSubkeyRangeSet,
+        subkeys: Option<ValueSubkeyRangeSet>,
     ) -> VeilidAPIResult<bool> {
         veilid_log!(self debug
             "RoutingContext::cancel_dht_watch(self: {:?}, key: {:?}, subkeys: {:?}", self, key, subkeys);
+        let subkeys = subkeys.unwrap_or_default();
 
         Crypto::validate_crypto_kind(key.kind)?;
 
@@ -457,8 +469,9 @@ impl RoutingContext {
     /// to see what needs updating locally.
     ///
     /// * `key` is the record key to inspect. it must first be opened for reading or writing.
-    /// * `subkeys` is the the range of subkeys to inspect. The range must not exceed 512 discrete non-overlapping or adjacent subranges.
+    /// * `subkeys`:
-    ///    If no range is specified, this is equivalent to inspecting the entire range of subkeys. In total, the list of subkeys returned will be truncated at 512 elements.
+    ///   - None: specifies inspecting the entire range of subkeys.
+    ///   - Some(range): is the the range of subkeys to inspect. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
     /// * `scope` is what kind of range the inspection has:
     ///
     ///   - DHTReportScope::Local
@@ -495,11 +508,12 @@ impl RoutingContext {
     pub async fn inspect_dht_record(
         &self,
         key: TypedKey,
-        subkeys: ValueSubkeyRangeSet,
+        subkeys: Option<ValueSubkeyRangeSet>,
         scope: DHTReportScope,
     ) -> VeilidAPIResult<DHTRecordReport> {
         veilid_log!(self debug
             "RoutingContext::inspect_dht_record(self: {:?}, key: {:?}, subkeys: {:?}, scope: {:?})", self, key, subkeys, scope);
+        let subkeys = subkeys.unwrap_or_default();
 
         Crypto::validate_crypto_kind(key.kind)?;
 

veilid-core/src/veilid_api/types/dht/dht_record_report.rs

@@ -51,6 +51,20 @@ impl DHTRecordReport {
     pub fn network_seqs(&self) -> &[ValueSeqNum] {
         &self.network_seqs
     }
+    pub fn changed_subkeys(&self) -> ValueSubkeyRangeSet {
+        let mut changed = ValueSubkeyRangeSet::new();
+        for ((sk, lseq), nseq) in self
+            .subkeys
+            .iter()
+            .zip(self.local_seqs.iter())
+            .zip(self.network_seqs.iter())
+        {
+            if nseq > lseq {
+                changed.insert(sk);
+            }
+        }
+        changed
+    }
 }
 
 impl fmt::Debug for DHTRecordReport {
@@ -65,9 +79,21 @@ impl fmt::Debug for DHTRecordReport {
         )
     }
 }
+
 /// DHT Record Report Scope
 #[derive(
-    Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
+    Copy,
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    JsonSchema,
+    Default,
 )]
 #[cfg_attr(
     all(target_arch = "wasm32", target_os = "unknown"),
@@ -77,6 +103,7 @@ impl fmt::Debug for DHTRecordReport {
 pub enum DHTReportScope {
     /// Return only the local copy sequence numbers
     /// Useful for seeing what subkeys you have locally and which ones have not been retrieved
+    #[default]
     Local = 0,
     /// Return the local sequence numbers and the network sequence numbers with GetValue fanout parameters
     /// Provides an independent view of both the local sequence numbers and the network sequence numbers for nodes that
@@ -100,8 +127,3 @@ pub enum DHTReportScope {
     /// Useful for determine which subkeys would change on an SetValue operation
     UpdateSet = 4,
 }
-impl Default for DHTReportScope {
-    fn default() -> Self {
-        Self::Local
-    }
-}

veilid-flutter/example/android/app/.gitignore

@@ -0,0 +1 @@
+.cxx

veilid-flutter/example/integration_test/app_test.dart

@@ -1,6 +1,6 @@
 @Timeout(Duration(seconds: 120))
 
-library veilid_flutter_integration_test;
+library;
 
 import 'package:flutter_test/flutter_test.dart';
 import 'package:integration_test/integration_test.dart';

veilid-flutter/example/lib/app.dart

@@ -76,19 +76,14 @@ class _MyAppState extends State<MyApp> with UiLoggy {
     switch (log.logLevel) {
       case VeilidLogLevel.error:
         loggy.error(log.message, error, stackTrace);
-        break;
       case VeilidLogLevel.warn:
         loggy.warning(log.message, error, stackTrace);
-        break;
       case VeilidLogLevel.info:
         loggy.info(log.message, error, stackTrace);
-        break;
       case VeilidLogLevel.debug:
         loggy.debug(log.message, error, stackTrace);
-        break;
       case VeilidLogLevel.trace:
         loggy.trace(log.message, error, stackTrace);
-        break;
     }
   }
 

veilid-flutter/example/pubspec.lock

@@ -450,7 +450,7 @@ packages:
       path: ".."
       relative: true
     source: path
-    version: "0.4.3"
+    version: "0.4.4"
   veilid_test:
     dependency: "direct dev"
     description:

veilid-flutter/example/test/widget_test.dart

@@ -11,14 +11,14 @@ import 'package:flutter_test/flutter_test.dart';
 import 'package:veilid_example/app.dart';
 
 void main() {
-  testWidgets('Verify Platform version', (WidgetTester tester) async {
+  testWidgets('Verify Platform version', (tester) async {
     // Build our app and trigger a frame.
     await tester.pumpWidget(const MyApp());
 
     // Verify that platform version is retrieved.
     expect(
       find.byWidgetPredicate(
-        (Widget widget) =>
+        (widget) =>
             widget is Text && widget.data!.startsWith('Running on:'),
       ),
       findsOneWidget,

veilid-flutter/lib/routing_context.dart

@@ -301,7 +301,7 @@ abstract class VeilidRoutingContext {
       {bool forceRefresh = false});
   Future<ValueData?> setDHTValue(TypedKey key, int subkey, Uint8List data,
       {KeyPair? writer});
-  Future<Timestamp> watchDHTValues(TypedKey key,
+  Future<bool> watchDHTValues(TypedKey key,
       {List<ValueSubkeyRange>? subkeys, Timestamp? expiration, int? count});
   Future<bool> cancelDHTWatch(TypedKey key, {List<ValueSubkeyRange>? subkeys});
   Future<DHTRecordReport> inspectDHTRecord(TypedKey key,

veilid-flutter/lib/veilid_ffi.dart

@@ -294,10 +294,11 @@ Future<T> processFuturePlain<T>(Future<dynamic> future) async =>
                 'Unexpected async return message type: ${list[0]}');
           }
       }
+      // Any errors at all from Veilid need to be caught
       // ignore: inference_failure_on_untyped_parameter
-    }).catchError((e) {
+    }).catchError((e, s) {
       // Wrap all other errors in VeilidAPIExceptionInternal
-      throw VeilidAPIExceptionInternal(e.toString());
+      throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
     }, test: (e) => e is! VeilidAPIException);
 
 Future<T> processFutureJson<T>(
@@ -332,10 +333,11 @@ Future<T> processFutureJson<T>(
                 'Unexpected async return message type: ${list[0]}');
           }
       }
+      // Any errors at all from Veilid need to be caught
       // ignore: inference_failure_on_untyped_parameter
-    }).catchError((e) {
+    }).catchError((e, s) {
       // Wrap all other errors in VeilidAPIExceptionInternal
-      throw VeilidAPIExceptionInternal(e.toString());
+      throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
     }, test: (e) => e is! VeilidAPIException);
 
 Future<T?> processFutureOptJson<T>(
@@ -372,10 +374,11 @@ Future<T?> processFutureOptJson<T>(
                 'Unexpected async return message type: ${list[0]}');
           }
       }
+      // Any errors at all from Veilid need to be caught
       // ignore: inference_failure_on_untyped_parameter
-    }).catchError((e) {
+    }).catchError((e, s) {
       // Wrap all other errors in VeilidAPIExceptionInternal
-      throw VeilidAPIExceptionInternal(e.toString());
+      throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
     }, test: (e) => e is! VeilidAPIException);
 
 Future<void> processFutureVoid(Future<dynamic> future) async =>
@@ -414,10 +417,11 @@ Future<void> processFutureVoid(Future<dynamic> future) async =>
                 'Unexpected async return message type: ${list[0]}');
           }
       }
+      // Any errors at all from Veilid need to be caught
       // ignore: inference_failure_on_untyped_parameter
-    }).catchError((e) {
+    }).catchError((e, s) {
       // Wrap all other errors in VeilidAPIExceptionInternal
-      throw VeilidAPIExceptionInternal(e.toString());
+      throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
     }, test: (e) => e is! VeilidAPIException);
 
 Future<Stream<T>> processFutureStream<T>(
@@ -457,10 +461,11 @@ Future<Stream<T>> processFutureStream<T>(
                 'Unexpected async return message type: ${list[0]}');
           }
       }
+      // Any errors at all from Veilid need to be caught
       // ignore: inference_failure_on_untyped_parameter
-    }).catchError((e) {
+    }).catchError((e, s) {
       // Wrap all other errors in VeilidAPIExceptionInternal
-      throw VeilidAPIExceptionInternal(e.toString());
+      throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
     }, test: (e) => e is! VeilidAPIException);
 
 Stream<T> processStreamJson<T>(
@@ -703,7 +708,7 @@ class VeilidRoutingContextFFI extends VeilidRoutingContext {
   }
 
   @override
-  Future<Timestamp> watchDHTValues(TypedKey key,
+  Future<bool> watchDHTValues(TypedKey key,
       {List<ValueSubkeyRange>? subkeys,
       Timestamp? expiration,
       int? count}) async {
@@ -720,9 +725,8 @@ class VeilidRoutingContextFFI extends VeilidRoutingContext {
     final sendPort = recvPort.sendPort;
     _ctx.ffi._routingContextWatchDHTValues(sendPort.nativePort, _ctx.id!,
         nativeKey, nativeSubkeys, nativeExpiration, count);
-    final actualExpiration = Timestamp(
+    final active = await processFuturePlain<bool>(recvPort.first);
-        value: BigInt.from(await processFuturePlain<int>(recvPort.first)));
+    return active;
-    return actualExpiration;
   }
 
   @override
@@ -738,8 +742,8 @@ class VeilidRoutingContextFFI extends VeilidRoutingContext {
     final sendPort = recvPort.sendPort;
     _ctx.ffi._routingContextCancelDHTWatch(
         sendPort.nativePort, _ctx.id!, nativeKey, nativeSubkeys);
-    final cancelled = await processFuturePlain<bool>(recvPort.first);
+    final active = await processFuturePlain<bool>(recvPort.first);
-    return cancelled;
+    return active;
   }
 
   @override

veilid-flutter/lib/veilid_js.dart

@@ -24,14 +24,15 @@ dynamic convertUint8ListToJson(Uint8List data) => data.toList().jsify();
 Future<T> _wrapApiPromise<T>(Object p) => js_util
         .promiseToFuture<T>(p)
         .then((value) => value)
+        // Any errors at all from Veilid need to be caught
         // ignore: inference_failure_on_untyped_parameter
-        .catchError((e) {
+        .catchError((e, s) {
       try {
         final ex = VeilidAPIException.fromJson(jsonDecode(e as String));
         throw ex;
       } on Exception catch (_) {
         // Wrap all other errors in VeilidAPIExceptionInternal
-        throw VeilidAPIExceptionInternal(e.toString());
+        throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
       }
     });
 
@@ -206,7 +207,7 @@ class VeilidRoutingContextJS extends VeilidRoutingContext {
   }
 
   @override
-  Future<Timestamp> watchDHTValues(TypedKey key,
+  Future<bool> watchDHTValues(TypedKey key,
       {List<ValueSubkeyRange>? subkeys,
       Timestamp? expiration,
       int? count}) async {
@@ -215,7 +216,7 @@ class VeilidRoutingContextJS extends VeilidRoutingContext {
     count ??= 0xFFFFFFFF;
 
     final id = _ctx.requireId();
-    final ts = await _wrapApiPromise<String>(js_util.callMethod(
+    return _wrapApiPromise<bool>(js_util.callMethod(
         wasm, 'routing_context_watch_dht_values', [
       id,
       jsonEncode(key),
@@ -223,7 +224,6 @@ class VeilidRoutingContextJS extends VeilidRoutingContext {
       expiration.toString(),
       count
     ]));
-    return Timestamp.fromString(ts);
   }
 
   @override

veilid-flutter/packages/veilid_test/.flutter-plugins

@@ -0,0 +1,8 @@
+# This is a generated file; do not edit or check into version control.
+path_provider=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider-2.1.5/
+path_provider_android=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_android-2.2.16/
+path_provider_foundation=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.1/
+path_provider_linux=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_linux-2.2.1/
+path_provider_windows=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_windows-2.3.0/
+system_info_plus=/Users/dildog/.pub-cache/hosted/pub.dev/system_info_plus-0.0.6/
+veilid=/Users/dildog/code/veilid/veilid-flutter/

veilid-flutter/packages/veilid_test/.flutter-plugins-dependencies

@@ -0,0 +1 @@
+{"info":"This is a generated file; do not edit or check into version control.","plugins":{"ios":[{"name":"path_provider_foundation","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.1/","shared_darwin_source":true,"native_build":true,"dependencies":[],"dev_dependency":false},{"name":"system_info_plus","path":"/Users/dildog/.pub-cache/hosted/pub.dev/system_info_plus-0.0.6/","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":["system_info_plus"],"dev_dependency":false}],"android":[{"name":"path_provider_android","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_android-2.2.16/","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"system_info_plus","path":"/Users/dildog/.pub-cache/hosted/pub.dev/system_info_plus-0.0.6/","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":["system_info_plus"],"dev_dependency":false}],"macos":[{"name":"path_provider_foundation","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.1/","shared_darwin_source":true,"native_build":true,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":[],"dev_dependency":false}],"linux":[{"name":"path_provider_linux","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_linux-2.2.1/","native_build":false,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":[],"dev_dependency":false}],"windows":[{"name":"path_provider_windows","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_windows-2.3.0/","native_build":false,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":[],"dev_dependency":false}],"web":[{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","dependencies":[],"dev_dependency":false}]},"dependencyGraph":[{"name":"path_provider","dependencies":["path_provider_android","path_provider_foundation","path_provider_linux","path_provider_windows"]},{"name":"path_provider_android","dependencies":[]},{"name":"path_provider_foundation","dependencies":[]},{"name":"path_provider_linux","dependencies":[]},{"name":"path_provider_windows","dependencies":[]},{"name":"system_info_plus","dependencies":[]},{"name":"veilid","dependencies":["path_provider","system_info_plus"]}],"date_created":"2025-03-21 12:28:02.057123","version":"3.29.2","swift_package_manager_enabled":{"ios":false,"macos":false}}

veilid-flutter/packages/veilid_test/lib/src/veilid_fixture.dart

@@ -149,7 +149,6 @@ class DefaultVeilidFixture implements VeilidFixture {
           case AttachmentState.overAttached:
           case AttachmentState.fullyAttached:
             done = true;
-            break;
         }
       }
       if (done) {

veilid-flutter/rust/src/dart_ffi.rs

@@ -821,9 +821,9 @@ pub extern "C" fn routing_context_watch_dht_values(
             let routing_context = get_routing_context(id, "routing_context_watch_dht_values")?;
 
             let res = routing_context
-                .watch_dht_values(key, subkeys, expiration, count)
+                .watch_dht_values(key, Some(subkeys), Some(expiration), Some(count))
                 .await?;
-            APIResult::Ok(res.as_u64())
+            APIResult::Ok(res)
         }
         .in_current_span(),
     );
@@ -846,7 +846,7 @@ pub extern "C" fn routing_context_cancel_dht_watch(
         async move {
             let routing_context = get_routing_context(id, "routing_context_cancel_dht_watch")?;
 
-            let res = routing_context.cancel_dht_watch(key, subkeys).await?;
+            let res = routing_context.cancel_dht_watch(key, Some(subkeys)).await?;
             APIResult::Ok(res)
         }
         .in_current_span(),
@@ -874,7 +874,7 @@ pub extern "C" fn routing_context_inspect_dht_record(
             let routing_context = get_routing_context(id, "routing_context_inspect_dht_record")?;
 
             let res = routing_context
-                .inspect_dht_record(key, subkeys, scope)
+                .inspect_dht_record(key, Some(subkeys), scope)
                 .await?;
             APIResult::Ok(res)
         }

veilid-python/tests/test_dht.py

@@ -1,5 +1,4 @@
 # Routing context veilid tests
-
 from typing import Any, Awaitable, Callable, Optional
 import pytest
 import asyncio
@@ -7,7 +6,8 @@ import time
 import os
 
 import veilid
-from veilid import ValueSubkey
+from veilid import ValueSubkey, Timestamp, SafetySelection
+from veilid.types import VeilidJSONEncoder
 
 ##################################################################
 BOGUS_KEY = veilid.TypedKey.from_value(
@@ -86,8 +86,8 @@ async def test_set_get_dht_value(api_connection: veilid.VeilidAPI):
         vd4 = await rc.get_dht_value(rec.key, ValueSubkey(1), False)
         assert vd4 is None
 
-        print("vd2: {}", vd2.__dict__)
+        #print("vd2: {}", vd2.__dict__)
-        print("vd3: {}", vd3.__dict__)
+        #print("vd3: {}", vd3.__dict__)
 
         assert vd2 == vd3
 
@@ -245,8 +245,7 @@ async def test_open_writer_dht_value(api_connection: veilid.VeilidAPI):
         await rc.delete_dht_record(key)
 
 
-# @pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running")
+@pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running")
-@pytest.mark.skip(reason = "don't work yet")
 @pytest.mark.asyncio
 async def test_watch_dht_values():
 
@@ -256,113 +255,257 @@ async def test_watch_dht_values():
         if update.kind == veilid.VeilidUpdateKind.VALUE_CHANGE:
             await value_change_queue.put(update)
 
+    async def null_update_callback(update: veilid.VeilidUpdate):
+        pass
+
     try:
-        api = await veilid.api_connector(value_change_update_callback)
+        api0 = await veilid.api_connector(value_change_update_callback, 0)
     except veilid.VeilidConnectionError:
-        pytest.skip("Unable to connect to veilid-server.")
+        pytest.skip("Unable to connect to veilid-server 0.")
 
-    # Make two routing contexts, one with and one without safety
-    # So we can pretend to be a different node and get the watch updates
-    # Normally they would not get sent if the set comes from the same target
-    # as the watch's target
-    
-    # XXX: this logic doesn't work because our node still suppresses updates
-    # XXX: if the value hasn't changed in the local record store
-    rcWatch = await api.new_routing_context()
-    
-    rcSet = await (await api.new_routing_context()).with_safety(veilid.SafetySelection.unsafe())
-    async with rcWatch, rcSet:
-        # Make a DHT record
-        rec = await rcWatch.create_dht_record(veilid.DHTSchema.dflt(10))
-
-        # Set some subkey we care about
-        vd = await rcWatch.set_dht_value(rec.key, ValueSubkey(3), b"BLAH BLAH BLAH")
-        assert vd is None
-
-        # Make a watch on that subkey
-        ts = await rcWatch.watch_dht_values(rec.key, [], 0, 0xFFFFFFFF)
-        assert ts != 0
-
-        # Reopen without closing to change routing context and not lose watch
-        rec = await rcSet.open_dht_record(rec.key, rec.owner_key_pair())
-        
-        # Now set the subkey and trigger an update
-        vd = await rcSet.set_dht_value(rec.key, ValueSubkey(3), b"BLAH")
-        assert vd is None
-        
-        # Now we should NOT get an update because the update is the same as our local copy
-        update = None
     try:
-            update = await asyncio.wait_for(value_change_queue.get(), timeout=5)
+        api1 = await veilid.api_connector(null_update_callback, 1)
+    except veilid.VeilidConnectionError:
+        pytest.skip("Unable to connect to veilid-server 1.")
+
+    async with api0, api1:
+        # purge local and remote record stores to ensure we start fresh
+        await api0.debug("record purge local")
+        await api0.debug("record purge remote")
+        await api1.debug("record purge local")
+        await api1.debug("record purge remote")
+
+        # Clear the change queue if record purge cancels old watches
+        while True:
+            try:
+                upd = await asyncio.wait_for(value_change_queue.get(), timeout=3)
+            except asyncio.TimeoutError:
+                break
+
+        # make routing contexts
+        rc0 = await api0.new_routing_context()
+        rc1 = await api1.new_routing_context()
+        async with rc0, rc1:
+
+            # Server 0: Make a DHT record
+            rec0 = await rc0.create_dht_record(veilid.DHTSchema.dflt(10))
+
+            # Server 0: Set some subkey we care about
+            vd = await rc0.set_dht_value(rec0.key, ValueSubkey(3), b"BLAH")
+            assert vd is None
+
+            await sync(rc0, [rec0])
+
+            # Server 0: Make a watch on all the subkeys
+            active = await rc0.watch_dht_values(rec0.key, [], Timestamp(0), 0xFFFFFFFF)
+            assert active
+
+            # Server 1: Open the subkey
+            rec1 = await rc1.open_dht_record(rec0.key, rec0.owner_key_pair())
+
+            # Server 1: Now set the subkey and trigger an update
+            vd = await rc1.set_dht_value(rec1.key, ValueSubkey(3), b"BLAH")
+            assert vd is None
+            await sync(rc1, [rec1])
+
+            # Server 0: Now we should NOT get an update because the update is the same as our local copy
+            upd = None
+            try:
+                upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
             except asyncio.TimeoutError:
                 pass
-        assert update is None
+            assert upd is None
 
-        # Now set multiple subkeys and trigger an update
+            # Server 1: Now set subkey and trigger an update
-        vd = await asyncio.gather(*[rcSet.set_dht_value(rec.key, ValueSubkey(3), b"BLAH BLAH"), rcSet.set_dht_value(rec.key, ValueSubkey(4), b"BZORT")])
+            vd = await rc1.set_dht_value(rec1.key, ValueSubkey(3), b"BLAH BLAH")
-        assert vd == [None, None]
+            assert vd is None
+            await sync(rc1, [rec1])
 
-        # Wait for the update
+            # Server 0: Wait for the update
-        upd = await asyncio.wait_for(value_change_queue.get(), timeout=5)
-
-        # Verify the update came back but we don't get a new value because the sequence number is the same
-        assert upd.detail.key == rec.key
-        assert upd.detail.count == 0xFFFFFFFD
-        assert upd.detail.subkeys == [(3, 4)]
-        assert upd.detail.value is None
-
-        # Reopen without closing to change routing context and not lose watch
-        rec = await rcWatch.open_dht_record(rec.key, rec.owner_key_pair())
-
-        # Cancel some subkeys we don't care about
-        still_active = await rcWatch.cancel_dht_watch(rec.key, [(ValueSubkey(0), ValueSubkey(2))])
-        assert still_active
-
-        # Reopen without closing to change routing context and not lose watch
-        rec = await rcSet.open_dht_record(rec.key, rec.owner_key_pair())
-
-        # Now set multiple subkeys and trigger an update
-        vd = await asyncio.gather(*[rcSet.set_dht_value(rec.key, ValueSubkey(3), b"BLAH BLAH BLAH"), rcSet.set_dht_value(rec.key, ValueSubkey(5), b"BZORT BZORT")])
-        assert vd == [None, None]
-
-        # Wait for the update, this longer timeout seems to help the flaky check below
             upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
 
-        # Verify the update came back but we don't get a new value because the sequence number is the same
+            # Server 0: Verify the update came back with the first changed subkey's data
-        assert upd.detail.key == rec.key
+            assert upd.detail.key == rec0.key
+            assert upd.detail.count == 0xFFFFFFFE
+            assert upd.detail.subkeys == [(3, 3)]
+            assert upd.detail.value.data == b"BLAH BLAH"
 
-        # This check is flaky on slow connections and often fails with different counts
+            # Server 1: Now set subkey and trigger an update
-        assert upd.detail.count == 0xFFFFFFFC
+            vd = await rc1.set_dht_value(rec1.key, ValueSubkey(4), b"BZORT")
-        assert upd.detail.subkeys == [(3, 3), (5, 5)]
+            assert vd is None
-        assert upd.detail.value is None
+            await sync(rc1, [rec1])
 
-        # Reopen without closing to change routing context and not lose watch
+            # Server 0: Wait for the update
-        rec = await rcWatch.open_dht_record(rec.key, rec.owner_key_pair())
+            upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
 
-        # Now cancel the update
+            # Server 0: Verify the update came back with the first changed subkey's data
-        still_active = await rcWatch.cancel_dht_watch(rec.key, [(ValueSubkey(3), ValueSubkey(9))])
+            assert upd.detail.key == rec0.key
-        assert not still_active
+            assert upd.detail.count == 0xFFFFFFFD
+            assert upd.detail.subkeys == [(4, 4)]
+            assert upd.detail.value.data == b"BZORT"
 
-        # Reopen without closing to change routing context and not lose watch
+            # Server 0: Cancel some subkeys we don't care about
-        rec = await rcSet.open_dht_record(rec.key, rec.owner_key_pair())
+            active = await rc0.cancel_dht_watch(rec0.key, [(ValueSubkey(0), ValueSubkey(3))])
+            assert active
 
-        # Now set multiple subkeys
+            # Server 1: Now set multiple subkeys and trigger an update
-        vd = await asyncio.gather(*[rcSet.set_dht_value(rec.key, ValueSubkey(3), b"BLAH BLAH BLAH BLAH"), rcSet.set_dht_value(rec.key, ValueSubkey(5), b"BZORT BZORT BZORT")])
+            vd = await asyncio.gather(*[rc1.set_dht_value(rec1.key, ValueSubkey(3), b"BLAH BLAH BLAH"), rc1.set_dht_value(rec1.key, ValueSubkey(4), b"BZORT BZORT")])
             assert vd == [None, None]
+            await sync(rc1, [rec1])
 
-        # Now we should NOT get an update
+            # Server 0: Wait for the update
-        update = None
+            upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
+
+            # Server 0: Verify only one update came back
+            assert upd.detail.key == rec0.key
+            assert upd.detail.count == 0xFFFFFFFC
+            assert upd.detail.subkeys == [(4, 4)]
+            assert upd.detail.value.data == b"BZORT BZORT"
+
+            # Server 0: Now we should NOT get any other update
+            upd = None
             try:
-            update = await asyncio.wait_for(value_change_queue.get(), timeout=5)
+                upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
             except asyncio.TimeoutError:
                 pass
-        assert update is None
+            if upd is not None:
+                print(f"bad update: {VeilidJSONEncoder.dumps(upd)}")
+            assert upd is None
+
+            # Now cancel the update
+            active = await rc0.cancel_dht_watch(rec0.key, [(ValueSubkey(3), ValueSubkey(9))])
+            assert not active
+
+            # Server 0: Wait for the cancellation update
+            upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
+
+            # Server 0: Verify only one update came back
+            assert upd.detail.key == rec0.key
+            assert upd.detail.count == 0
+            assert upd.detail.subkeys == []
+            assert upd.detail.value is None
+
+            # Now set multiple subkeys
+            vd = await asyncio.gather(*[rc1.set_dht_value(rec1.key, ValueSubkey(3), b"BLAH BLAH BLAH BLAH"), rc1.set_dht_value(rec1.key, ValueSubkey(5), b"BZORT BZORT BZORT")])
+            assert vd == [None, None]
+            await sync(rc1, [rec1])
+
+            # Now we should NOT get an update
+            upd = None
+            try:
+                upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
+            except asyncio.TimeoutError:
+                pass
+            if upd is not None:
+                print(f"bad update: {VeilidJSONEncoder.dumps(upd)}")
+            assert upd is None
 
             # Clean up
-        await rcSet.close_dht_record(rec.key)
+            await rc1.close_dht_record(rec1.key)
-        await rcSet.delete_dht_record(rec.key)
+            await rc1.delete_dht_record(rec1.key)
+            await rc0.close_dht_record(rec0.key)
+            await rc0.delete_dht_record(rec0.key)
 
 
+@pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running")
+@pytest.mark.skipif(os.getenv("STRESS") != "1", reason="stress test takes a long time")
+@pytest.mark.asyncio
+async def test_watch_many_dht_values():
+
+    value_change_queue: asyncio.Queue[veilid.VeilidUpdate] = asyncio.Queue()
+
+    async def value_change_update_callback(update: veilid.VeilidUpdate):
+        if update.kind == veilid.VeilidUpdateKind.VALUE_CHANGE:
+            await value_change_queue.put(update)
+
+    async def null_update_callback(update: veilid.VeilidUpdate):
+        pass
+
+    try:
+        api0 = await veilid.api_connector(value_change_update_callback, 0)
+    except veilid.VeilidConnectionError:
+        pytest.skip("Unable to connect to veilid-server 0.")
+
+    try:
+        api1 = await veilid.api_connector(null_update_callback, 1)
+    except veilid.VeilidConnectionError:
+        pytest.skip("Unable to connect to veilid-server 1.")
+
+    async with api0, api1:
+        # purge local and remote record stores to ensure we start fresh
+        await api0.debug("record purge local")
+        await api0.debug("record purge remote")
+        await api1.debug("record purge local")
+        await api1.debug("record purge remote")
+
+        # make routing contexts
+        # unsafe version for debugging
+        rc0 = await (await api0.new_routing_context()).with_safety(SafetySelection.unsafe())
+        rc1 = await (await api1.new_routing_context()).with_safety(SafetySelection.unsafe())
+        # safe default version
+        # rc0 = await api0.new_routing_context()
+        # rc1 = await api1.new_routing_context()
+
+        async with rc0, rc1:
+
+            COUNT = 10
+            records = []
+
+            # Make and watch all records
+            for n in range(COUNT):
+                print(f"making record {n}")
+                # Server 0: Make a DHT record
+                records.append(await rc0.create_dht_record(veilid.DHTSchema.dflt(1)))
+
+                # Server 0: Set some subkey we care about
+                vd = await rc0.set_dht_value(records[n].key, ValueSubkey(0), b"BLAH")
+                assert vd is None
+
+                # Server 0: Make a watch on all the subkeys
+                active = await rc0.watch_dht_values(records[n].key, [], Timestamp(0), 0xFFFFFFFF)
+                assert active
+
+            # Open and set all records
+            missing_records = set()
+            for (n, record) in enumerate(records):
+                print(f"setting record {n}")
+
+                # Server 1: Open the subkey
+                _ignore = await rc1.open_dht_record(record.key, record.owner_key_pair())
+
+                # Server 1: Now set the subkey and trigger an update
+                vd = await rc1.set_dht_value(record.key, ValueSubkey(0), b"BLAH BLAH")
+                assert vd is None
+
+                missing_records.add(record.key)
+
+            # Server 0: Now we should get an update for every change
+            for n in range(len(records)):
+                print(f"waiting for change {n}")
+
+                # Server 0: Wait for the update
+                try:
+                    upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
+                    missing_records.remove(upd.detail.key)
+                except:
+                    # Dump which records didn't get updates
+                    for (m, record) in enumerate(records):
+                        if record.key not in missing_records:
+                            continue
+                        print(f"missing update for record {m}: {record}")
+                        info0 = await api0.debug(f"record info {record.key}")
+                        info1 = await api1.debug(f"record info {record.key}")
+                        print(f"from rc0: {info0}")
+                        print(f"from rc1: {info1}")
+                    raise
+
+            # Clean up
+            for record in records:
+                await rc1.close_dht_record(record.key)
+                await rc1.delete_dht_record(record.key)
+                await rc0.close_dht_record(record.key)
+                await rc0.delete_dht_record(record.key)
+
 @pytest.mark.asyncio
 async def test_inspect_dht_record(api_connection: veilid.VeilidAPI):
     rc = await api_connection.new_routing_context()
@@ -486,8 +629,6 @@ async def test_schema_limit_smpl(api_connection: veilid.VeilidAPI):
 
 
 
-
-
 @pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running")
 @pytest.mark.asyncio
 async def test_dht_integration_writer_reader():
@@ -702,21 +843,23 @@ async def test_dht_write_read_full_subkeys_local():
 
 
 async def sync(rc: veilid.RoutingContext, records: list[veilid.DHTRecordDescriptor]):
-    print('syncing records to the network')
     syncrecords = records.copy()
-    while len(syncrecords) > 0:
+    if len(syncrecords) == 0:
+        return
+    while True:
         donerecords = set()
         subkeysleft = 0
         for desc in records:
             rr = await rc.inspect_dht_record(desc.key, [])
             left = 0; [left := left + (x[1]-x[0]+1) for x in rr.offline_subkeys]
             if left == 0:
-                if veilid.ValueSeqNum.NONE not in rr.local_seqs:
                 donerecords.add(desc)
             else:
                 subkeysleft += left
         syncrecords = [x for x in syncrecords if x not in donerecords]
-        print(f'  {len(syncrecords)} records {subkeysleft} subkeys left')
+        if len(syncrecords) == 0:
+            break
+        print(f'  syncing {len(syncrecords)} records {subkeysleft} subkeys left')
         time.sleep(1)
 
 

veilid-python/veilid/api.py

@@ -92,17 +92,17 @@ class RoutingContext(ABC):
     async def watch_dht_values(
         self,
         key: types.TypedKey,
-        subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]],
+        subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]] = [],
-        expiration: types.Timestamp = 0,
+        expiration: types.Timestamp = types.Timestamp(0),
         count: int = 0xFFFFFFFF,
-    ) -> types.Timestamp:
+    ) -> bool:
         pass
 
     @abstractmethod
     async def cancel_dht_watch(
         self,
         key: types.TypedKey,
-        subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]],
+        subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]] = [],
     ) -> bool:
         pass
 

veilid-python/veilid/json_api.py

@@ -740,10 +740,10 @@ class _JsonRoutingContext(RoutingContext):
     async def watch_dht_values(
         self,
         key: TypedKey,
-        subkeys: list[tuple[ValueSubkey, ValueSubkey]],
+        subkeys: list[tuple[ValueSubkey, ValueSubkey]] = [],
-        expiration: Timestamp = 0,
+        expiration: Timestamp = Timestamp(0),
         count: int = 0xFFFFFFFF,
-    ) -> Timestamp:
+    ) -> bool:
         assert isinstance(key, TypedKey)
         assert isinstance(subkeys, list)
         for s in subkeys:
@@ -753,8 +753,7 @@ class _JsonRoutingContext(RoutingContext):
         assert isinstance(expiration, Timestamp)
         assert isinstance(count, int)
 
-        return Timestamp(
+        return raise_api_result(
-            raise_api_result(
             await self.api.send_ndjson_request(
                 Operation.ROUTING_CONTEXT,
                 validate=validate_rc_op,
@@ -766,10 +765,10 @@ class _JsonRoutingContext(RoutingContext):
                 count=count,
             )
         )
-        )
+
 
     async def cancel_dht_watch(
-        self, key: TypedKey, subkeys: list[tuple[ValueSubkey, ValueSubkey]]
+        self, key: TypedKey, subkeys: list[tuple[ValueSubkey, ValueSubkey]] = []
     ) -> bool:
         assert isinstance(key, TypedKey)
         assert isinstance(subkeys, list)

veilid-python/veilid/schema/RecvMessage.json

@@ -858,7 +858,7 @@
                   ],
                   "properties": {
                     "value": {
-                      "type": "string"
+                      "type": "boolean"
                     }
                   }
                 },

veilid-python/veilid/schema/Request.json

@@ -469,20 +469,23 @@
         {
           "type": "object",
           "required": [
-            "count",
-            "expiration",
             "key",
-            "rc_op",
+            "rc_op"
-            "subkeys"
           ],
           "properties": {
             "count": {
-              "type": "integer",
+              "type": [
+                "integer",
+                "null"
+              ],
               "format": "uint32",
               "minimum": 0.0
             },
             "expiration": {
-              "type": "string"
+              "type": [
+                "string",
+                "null"
+              ]
             },
             "key": {
               "type": "string"
@@ -494,7 +497,10 @@
               ]
             },
             "subkeys": {
-              "type": "array",
+              "type": [
+                "array",
+                "null"
+              ],
               "items": {
                 "type": "array",
                 "items": [
@@ -519,8 +525,7 @@
           "type": "object",
           "required": [
             "key",
-            "rc_op",
+            "rc_op"
-            "subkeys"
           ],
           "properties": {
             "key": {
@@ -533,7 +538,10 @@
               ]
             },
             "subkeys": {
-              "type": "array",
+              "type": [
+                "array",
+                "null"
+              ],
               "items": {
                 "type": "array",
                 "items": [
@@ -558,9 +566,7 @@
           "type": "object",
           "required": [
             "key",
-            "rc_op",
+            "rc_op"
-            "scope",
-            "subkeys"
           ],
           "properties": {
             "key": {
@@ -573,10 +579,18 @@
               ]
             },
             "scope": {
+              "default": "Local",
+              "allOf": [
+                {
                   "$ref": "#/definitions/DHTReportScope"
+                }
+              ]
             },
             "subkeys": {
-              "type": "array",
+              "type": [
+                "array",
+                "null"
+              ],
               "items": {
                 "type": "array",
                 "items": [

veilid-python/veilid/state.py

@@ -59,6 +59,8 @@ class VeilidStateAttachment:
             j["attached_uptime"],
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
 
 
 class AnswerStats:
@@ -114,6 +116,9 @@ class AnswerStats:
             j["consecutive_lost_answers_minimum"],
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 class RPCStats:
     messages_sent: int
     messages_rcvd: int
@@ -172,6 +177,9 @@ class RPCStats:
             AnswerStats.from_json(j["answer_ordered"]),
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class LatencyStats:
     fastest: TimestampDuration
@@ -213,6 +221,9 @@ class LatencyStats:
             TimestampDuration(j["p75"]),
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class TransferStats:
     total: ByteCount
@@ -365,6 +376,9 @@ class PeerStats:
             StateStats.from_json(j["state"]),
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class PeerTableData:
     node_ids: list[str]
@@ -381,6 +395,9 @@ class PeerTableData:
         """JSON object hook"""
         return cls(j["node_ids"], j["peer_address"], PeerStats.from_json(j["peer_stats"]))
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidStateNetwork:
     started: bool
@@ -410,6 +427,9 @@ class VeilidStateNetwork:
             [PeerTableData.from_json(peer) for peer in j["peers"]],
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidStateConfig:
     config: VeilidConfig
@@ -422,6 +442,9 @@ class VeilidStateConfig:
         """JSON object hook"""
         return cls(VeilidConfig.from_json(j["config"]))
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidState:
     attachment: VeilidStateAttachment
@@ -447,6 +470,9 @@ class VeilidState:
             VeilidStateConfig.from_json(j["config"]),
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidLog:
     log_level: VeilidLogLevel
@@ -463,6 +489,9 @@ class VeilidLog:
         """JSON object hook"""
         return cls(VeilidLogLevel(j["log_level"]), j["message"], j["backtrace"])
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidAppMessage:
     sender: Optional[TypedKey]
@@ -483,6 +512,9 @@ class VeilidAppMessage:
             urlsafe_b64decode_no_pad(j["message"]),
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidAppCall:
     sender: Optional[TypedKey]
@@ -506,6 +538,9 @@ class VeilidAppCall:
             OperationId(j["call_id"]),
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidRouteChange:
     dead_routes: list[RouteId]
@@ -523,6 +558,9 @@ class VeilidRouteChange:
             [RouteId(route) for route in j["dead_remote_routes"]],
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidValueChange:
     key: TypedKey
@@ -546,6 +584,9 @@ class VeilidValueChange:
             None if j["value"] is None else ValueData.from_json(j["value"]),
         )
 
+    def to_json(self) -> dict:
+        return self.__dict__
+
 
 class VeilidUpdateKind(StrEnum):
     LOG = "Log"
@@ -610,3 +651,6 @@ class VeilidUpdate:
             case _:
                 raise ValueError("Unknown VeilidUpdateKind")
         return cls(kind, detail)
+
+    def to_json(self) -> dict:
+        return self.__dict__

veilid-server/src/client_api.rs

@@ -395,7 +395,7 @@ impl ClientApi {
         // Request receive processor future
         // Receives from socket and enqueues RequestLines
         // Completes when the connection is closed or there is a failure
-        unord.push(system_boxed(self.clone().receive_requests(
+        unord.push(pin_dyn_future!(self.clone().receive_requests(
             reader,
             requests_tx,
             responses_tx,
@@ -404,12 +404,14 @@ impl ClientApi {
         // Response send processor
         // Sends finished response strings out the socket
         // Completes when the responses channel is closed
-        unord.push(system_boxed(
+        unord.push(pin_dyn_future!(self
-            self.clone().send_responses(responses_rx, writer),
+            .clone()
-        ));
+            .send_responses(responses_rx, writer)));
 
         // Add future to process first request
-        unord.push(system_boxed(Self::next_request_line(requests_rx.clone())));
+        unord.push(pin_dyn_future!(Self::next_request_line(
+            requests_rx.clone()
+        )));
 
         // Send and receive until we're done or a stop is requested
         while let Ok(Some(r)) = unord.next().timeout_at(stop_token.clone()).await {
@@ -417,7 +419,9 @@ impl ClientApi {
             let request_line = match r {
                 Ok(Some(request_line)) => {
                     // Add future to process next request
-                    unord.push(system_boxed(Self::next_request_line(requests_rx.clone())));
+                    unord.push(pin_dyn_future!(Self::next_request_line(
+                        requests_rx.clone()
+                    )));
 
                     // Socket receive future returned something to process
                     request_line
@@ -434,9 +438,9 @@ impl ClientApi {
             };
 
             // Enqueue unordered future to process request line in parallel
-            unord.push(system_boxed(
+            unord.push(pin_dyn_future!(self
-                self.clone().process_request_line(jrp.clone(), request_line),
+                .clone()
-            ));
+                .process_request_line(jrp.clone(), request_line)));
         }
 
         // Stop sending updates

veilid-tools/src/async_tag_lock.rs

@@ -3,8 +3,26 @@ use super::*;
 use core::fmt::Debug;
 use core::hash::Hash;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct AsyncTagLockGuard<T>
+where
+    T: Hash + Eq + Clone + Debug,
+{
+    inner: Arc<AsyncTagLockGuardInner<T>>,
+}
+
+impl<T> AsyncTagLockGuard<T>
+where
+    T: Hash + Eq + Clone + Debug,
+{
+    #[must_use]
+    pub fn tag(&self) -> T {
+        self.inner.tag()
+    }
+}
+
+#[derive(Debug)]
+struct AsyncTagLockGuardInner<T>
 where
     T: Hash + Eq + Clone + Debug,
 {
@@ -13,7 +31,7 @@ where
     guard: Option<AsyncMutexGuardArc<()>>,
 }
 
-impl<T> AsyncTagLockGuard<T>
+impl<T> AsyncTagLockGuardInner<T>
 where
     T: Hash + Eq + Clone + Debug,
 {
@@ -24,9 +42,13 @@ where
             guard: Some(guard),
         }
     }
+
+    fn tag(&self) -> T {
+        self.tag.clone()
+    }
 }
 
-impl<T> Drop for AsyncTagLockGuard<T>
+impl<T> Drop for AsyncTagLockGuardInner<T>
 where
     T: Hash + Eq + Clone + Debug,
 {
@@ -133,7 +155,9 @@ where
         let guard = asyncmutex_lock_arc!(mutex);
 
         // Return the locked guard
-        AsyncTagLockGuard::new(self.clone(), tag, guard)
+        AsyncTagLockGuard {
+            inner: Arc::new(AsyncTagLockGuardInner::new(self.clone(), tag, guard)),
+        }
     }
 
     pub fn try_lock_tag(&self, tag: T) -> Option<AsyncTagLockGuard<T>> {
@@ -160,7 +184,9 @@ where
             }
         };
         // Return guard
-        Some(AsyncTagLockGuard::new(self.clone(), tag, guard))
+        Some(AsyncTagLockGuard {
+            inner: Arc::new(AsyncTagLockGuardInner::new(self.clone(), tag, guard)),
+        })
     }
 }
 

veilid-tools/src/deferred_stream_processor.rs

@@ -113,7 +113,10 @@ impl DeferredStreamProcessor {
     /// * 'handler' is the callback to handle each item from the stream
     ///
     /// Returns 'true' if the stream was added for processing, and 'false' if the stream could not be added, possibly due to not being initialized.
-    pub fn add<T: Send + 'static, S: futures_util::Stream<Item = T> + Unpin + Send + 'static>(
+    pub fn add_stream<
+        T: Send + 'static,
+        S: futures_util::Stream<Item = T> + Unpin + Send + 'static,
+    >(
         &self,
         mut receiver: S,
         mut handler: impl FnMut(T) -> PinBoxFutureStatic<bool> + Send + 'static,
@@ -140,6 +143,24 @@ impl DeferredStreamProcessor {
         }
         true
     }
+
+    /// Queue a single future to process in the background
+    pub fn add_future<F>(&self, fut: F) -> bool
+    where
+        F: Future<Output = ()> + Send + 'static,
+    {
+        let dsc_tx = {
+            let inner = self.inner.lock();
+            let Some(dsc_tx) = inner.opt_deferred_stream_channel.clone() else {
+                return false;
+            };
+            dsc_tx
+        };
+        if dsc_tx.send(Box::pin(fut)).is_err() {
+            return false;
+        }
+        true
+    }
 }
 
 impl Default for DeferredStreamProcessor {

veilid-tools/src/tools.rs

@@ -119,14 +119,6 @@ macro_rules! asyncrwlock_try_write_arc {
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-pub fn system_boxed<'a, Out>(
-    future: impl Future<Output = Out> + Send + 'a,
-) -> PinBoxFuture<'a, Out> {
-    Box::pin(future)
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
 cfg_if! {
     if #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] {
         #[must_use]

veilid-tools/src/virtual_network/router_client.rs

@@ -589,7 +589,7 @@ impl RouterClient {
         let framed_reader = FramedRead::new(reader, BytesCodec);
         let framed_writer = FramedWrite::new(writer, BytesCodec);
 
-        let framed_writer_fut = system_boxed(async move {
+        let framed_writer_fut = Box::pin(async move {
             if let Err(e) = receiver
                 .into_stream()
                 .map(|command| {
@@ -603,7 +603,7 @@ impl RouterClient {
                 error!("{}", e);
             }
         });
-        let framed_reader_fut = system_boxed(async move {
+        let framed_reader_fut = Box::pin(async move {
             let fut = framed_reader.try_for_each(|x| async {
                 let x = x;
                 let evt = from_bytes::<ServerProcessorEvent>(&x)
@@ -631,7 +631,7 @@ impl RouterClient {
             .into_stream()
             .map(io::Result::<ServerProcessorEvent>::Ok);
 
-        let receiver_fut = system_boxed(async move {
+        let receiver_fut = Box::pin(async move {
             let fut =
                 receiver.try_for_each(|evt| Self::process_event(evt, router_op_waiter.clone()));
             if let Err(e) = fut.await {

veilid-tools/src/virtual_network/router_server/mod.rs

@@ -117,7 +117,7 @@ impl RouterServer {
         let stop_token = stop_source.token();
 
         let this = self.clone();
-        let listener_fut = system_boxed(async move {
+        let listener_fut = Box::pin(async move {
             loop {
                 // Wait for a new connection
                 match listener.accept().timeout_at(stop_token.clone()).await {
@@ -125,7 +125,7 @@ impl RouterServer {
                         let conn = conn.compat();
                         // Register a connection processing inbound receiver
                         let this2 = this.clone();
-                        let inbound_receiver_fut = system_boxed(async move {
+                        let inbound_receiver_fut = Box::pin(async move {
                             let (reader, writer) = conn.split();
 
                             this2.process_connection(reader, writer).await
@@ -178,7 +178,7 @@ impl RouterServer {
         let stop_token = stop_source.token();
 
         let this = self.clone();
-        let listener_fut = system_boxed(async move {
+        let listener_fut = Box::pin(async move {
             loop {
                 // Wait for a new connection
                 match listener.accept().timeout_at(stop_token.clone()).await {
@@ -188,7 +188,7 @@ impl RouterServer {
                             let ws = WsStream::new(s);
                             // Register a connection processing inbound receiver
                             let this2 = this.clone();
-                            let inbound_receiver_fut = system_boxed(async move {
+                            let inbound_receiver_fut = Box::pin(async move {
                                 let (reader, writer) = ws.split();
                                 this2.process_connection(reader, writer).await
                             });
@@ -233,7 +233,7 @@ impl RouterServer {
         let (local_outbound_sender, local_outbound_receiver) = flume::unbounded();
 
         let this = self.clone();
-        let inbound_receiver_fut = system_boxed(async move {
+        let inbound_receiver_fut = Box::pin(async move {
             local_inbound_receiver
                 .into_stream()
                 .for_each(|cmd| async {
@@ -316,7 +316,7 @@ impl RouterServer {
         let framed_writer = FramedWrite::new(writer, BytesCodec);
 
         let (outbound_sender, outbound_receiver) = flume::unbounded();
-        let outbound_fut = system_boxed(
+        let outbound_fut = Box::pin(
             outbound_receiver
                 .into_stream()
                 .map(|command| {
@@ -327,7 +327,7 @@ impl RouterServer {
                 .forward(framed_writer),
         );
 
-        let inbound_fut = system_boxed(framed_reader.try_for_each(|x| async {
+        let inbound_fut = Box::pin(framed_reader.try_for_each(|x| async {
             let x = x;
             let cmd = from_bytes::<ServerProcessorCommand>(&x).map_err(io::Error::other)?;
 

veilid-wasm/src/lib.rs

@@ -626,9 +626,9 @@ pub fn routing_context_watch_dht_values(
         let routing_context = get_routing_context(id, "routing_context_watch_dht_values")?;
 
         let res = routing_context
-            .watch_dht_values(key, subkeys, expiration, count)
+            .watch_dht_values(key, Some(subkeys), Some(expiration), Some(count))
             .await?;
-        APIResult::Ok(res.as_u64().to_string())
+        APIResult::Ok(res)
     })
 }
 
@@ -642,7 +642,7 @@ pub fn routing_context_cancel_dht_watch(id: u32, key: String, subkeys: String) -
 
         let routing_context = get_routing_context(id, "routing_context_cancel_dht_watch")?;
 
-        let res = routing_context.cancel_dht_watch(key, subkeys).await?;
+        let res = routing_context.cancel_dht_watch(key, Some(subkeys)).await?;
         APIResult::Ok(res)
     })
 }
@@ -665,7 +665,7 @@ pub fn routing_context_inspect_dht_record(
         let routing_context = get_routing_context(id, "routing_context_inspect_dht_record")?;
 
         let res = routing_context
-            .inspect_dht_record(key, subkeys, scope)
+            .inspect_dht_record(key, Some(subkeys), scope)
             .await?;
 
         APIResult::Ok(res)

veilid-wasm/src/veilid_routing_context_js.rs

@@ -344,17 +344,22 @@ impl VeilidRoutingContext {
     ///
     /// There is only one watch permitted per record. If a change to a watch is desired, the previous one will be overwritten.
     /// * `key` is the record key to watch. it must first be opened for reading or writing.
-    /// * `subkeys` is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
+    /// * `subkeys`:
-    /// * `expiration` is the desired timestamp of when to automatically terminate the watch, in microseconds. If this value is less than `network.rpc.timeout_ms` milliseconds in the future, this function will return an error immediately.
+    ///   - None: specifies watching the entire range of subkeys.
-    /// * `count` is the number of times the watch will be sent, maximum. A zero value here is equivalent to a cancellation.
+    ///   - Some(range): is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
+    /// * `expiration`:
+    ///   - None: specifies a watch with no expiration
+    ///   - Some(timestamp): the desired timestamp of when to automatically terminate the watch, in microseconds. If this value is less than `network.rpc.timeout_ms` milliseconds in the future, this function will return an error immediately.
+    /// * `count:
+    ///   - None: specifies a watch count of u32::MAX
+    ///   - Some(count): is the number of times the watch will be sent, maximum. A zero value here is equivalent to a cancellation.
     ///
-    /// Returns a timestamp of when the watch will expire. All watches are guaranteed to expire at some point in the future,
+    /// Returns Ok(true) if a watch is active for this record.
-    /// and the returned timestamp will be no later than the requested expiration, but -may- be before the requested expiration.
+    /// Returns Ok(false) if the entire watch has been cancelled.
-    /// If the returned timestamp is zero it indicates that the watch creation or update has failed. In the case of a faild update, the watch is considered cancelled.
     ///
     /// DHT watches are accepted with the following conditions:
-    /// * First-come first-served basis for arbitrary unauthenticated readers, up to network.dht.public_watch_limit per record
+    /// * First-come first-served basis for arbitrary unauthenticated readers, up to network.dht.public_watch_limit per record.
-    /// * If a member (either the owner or a SMPL schema member) has opened the key for writing (even if no writing is performed) then the watch will be signed and guaranteed network.dht.member_watch_limit per writer
+    /// * If a member (either the owner or a SMPL schema member) has opened the key for writing (even if no writing is performed) then the watch will be signed and guaranteed network.dht.member_watch_limit per writer.
     ///
     /// Members can be specified via the SMPL schema and do not need to allocate writable subkeys in order to offer a member watch capability.
     pub async fn watchDhtValues(
@@ -363,42 +368,38 @@ impl VeilidRoutingContext {
         subkeys: Option<ValueSubkeyRangeSet>,
         expiration: Option<String>,
         count: Option<u32>,
-    ) -> APIResult<String> {
+    ) -> APIResult<bool> {
         let key = TypedKey::from_str(&key)?;
-        let subkeys = subkeys.unwrap_or_default();
         let expiration = if let Some(expiration) = expiration {
-            veilid_core::Timestamp::new(
+            Some(veilid_core::Timestamp::new(
                 u64::from_str(&expiration).map_err(VeilidAPIError::generic)?,
-            )
+            ))
         } else {
-            veilid_core::Timestamp::default()
+            None
         };
-        let count = count.unwrap_or(u32::MAX);
-
         let routing_context = self.getRoutingContext()?;
         let res = routing_context
             .watch_dht_values(key, subkeys, expiration, count)
             .await?;
-        APIResult::Ok(res.as_u64().to_string())
+        APIResult::Ok(res)
     }
 
-    /// Cancels a watch early
+    /// Cancels a watch early.
     ///
     /// This is a convenience function that cancels watching all subkeys in a range. The subkeys specified here
-    /// are subtracted from the watched subkey range. If no range is specified, this is equivalent to cancelling the entire range of subkeys.
+    /// are subtracted from the currently-watched subkey range.
+    /// If no range is specified, this is equivalent to cancelling the entire range of subkeys.
     /// Only the subkey range is changed, the expiration and count remain the same.
     /// If no subkeys remain, the watch is entirely cancelled and will receive no more updates.
     ///
-    /// Returns true if there is any remaining watch for this record
+    /// Returns Ok(true) if a watch is active for this record.
-    /// Returns false if the entire watch has been cancelled
+    /// Returns Ok(false) if the entire watch has been cancelled.
     pub async fn cancelDhtWatch(
         &self,
         key: String,
         subkeys: Option<ValueSubkeyRangeSet>,
     ) -> APIResult<bool> {
         let key = TypedKey::from_str(&key)?;
-        let subkeys = subkeys.unwrap_or_default();
-
         let routing_context = self.getRoutingContext()?;
         let res = routing_context.cancel_dht_watch(key, subkeys).await?;
         APIResult::Ok(res)
@@ -450,7 +451,6 @@ impl VeilidRoutingContext {
         scope: Option<DHTReportScope>,
     ) -> APIResult<DHTRecordReport> {
         let key = TypedKey::from_str(&key)?;
-        let subkeys = subkeys.unwrap_or_default();
         let scope = scope.unwrap_or_default();
 
         let routing_context = self.getRoutingContext()?;

veilid-wasm/tests/package-lock.json

@@ -21,7 +21,7 @@
     },
     "../pkg": {
       "name": "veilid-wasm",
-      "version": "0.4.3",
+      "version": "0.4.4",
       "dev": true,
       "license": "MPL-2.0"
     },

veilid-wasm/tests/src/VeilidRoutingContext.test.ts

@@ -236,9 +236,7 @@ describe('VeilidRoutingContext', () => {
           "0",
           0xFFFFFFFF,
         );
-        expect(watchValueRes).toBeDefined();
+        expect(watchValueRes).toEqual(true);
-        expect(watchValueRes).not.toEqual("");
-        expect(watchValueRes).not.toEqual("0");
 
         const cancelValueRes = await routingContext.cancelDhtWatch(
           dhtRecord.key,
@@ -261,9 +259,7 @@ describe('VeilidRoutingContext', () => {
         const watchValueRes = await routingContext.watchDhtValues(
           dhtRecord.key,
         );
-        expect(watchValueRes).toBeDefined();
+        expect(watchValueRes).toEqual(true);
-        expect(watchValueRes).not.toEqual("");
-        expect(watchValueRes).not.toEqual("0");
 
         const cancelValueRes = await routingContext.cancelDhtWatch(
           dhtRecord.key,