more offline subkey write improvements

veilid-core/src/storage_manager/debug.rs

@@ -33,6 +33,14 @@ impl StorageManager {
         }
         format!("{}]\n", out)
     }
+    pub(crate) async fn debug_offline_records(&self) -> String {
+        let inner = self.inner.lock().await;
+        let mut out = "[\n".to_owned();
+        for (k, v) in &inner.offline_subkey_writes {
+            out += &format!("  {}:{:?}\n", k, v);
+        }
+        format!("{}]\n", out)
+    }
 
     pub(crate) async fn purge_local_records(&self, reclaim: Option<usize>) -> String {
         let mut inner = self.inner.lock().await;

veilid-core/src/storage_manager/mod.rs

@@ -557,8 +557,17 @@ impl StorageManager {
             }
         };
 
-        // Keep the list of nodes that returned a value for later reference
+        // Regain the lock after network access
         let mut inner = self.lock().await?;
+
+        // Report on fanout result offline
+        let was_offline = self.check_fanout_set_offline(key, subkey, &result.fanout_result);
+        if was_offline {
+            // Failed to write, try again later
+            inner.add_offline_subkey_write(key, subkey, safety_selection);
+        }
+
+        // Keep the list of nodes that returned a value for later reference
         inner.process_fanout_results(key, core::iter::once((subkey, &result.fanout_result)), true);
 
         // Return the new value if it differs from what was asked to set
@@ -910,4 +919,32 @@ impl StorageManager {
 
         Ok(())
     }
+
+    fn check_fanout_set_offline(
+        &self,
+        key: TypedKey,
+        subkey: ValueSubkey,
+        fanout_result: &FanoutResult,
+    ) -> bool {
+        match fanout_result.kind {
+            FanoutResultKind::Timeout => {
+                log_stor!(debug "timeout in set_value, adding offline subkey: {}:{}", key, subkey);
+                true
+            }
+            FanoutResultKind::Exhausted => {
+                let get_consensus =
+                    self.unlocked_inner.config.get().network.dht.get_value_count as usize;
+                let value_node_count = fanout_result.value_nodes.len();
+                if value_node_count < get_consensus {
+                    log_stor!(debug "exhausted with insufficient consensus ({}<{}), adding offline subkey: {}:{}", 
+                        value_node_count, get_consensus,
+                        key, subkey);
+                    true
+                } else {
+                    false
+                }
+            }
+            FanoutResultKind::Finished => false,
+        }
+    }
 }

veilid-core/src/storage_manager/tasks/offline_subkey_writes.rs

@@ -62,12 +62,10 @@ impl StorageManager {
                     )
                     .await;
                 match osvres {
-                    Ok(osv) => {
-                        match osv.fanout_result.kind {
-                            FanoutResultKind::Timeout => {
-                                log_stor!(debug "timed out writing offline subkey: {}:{}", key, subkey);
-                            }
-                            FanoutResultKind::Finished | FanoutResultKind::Exhausted => {
+                    Ok(result) => {
+                        let was_offline =
+                            self.check_fanout_set_offline(*key, subkey, &result.fanout_result);
+                        if !was_offline {
                             if let Some(update_callback) = opt_update_callback.clone() {
                                 // Send valuechange with dead count and no subkeys
                                 update_callback(VeilidUpdate::ValueChange(Box::new(
@@ -75,15 +73,13 @@ impl StorageManager {
                                         key: *key,
                                         subkeys: ValueSubkeyRangeSet::single(subkey),
                                         count: u32::MAX,
-                                            value: Some(osv.signed_value_data.value_data().clone()),
+                                        value: Some(result.signed_value_data.value_data().clone()),
                                     },
                                 )));
                             }
                             written_subkeys.insert(subkey);
-                            }
                         };
-
-                        fanout_results.push((subkey, osv.fanout_result));
+                        fanout_results.push((subkey, result.fanout_result));
                     }
                     Err(e) => {
                         log_stor!(debug "failed to write offline subkey: {}:{} {}", key, subkey, e);

veilid-core/src/veilid_api/debug.rs

@@ -1414,6 +1414,11 @@ impl VeilidAPI {
                 out += &storage_manager.debug_opened_records().await;
                 out
             }
+            "offline" => {
+                let mut out = "Offline Records:\n".to_string();
+                out += &storage_manager.debug_offline_records().await;
+                out
+            }
             _ => "Invalid scope\n".to_owned(),
         };
         Ok(out)
@@ -1951,7 +1956,7 @@ route allocate [ord|*ord] [rel] [<count>] [in|out]
       list
       import <blob>
       test <route>
-record list <local|remote|opened>
+record list <local|remote|opened|offline>
        purge <local|remote> [bytes]
        create <dhtschema> [<cryptokind> [<safety>]]
        open <key>[+<safety>] [<writer>]