Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2025-07-27 08:55:35 -04:00
Code Issues Projects Releases Packages Wiki Activity

Improved WatchValue

Browse source
This commit is contained in:
Christien Rioux 2025-04-21 14:05:44 -04:00
parent 72b1434abc
commit e6c7c28746
89 changed files with 1891892 additions and 1807 deletions
Download patch file Download diff file Expand all files Collapse all files

51
Cargo.lock generated
View file

@ -691,6 +691,12 @@ version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "base64"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64ct"
version = "1.7.3"
@ -954,6 +960,7 @@ dependencies = [
"iana-time-zone",
"js-sys",
"num-traits",
"serde",
"wasm-bindgen",
"windows-link",
]
@ -1575,6 +1582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058"
dependencies = [
"powerfmt",
"serde",
]
[[package]]
@ -2395,7 +2403,7 @@ version = "7.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
dependencies = [
"base64",
"base64 0.21.7",
"byteorder",
"flate2",
"nom",
@ -2877,6 +2885,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
"serde",
]
[[package]]
@ -2887,6 +2896,7 @@ checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
"serde",
]
[[package]]
@ -4639,7 +4649,7 @@ version = "0.11.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
dependencies = [
"base64",
"base64 0.21.7",
"bytes 1.10.1",
"encoding_rs",
"futures-core",
@ -4829,7 +4839,7 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
"base64",
"base64 0.21.7",
]
[[package]]
@ -5158,6 +5168,36 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_with"
version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa"
dependencies = [
"base64 0.22.1",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.8.0",
"serde",
"serde_derive",
"serde_json",
"serde_with_macros",
"time",
]
[[package]]
name = "serde_with_macros"
version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e"
dependencies = [
"darling 0.20.10",
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]]
name = "serde_yaml"
version = "0.9.34+deprecated"
@ -5845,7 +5885,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
dependencies = [
"async-trait",
"axum",
"base64",
"base64 0.21.7",
"bytes 1.10.1",
"futures-core",
"futures-util",
@ -5874,7 +5914,7 @@ dependencies = [
"async-stream",
"async-trait",
"axum",
"base64",
"base64 0.21.7",
"bytes 1.10.1",
"h2",
"http 0.2.12",
@ -6475,6 +6515,7 @@ dependencies = [
"serde-wasm-bindgen 0.6.5",
"serde_bytes",
"serde_json",
"serde_with",
"serial_test 2.0.0",
"sha2 0.10.8",
"shell-words",

4
veilid-cli/src/client_api_connection.rs
View file

@ -173,7 +173,7 @@ impl ClientApiConnection {
let mut inner = this.inner.lock();
inner.request_sender = None;
};
unord.push(system_boxed(recv_messages_future));
unord.push(pin_dyn_future!(recv_messages_future));
// Requests send processor
let send_requests_future = async move {
@ -183,7 +183,7 @@ impl ClientApiConnection {
}
}
};
unord.push(system_boxed(send_requests_future));
unord.push(pin_dyn_future!(send_requests_future));
// Request initial server state
let capi = self.clone();

1
veilid-core/Cargo.toml
View file

@ -147,6 +147,7 @@ lz4_flex = { version = "0.11.3", default-features = false, features = [
] }
indent = "0.1.1"
sanitize-filename = "0.5.0"
serde_with = "3.12.0"
# Dependencies for native builds only
# Linux, Windows, Mac, iOS, Android

4
veilid-core/proto/veilid.capnp
View file

@ -353,8 +353,8 @@ struct OperationSetValueA @0x9378d0732dc95be2 {
struct OperationWatchValueQ @0xf9a5a6c547b9b228 {
key @0 :TypedKey; # key for value to watch
subkeys @1 :List(SubkeyRange); # subkey range to watch (up to 512 subranges), if empty this implies 0..=UINT32_MAX
expiration @2 :UInt64; # requested timestamp when this watch will expire in usec since epoch (can be return less, 0 for max)
subkeys @1 :List(SubkeyRange); # subkey range to watch (up to 512 subranges). An empty range here should not be specified unless cancelling a watch (count=0).
expiration @2 :UInt64; # requested timestamp when this watch will expire in usec since epoch (watch can return less, 0 for max)
count @3 :UInt32; # requested number of changes to watch for (0 = cancel, 1 = single shot, 2+ = counter, UINT32_MAX = continuous)
watchId @4 :UInt64; # if 0, request a new watch. if >0, existing watch id
watcher @5 :PublicKey; # the watcher performing the watch, can be the owner or a schema member, or a generated anonymous watch keypair

427
veilid-core/proto/veilid_capnp.rs
View file

File diff suppressed because it is too large Load diff

5
veilid-core/src/component.rs
View file

@ -105,6 +105,7 @@ impl VeilidComponentRegistry {
self.namespace
}
#[allow(dead_code)]
pub fn program_name(&self) -> &'static str {
self.program_name
}
@ -293,7 +294,7 @@ impl VeilidComponentRegistryAccessor for VeilidComponentRegistry {
////////////////////////////////////////////////////////////////////
macro_rules! impl_veilid_component_registry_accessor {
($struct_name:ident) => {
($struct_name:ty) => {
impl VeilidComponentRegistryAccessor for $struct_name {
fn registry(&self) -> VeilidComponentRegistry {
self.registry.clone()
@ -307,7 +308,7 @@ pub(crate) use impl_veilid_component_registry_accessor;
/////////////////////////////////////////////////////////////////////
macro_rules! impl_veilid_component {
($component_name:ident) => {
($component_name:ty) => {
impl_veilid_component_registry_accessor!($component_name);
impl VeilidComponent for $component_name {

13
veilid-core/src/network_manager/connection_manager.rs
View file

@ -691,10 +691,7 @@ impl ConnectionManager {
fn spawn_reconnector(&self, dial_info: DialInfo) {
let this = self.clone();
self.arc.reconnection_processor.add(
Box::pin(futures_util::stream::once(async { dial_info })),
move |dial_info| {
let this = this.clone();
self.arc.reconnection_processor.add_future(
Box::pin(async move {
match this.get_or_create_connection(dial_info.clone()).await {
Ok(NetworkResult::Value(conn)) => {
@ -706,15 +703,11 @@ impl ConnectionManager {
Err(e) => {
veilid_log!(this debug "Reconnection error to {}: {}", dial_info, e);
}
}
false
})
},
);
};
}));
}
pub fn debug_print(&self) -> String {
//let inner = self.arc.inner.lock();
format!(
"Connection Table:\n\n{}",
self.arc.connection_table.debug_print_table()

6
veilid-core/src/network_manager/network_connection.rs
View file

@ -352,7 +352,7 @@ impl NetworkConnection {
};
let timer = MutableFuture::new(new_timer());
unord.push(system_boxed(timer.clone().in_current_span()));
unord.push(pin_dyn_future!(timer.clone().in_current_span()));
loop {
// Add another message sender future if necessary
@ -386,7 +386,7 @@ impl NetworkConnection {
}
}
}.in_current_span());
unord.push(system_boxed(sender_fut.in_current_span()));
unord.push(pin_dyn_future!(sender_fut.in_current_span()));
}
// Add another message receiver future if necessary
@ -445,7 +445,7 @@ impl NetworkConnection {
}
}.in_current_span());
unord.push(system_boxed(receiver_fut.in_current_span()));
unord.push(pin_dyn_future!(receiver_fut.in_current_span()));
}
// Process futures

1
veilid-core/src/routing_table/node_ref/node_ref_lock.rs
View file

@ -39,6 +39,7 @@ impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Disp
}
}
#[expect(dead_code)]
pub fn unlocked(&self) -> N {
self.nr.clone()
}

2
veilid-core/src/routing_table/tasks/ping_validator.rs
View file

@ -236,7 +236,7 @@ impl RoutingTable {
}
// Get all the active watches from the storage manager
let watch_destinations = self.storage_manager().get_active_watch_nodes().await;
let watch_destinations = self.storage_manager().get_outbound_watch_nodes().await;
for watch_destination in watch_destinations {
let registry = self.registry();

2
veilid-core/src/rpc_processor/destination.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
/// Where to send an RPC message
#[derive(Debug, Clone)]
pub(crate) enum Destination {

413
veilid-core/src/rpc_processor/fanout/fanout_call.rs
View file

@ -1,40 +1,63 @@
use super::*;
struct FanoutContext<R>
where
R: Unpin,
{
fanout_queue: FanoutQueue,
result: Option<Result<R, RPCError>>,
impl_veilid_log_facility!("fanout");
#[derive(Debug)]
struct FanoutContext<'a> {
fanout_queue: FanoutQueue<'a>,
result: FanoutResult,
done: bool,
}
#[derive(Debug, Copy, Clone)]
#[derive(Debug, Copy, Clone, Default)]
pub enum FanoutResultKind {
Partial,
#[default]
Incomplete,
Timeout,
Finished,
Consensus,
Exhausted,
}
impl FanoutResultKind {
pub fn is_partial(&self) -> bool {
matches!(self, Self::Partial)
pub fn is_incomplete(&self) -> bool {
matches!(self, Self::Incomplete)
}
}
#[derive(Debug, Clone)]
#[derive(Clone, Debug, Default)]
pub struct FanoutResult {
/// How the fanout completed
pub kind: FanoutResultKind,
/// The set of nodes that counted toward consensus
/// (for example, had the most recent value for this subkey)
pub consensus_nodes: Vec<NodeRef>,
/// Which nodes accepted the request
pub value_nodes: Vec<NodeRef>,
}
pub fn debug_fanout_result(result: &FanoutResult) -> String {
let kc = match result.kind {
FanoutResultKind::Partial => "P",
impl fmt::Display for FanoutResult {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let kc = match self.kind {
FanoutResultKind::Incomplete => "I",
FanoutResultKind::Timeout => "T",
FanoutResultKind::Finished => "F",
FanoutResultKind::Consensus => "C",
FanoutResultKind::Exhausted => "E",
};
format!("{}:{}", kc, result.value_nodes.len())
if f.alternate() {
write!(
f,
"{}:{}[{}]",
kc,
self.consensus_nodes.len(),
self.consensus_nodes
.iter()
.map(|x| x.to_string())
.collect::<Vec<_>>()
.join(","),
)
} else {
write!(f, "{}:{}", kc, self.consensus_nodes.len())
}
}
}
pub fn debug_fanout_results(results: &[FanoutResult]) -> String {
@ -45,7 +68,7 @@ pub fn debug_fanout_results(results: &[FanoutResult]) -> String {
if col == 0 {
out += " ";
}
let sr = debug_fanout_result(r);
let sr = format!("{}", r);
out += &sr;
out += ",";
col += 1;
@ -61,11 +84,36 @@ pub fn debug_fanout_results(results: &[FanoutResult]) -> String {
#[derive(Debug)]
pub struct FanoutCallOutput {
pub peer_info_list: Vec<Arc<PeerInfo>>,
pub disposition: FanoutCallDisposition,
}
pub type FanoutCallResult = RPCNetworkResult<FanoutCallOutput>;
/// The return type of the fanout call routine
#[derive(Debug)]
pub enum FanoutCallDisposition {
/// The call routine timed out
Timeout,
/// The call routine returned an invalid result
Invalid,
/// The called node rejected the rpc request but may have returned more nodes
Rejected,
/// The called node accepted the rpc request and may have returned more nodes,
/// but we don't count the result toward our consensus
Stale,
/// The called node accepted the rpc request and may have returned more nodes,
/// counting the result toward our consensus
Accepted,
/// The called node accepted the rpc request and may have returned more nodes,
/// returning a newer value that indicates we should restart our consensus
AcceptedNewerRestart,
/// The called node accepted the rpc request and may have returned more nodes,
/// returning a newer value that indicates our current consensus is stale and should be ignored,
/// and counting the result toward a new consensus
AcceptedNewer,
}
pub type FanoutCallResult = Result<FanoutCallOutput, RPCError>;
pub type FanoutNodeInfoFilter = Arc<dyn (Fn(&[TypedKey], &NodeInfo) -> bool) + Send + Sync>;
pub type FanoutCheckDone<R> = Arc<dyn (Fn(&[NodeRef]) -> Option<R>) + Send + Sync>;
pub type FanoutCheckDone = Arc<dyn (Fn(&FanoutResult) -> bool) + Send + Sync>;
pub type FanoutCallRoutine =
Arc<dyn (Fn(NodeRef) -> PinBoxFutureStatic<FanoutCallResult>) + Send + Sync>;
@ -89,52 +137,50 @@ pub fn capability_fanout_node_info_filter(caps: Vec<Capability>) -> FanoutNodeIn
/// The algorithm is parameterized by:
/// * 'node_count' - the number of nodes to keep in the closest_nodes set
/// * 'fanout' - the number of concurrent calls being processed at the same time
/// * 'consensus_count' - the number of nodes in the processed queue that need to be in the
/// 'Accepted' state before we terminate the fanout early.
///
/// The algorithm returns early if 'check_done' returns some value, or if an error is found during the process.
/// If the algorithm times out, a Timeout result is returned, however operations will still have been performed and a
/// timeout is not necessarily indicative of an algorithmic 'failure', just that no definitive stopping condition was found
/// in the given time
pub(crate) struct FanoutCall<'a, R>
where
R: Unpin,
{
pub(crate) struct FanoutCall<'a> {
routing_table: &'a RoutingTable,
node_id: TypedKey,
context: Mutex<FanoutContext<R>>,
node_count: usize,
fanout: usize,
fanout_tasks: usize,
consensus_count: usize,
timeout_us: TimestampDuration,
node_info_filter: FanoutNodeInfoFilter,
call_routine: FanoutCallRoutine,
check_done: FanoutCheckDone<R>,
check_done: FanoutCheckDone,
}
impl<'a, R> FanoutCall<'a, R>
where
R: Unpin,
{
impl VeilidComponentRegistryAccessor for FanoutCall<'_> {
fn registry(&self) -> VeilidComponentRegistry {
self.routing_table.registry()
}
}
impl<'a> FanoutCall<'a> {
#[allow(clippy::too_many_arguments)]
pub fn new(
routing_table: &'a RoutingTable,
node_id: TypedKey,
node_count: usize,
fanout: usize,
fanout_tasks: usize,
consensus_count: usize,
timeout_us: TimestampDuration,
node_info_filter: FanoutNodeInfoFilter,
call_routine: FanoutCallRoutine,
check_done: FanoutCheckDone<R>,
check_done: FanoutCheckDone,
) -> Self {
let context = Mutex::new(FanoutContext {
fanout_queue: FanoutQueue::new(node_id.kind),
result: None,
});
Self {
routing_table,
node_id,
context,
node_count,
fanout,
fanout_tasks,
consensus_count,
timeout_us,
node_info_filter,
call_routine,
@ -143,61 +189,104 @@ where
}
#[instrument(level = "trace", target = "fanout", skip_all)]
fn evaluate_done(&self, ctx: &mut FanoutContext<R>) -> bool {
// If we have a result, then we're done
if ctx.result.is_some() {
fn evaluate_done(&self, ctx: &mut FanoutContext) -> bool {
// If we already finished, just return
if ctx.done {
return true;
}
// Check for a new done result
ctx.result = (self.check_done)(ctx.fanout_queue.nodes()).map(|o| Ok(o));
ctx.result.is_some()
// Calculate fanout result so far
let fanout_result = ctx.fanout_queue.with_nodes(|nodes, sorted_nodes| {
// Count up nodes we have seen in order and see if our closest nodes have a consensus
let mut consensus: Option<bool> = None;
let mut consensus_nodes: Vec<NodeRef> = vec![];
let mut value_nodes: Vec<NodeRef> = vec![];
for sn in sorted_nodes {
let node = nodes.get(sn).unwrap();
match node.status {
FanoutNodeStatus::Queued | FanoutNodeStatus::InProgress => {
// Still have a closer node to do before reaching consensus,
// or are doing it still, then wait until those are done
if consensus.is_none() {
consensus = Some(false);
}
}
FanoutNodeStatus::Timeout
| FanoutNodeStatus::Rejected
| FanoutNodeStatus::Disqualified => {
// Node does not count toward consensus or value node list
}
FanoutNodeStatus::Stale => {
// Node does not count toward consensus but does count toward value node list
value_nodes.push(node.node_ref.clone());
}
FanoutNodeStatus::Accepted => {
// Node counts toward consensus and value node list
value_nodes.push(node.node_ref.clone());
consensus_nodes.push(node.node_ref.clone());
if consensus.is_none() && consensus_nodes.len() >= self.consensus_count {
consensus = Some(true);
}
}
}
}
#[instrument(level = "trace", target = "fanout", skip_all)]
fn add_to_fanout_queue(&self, new_nodes: &[NodeRef]) {
event!(target: "fanout", Level::DEBUG,
"FanoutCall::add_to_fanout_queue:\n new_nodes={{\n{}}}\n",
new_nodes
.iter()
.map(|x| format!(" {}", x))
.collect::<Vec<String>>()
.join(",\n"),
);
let ctx = &mut *self.context.lock();
ctx.fanout_queue.add(new_nodes, |current_nodes| {
let mut current_nodes_vec = self
.routing_table
.sort_and_clean_closest_noderefs(self.node_id, current_nodes);
current_nodes_vec.truncate(self.node_count);
current_nodes_vec
// If we have reached sufficient consensus, return done
match consensus {
Some(true) => FanoutResult {
kind: FanoutResultKind::Consensus,
consensus_nodes,
value_nodes,
},
Some(false) => FanoutResult {
kind: FanoutResultKind::Incomplete,
consensus_nodes,
value_nodes,
},
None => FanoutResult {
kind: FanoutResultKind::Exhausted,
consensus_nodes,
value_nodes,
},
}
});
let done = (self.check_done)(&fanout_result);
ctx.result = fanout_result;
ctx.done = done;
done
}
#[instrument(level = "trace", target = "fanout", skip_all)]
async fn fanout_processor(&self) -> bool {
async fn fanout_processor<'b>(
&self,
context: &Mutex<FanoutContext<'b>>,
) -> Result<bool, RPCError> {
// Make a work request channel
let (work_sender, work_receiver) = flume::bounded(1);
// Loop until we have a result or are done
loop {
// Get the closest node we haven't processed yet if we're not done yet
let next_node = {
let mut ctx = self.context.lock();
if self.evaluate_done(&mut ctx) {
break true;
// Put in a work request
{
let mut context_locked = context.lock();
context_locked
.fanout_queue
.request_work(work_sender.clone());
}
ctx.fanout_queue.next()
};
// Wait around for some work to do
let Ok(next_node) = work_receiver.recv_async().await else {
// If we don't have a node to process, stop fanning out
let Some(next_node) = next_node else {
break false;
break Ok(false);
};
// Do the call for this node
match (self.call_routine)(next_node.clone()).await {
Ok(NetworkResult::Value(v)) => {
Ok(output) => {
// Filter returned nodes
let filtered_v: Vec<Arc<PeerInfo>> = v
let filtered_v: Vec<Arc<PeerInfo>> = output
.peer_info_list
.into_iter()
.filter(|pi| {
@ -217,25 +306,58 @@ where
let new_nodes = self
.routing_table
.register_nodes_with_peer_info_list(filtered_v);
self.add_to_fanout_queue(&new_nodes);
// Update queue
{
let mut context_locked = context.lock();
context_locked.fanout_queue.add(&new_nodes);
// Process disposition of the output of the fanout call routine
match output.disposition {
FanoutCallDisposition::Timeout => {
context_locked.fanout_queue.timeout(next_node);
}
FanoutCallDisposition::Rejected => {
context_locked.fanout_queue.rejected(next_node);
}
FanoutCallDisposition::Accepted => {
context_locked.fanout_queue.accepted(next_node);
}
FanoutCallDisposition::AcceptedNewerRestart => {
context_locked.fanout_queue.all_accepted_to_queued();
context_locked.fanout_queue.accepted(next_node);
}
FanoutCallDisposition::AcceptedNewer => {
context_locked.fanout_queue.all_accepted_to_stale();
context_locked.fanout_queue.accepted(next_node);
}
FanoutCallDisposition::Invalid => {
// Do nothing with invalid fanout calls
}
FanoutCallDisposition::Stale => {
context_locked.fanout_queue.stale(next_node);
}
}
// See if we're done before going back for more processing
if self.evaluate_done(&mut context_locked) {
break Ok(true);
}
// We modified the queue so we may have more work to do now,
// tell the queue it should send more work to the workers
context_locked.fanout_queue.send_more_work();
}
#[allow(unused_variables)]
Ok(x) => {
// Call failed, node will not be considered again
event!(target: "fanout", Level::DEBUG,
"Fanout result {}: {:?}", &next_node, x);
}
Err(e) => {
// Error happened, abort everything and return the error
self.context.lock().result = Some(Err(e));
break true;
break Err(e);
}
};
}
}
#[instrument(level = "trace", target = "fanout", skip_all)]
fn init_closest_nodes(&self) -> Result<(), RPCError> {
fn init_closest_nodes(&self, context: &mut FanoutContext) -> Result<(), RPCError> {
// Get the 'node_count' closest nodes to the key out of our routing table
let closest_nodes = {
let node_info_filter = self.node_info_filter.clone();
@ -279,36 +401,58 @@ where
.find_preferred_closest_nodes(self.node_count, self.node_id, filters, transform)
.map_err(RPCError::invalid_format)?
};
self.add_to_fanout_queue(&closest_nodes);
context.fanout_queue.add(&closest_nodes);
Ok(())
}
#[instrument(level = "trace", target = "fanout", skip_all)]
pub async fn run(
&self,
init_fanout_queue: Vec<NodeRef>,
) -> TimeoutOr<Result<Option<R>, RPCError>> {
// Get timeout in milliseconds
let timeout_ms = match us_to_ms(self.timeout_us.as_u64()).map_err(RPCError::internal) {
Ok(v) => v,
Err(e) => {
return TimeoutOr::value(Err(e));
}
pub async fn run(&self, init_fanout_queue: Vec<NodeRef>) -> Result<FanoutResult, RPCError> {
// Create context for this run
let crypto = self.routing_table.crypto();
let Some(vcrypto) = crypto.get(self.node_id.kind) else {
return Err(RPCError::internal(
"should not try this on crypto we don't support",
));
};
let node_sort = Box::new(
|a_key: &CryptoTyped<CryptoKey>,
b_key: &CryptoTyped<CryptoKey>|
-> core::cmp::Ordering {
let da = vcrypto.distance(&a_key.value, &self.node_id.value);
let db = vcrypto.distance(&b_key.value, &self.node_id.value);
da.cmp(&db)
},
);
let context = Arc::new(Mutex::new(FanoutContext {
fanout_queue: FanoutQueue::new(
self.routing_table.registry(),
self.node_id.kind,
node_sort,
self.consensus_count,
),
result: FanoutResult {
kind: FanoutResultKind::Incomplete,
consensus_nodes: vec![],
value_nodes: vec![],
},
done: false,
}));
// Get timeout in milliseconds
let timeout_ms = us_to_ms(self.timeout_us.as_u64()).map_err(RPCError::internal)?;
// Initialize closest nodes list
if let Err(e) = self.init_closest_nodes() {
return TimeoutOr::value(Err(e));
}
{
let context_locked = &mut *context.lock();
self.init_closest_nodes(context_locked)?;
// Ensure we include the most recent nodes
self.add_to_fanout_queue(&init_fanout_queue);
context_locked.fanout_queue.add(&init_fanout_queue);
// Do a quick check to see if we're already done
{
let mut ctx = self.context.lock();
if self.evaluate_done(&mut ctx) {
return TimeoutOr::value(ctx.result.take().transpose());
if self.evaluate_done(context_locked) {
return Ok(core::mem::take(&mut context_locked.result));
}
}
@ -316,28 +460,67 @@ where
let mut unord = FuturesUnordered::new();
{
// Spin up 'fanout' tasks to process the fanout
for _ in 0..self.fanout {
let h = self.fanout_processor();
for _ in 0..self.fanout_tasks {
let h = self.fanout_processor(&context);
unord.push(h);
}
}
// Wait for them to complete
timeout(
match timeout(
timeout_ms,
async {
while let Some(is_done) = unord.next().in_current_span().await {
loop {
if let Some(res) = unord.next().in_current_span().await {
match res {
Ok(is_done) => {
if is_done {
break;
break Ok(());
}
}
Err(e) => {
break Err(e);
}
}
} else {
break Ok(());
}
}
}
.in_current_span(),
)
.await
.into_timeout_or()
.map(|_| {
// Finished, return whatever value we came up with
self.context.lock().result.take().transpose()
})
{
Ok(Ok(())) => {
// Finished, either by exhaustion or consensus,
// time to return whatever value we came up with
let context_locked = &mut *context.lock();
// Print final queue
veilid_log!(self debug "Finished FanoutQueue: {}", context_locked.fanout_queue);
return Ok(core::mem::take(&mut context_locked.result));
}
Ok(Err(e)) => {
// Fanout died with an error
return Err(e);
}
Err(_) => {
// Timeout, do one last evaluate with remaining nodes in timeout state
let context_locked = &mut *context.lock();
context_locked.fanout_queue.all_unfinished_to_timeout();
// Print final queue
veilid_log!(self debug "Timeout FanoutQueue: {}", context_locked.fanout_queue);
// Final evaluate
if self.evaluate_done(context_locked) {
// Last-chance value returned at timeout
return Ok(core::mem::take(&mut context_locked.result));
}
// We definitely weren't done, so this is just a plain timeout
let mut result = core::mem::take(&mut context_locked.result);
result.kind = FanoutResultKind::Timeout;
return Ok(result);
}
}
}
}

299
veilid-core/src/rpc_processor/fanout/fanout_queue.rs
View file

@ -4,93 +4,270 @@
/// When passing in a 'cleanup' function, if it sorts the queue, the 'first' items in the queue are the 'next' out.
use super::*;
#[derive(Debug)]
pub struct FanoutQueue {
crypto_kind: CryptoKind,
current_nodes: VecDeque<NodeRef>,
returned_nodes: HashSet<TypedKey>,
impl_veilid_log_facility!("fanout");
impl_veilid_component_registry_accessor!(FanoutQueue<'_>);
/// The status of a particular node we fanned out to
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum FanoutNodeStatus {
/// Node that needs processing
Queued,
/// Node currently being processed
InProgress,
/// Node that timed out during processing
Timeout,
/// Node that rejected the query
Rejected,
/// Node that accepted the query with a current result
Accepted,
/// Node that accepted the query but had an older result
Stale,
/// Node that has been disqualified for being too far away from the key
Disqualified,
}
impl FanoutQueue {
// Create a queue for fanout candidates that have a crypto-kind compatible node id
pub fn new(crypto_kind: CryptoKind) -> Self {
#[derive(Debug, Clone)]
pub struct FanoutNode {
pub node_ref: NodeRef,
pub status: FanoutNodeStatus,
}
pub type FanoutQueueSort<'a> = Box<dyn Fn(&TypedKey, &TypedKey) -> core::cmp::Ordering + Send + 'a>;
pub struct FanoutQueue<'a> {
/// Link back to veilid component registry for logging
registry: VeilidComponentRegistry,
/// Crypto kind in use for this queue
crypto_kind: CryptoKind,
/// The status of all the nodes we have added so far
nodes: HashMap<TypedKey, FanoutNode>,
/// Closer nodes to the record key are at the front of the list
sorted_nodes: Vec<TypedKey>,
/// The sort function to use for the nodes
node_sort: FanoutQueueSort<'a>,
/// The channel to receive work requests to process
sender: flume::Sender<flume::Sender<NodeRef>>,
receiver: flume::Receiver<flume::Sender<NodeRef>>,
/// Consensus count to use
consensus_count: usize,
}
impl fmt::Debug for FanoutQueue<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("FanoutQueue")
.field("crypto_kind", &self.crypto_kind)
.field("nodes", &self.nodes)
.field("sorted_nodes", &self.sorted_nodes)
// .field("node_sort", &self.node_sort)
.field("sender", &self.sender)
.field("receiver", &self.receiver)
.finish()
}
}
impl fmt::Display for FanoutQueue<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"nodes:\n{}",
self.sorted_nodes
.iter()
.map(|x| format!("{}: {:?}", x, self.nodes.get(x).unwrap().status))
.collect::<Vec<_>>()
.join("\n")
)
}
}
impl<'a> FanoutQueue<'a> {
/// Create a queue for fanout candidates that have a crypto-kind compatible node id
pub fn new(
registry: VeilidComponentRegistry,
crypto_kind: CryptoKind,
node_sort: FanoutQueueSort<'a>,
consensus_count: usize,
) -> Self {
let (sender, receiver) = flume::unbounded();
Self {
registry,
crypto_kind,
current_nodes: VecDeque::new(),
returned_nodes: HashSet::new(),
nodes: HashMap::new(),
sorted_nodes: Vec::new(),
node_sort,
sender,
receiver,
consensus_count,
}
}
// Add new nodes to list of fanout candidates
// Run a cleanup routine afterwards to trim down the list of candidates so it doesn't grow too large
pub fn add<F: FnOnce(&[NodeRef]) -> Vec<NodeRef>>(
&mut self,
new_nodes: &[NodeRef],
cleanup: F,
) {
for nn in new_nodes {
/// Ask for more work when some is ready
/// When work is ready it will be sent to work_sender so it can be received
/// by the worker
pub fn request_work(&mut self, work_sender: flume::Sender<NodeRef>) {
let _ = self.sender.send(work_sender);
// Send whatever work is available immediately
self.send_more_work();
}
/// Add new nodes to a filtered and sorted list of fanout candidates
pub fn add(&mut self, new_nodes: &[NodeRef]) {
for node_ref in new_nodes {
// Ensure the node has a comparable key with our current crypto kind
let Some(key) = nn.node_ids().get(self.crypto_kind) else {
let Some(key) = node_ref.node_ids().get(self.crypto_kind) else {
continue;
};
// Check if we have already done this node before (only one call per node ever)
if self.returned_nodes.contains(&key) {
// Check if we have already seen this node before (only one call per node ever)
if self.nodes.contains_key(&key) {
continue;
}
// Make sure the new node isnt already in the list
let mut dup = false;
for cn in &self.current_nodes {
if cn.same_entry(nn) {
dup = true;
break;
}
}
if !dup {
// Add the new node
self.current_nodes.push_back(nn.clone());
}
self.nodes.insert(
key,
FanoutNode {
node_ref: node_ref.clone(),
status: FanoutNodeStatus::Queued,
},
);
self.sorted_nodes.push(key);
}
// Make sure the deque is a single slice
self.current_nodes.make_contiguous();
// Sort the node list
self.sorted_nodes.sort_by(&self.node_sort);
// Sort and trim the candidate set
self.current_nodes =
VecDeque::from_iter(cleanup(self.current_nodes.as_slices().0).iter().cloned());
// Disqualify any nodes that can be
self.disqualify();
event!(target: "fanout", Level::DEBUG,
"FanoutQueue::add:\n current_nodes={{\n{}}}\n returned_nodes={{\n{}}}\n",
self.current_nodes
.iter()
.map(|x| format!(" {}", x))
veilid_log!(self debug
"FanoutQueue::add:\n new_nodes={{\n{}}}\n nodes={{\n{}}}\n",
new_nodes.iter().map(|x| format!(" {}", x))
.collect::<Vec<String>>()
.join(",\n"),
self.returned_nodes
self.sorted_nodes
.iter()
.map(|x| format!(" {}", x))
.map(|x| format!(" {:?}", self.nodes.get(x).unwrap()))
.collect::<Vec<String>>()
.join(",\n")
);
}
// Return next fanout candidate
pub fn next(&mut self) -> Option<NodeRef> {
let cn = self.current_nodes.pop_front()?;
self.current_nodes.make_contiguous();
let key = cn.node_ids().get(self.crypto_kind).unwrap();
// Ensure we don't return this node again
self.returned_nodes.insert(key);
event!(target: "fanout", Level::DEBUG,
"FanoutQueue::next: => {}", cn);
Some(cn)
/// Send next fanout candidates if available to whatever workers are ready
pub fn send_more_work(&mut self) {
// Get the next work and send it along
let registry = self.registry();
for x in &mut self.sorted_nodes {
// If there are no work receivers left then we should stop trying to send
if self.receiver.is_empty() {
break;
}
// Get a slice of all the current fanout candidates
pub fn nodes(&self) -> &[NodeRef] {
self.current_nodes.as_slices().0
let node = self.nodes.get_mut(x).unwrap();
if matches!(node.status, FanoutNodeStatus::Queued) {
// Send node to a work request
while let Ok(work_sender) = self.receiver.try_recv() {
let node_ref = node.node_ref.clone();
if work_sender.send(node_ref).is_ok() {
// Queued -> InProgress
node.status = FanoutNodeStatus::InProgress;
veilid_log!(registry debug "FanoutQueue::next: => {}", node.node_ref);
break;
}
}
}
}
}
/// Transition node InProgress -> Timeout
pub fn timeout(&mut self, node_ref: NodeRef) {
let key = node_ref.node_ids().get(self.crypto_kind).unwrap();
let node = self.nodes.get_mut(&key).unwrap();
assert_eq!(node.status, FanoutNodeStatus::InProgress);
node.status = FanoutNodeStatus::Timeout;
}
/// Transition node InProgress -> Rejected
pub fn rejected(&mut self, node_ref: NodeRef) {
let key = node_ref.node_ids().get(self.crypto_kind).unwrap();
let node = self.nodes.get_mut(&key).unwrap();
assert_eq!(node.status, FanoutNodeStatus::InProgress);
node.status = FanoutNodeStatus::Rejected;
self.disqualify();
}
/// Transition node InProgress -> Accepted
pub fn accepted(&mut self, node_ref: NodeRef) {
let key = node_ref.node_ids().get(self.crypto_kind).unwrap();
let node = self.nodes.get_mut(&key).unwrap();
assert_eq!(node.status, FanoutNodeStatus::InProgress);
node.status = FanoutNodeStatus::Accepted;
}
/// Transition node InProgress -> Stale
pub fn stale(&mut self, node_ref: NodeRef) {
let key = node_ref.node_ids().get(self.crypto_kind).unwrap();
let node = self.nodes.get_mut(&key).unwrap();
assert_eq!(node.status, FanoutNodeStatus::InProgress);
node.status = FanoutNodeStatus::Stale;
}
/// Transition all Accepted -> Queued, in the event a newer value for consensus is found and we want to try again
pub fn all_accepted_to_queued(&mut self) {
for node in &mut self.nodes {
if matches!(node.1.status, FanoutNodeStatus::Accepted) {
node.1.status = FanoutNodeStatus::Queued;
}
}
}
/// Transition all Accepted -> Stale, in the event a newer value for consensus is found but we don't want to try again
pub fn all_accepted_to_stale(&mut self) {
for node in &mut self.nodes {
if matches!(node.1.status, FanoutNodeStatus::Accepted) {
node.1.status = FanoutNodeStatus::Stale;
}
}
}
/// Transition all Queued | InProgress -> Timeout, in the event that the fanout is being cut short by a timeout
pub fn all_unfinished_to_timeout(&mut self) {
for node in &mut self.nodes {
if matches!(
node.1.status,
FanoutNodeStatus::Queued | FanoutNodeStatus::InProgress
) {
node.1.status = FanoutNodeStatus::Timeout;
}
}
}
/// Transition Queued -> Disqualified that are too far away from the record key
fn disqualify(&mut self) {
let mut consecutive_rejections = 0usize;
let mut rejected_consensus = false;
for node_id in &self.sorted_nodes {
let node = self.nodes.get_mut(node_id).unwrap();
if !rejected_consensus {
if matches!(node.status, FanoutNodeStatus::Rejected) {
consecutive_rejections += 1;
if consecutive_rejections >= self.consensus_count {
rejected_consensus = true;
}
continue;
} else {
consecutive_rejections = 0;
}
} else if matches!(node.status, FanoutNodeStatus::Queued) {
node.status = FanoutNodeStatus::Disqualified;
}
}
}
/// Review the nodes in the queue
pub fn with_nodes<R, F: FnOnce(&HashMap<TypedKey, FanoutNode>, &[TypedKey]) -> R>(
&self,
func: F,
) -> R {
func(&self.nodes, &self.sorted_nodes)
}
}

3
veilid-core/src/rpc_processor/fanout/mod.rs
View file

@ -2,7 +2,6 @@ mod fanout_call;
mod fanout_queue;
pub(crate) use fanout_call::*;
pub(crate) use fanout_queue::*;
use super::*;
use fanout_queue::*;

47
veilid-core/src/rpc_processor/mod.rs
View file

@ -379,40 +379,57 @@ impl RPCProcessor {
}
// Routine to call to generate fanout
let result = Arc::new(Mutex::new(Option::<NodeRef>::None));
let registry = self.registry();
let call_routine = Arc::new(move |next_node: NodeRef| {
let registry = registry.clone();
Box::pin(async move {
let this = registry.rpc_processor();
let v = network_result_try!(
this.rpc_call_find_node(
match this
.rpc_call_find_node(
Destination::direct(next_node.routing_domain_filtered(routing_domain))
.with_safety(safety_selection),
node_id,
vec![],
)
.await?
);
Ok(NetworkResult::value(FanoutCallOutput {
{
NetworkResult::Timeout => Ok(FanoutCallOutput {
peer_info_list: vec![],
disposition: FanoutCallDisposition::Timeout,
}),
NetworkResult::ServiceUnavailable(_)
| NetworkResult::NoConnection(_)
| NetworkResult::AlreadyExists(_)
| NetworkResult::InvalidMessage(_) => Ok(FanoutCallOutput {
peer_info_list: vec![],
disposition: FanoutCallDisposition::Rejected,
}),
NetworkResult::Value(v) => Ok(FanoutCallOutput {
peer_info_list: v.answer,
}))
disposition: FanoutCallDisposition::Accepted,
}),
}
}) as PinBoxFuture<FanoutCallResult>
});
// Routine to call to check if we're done at each step
let check_done = Arc::new(move |_: &[NodeRef]| {
let result2 = result.clone();
let check_done = Arc::new(move |_: &FanoutResult| -> bool {
let Ok(Some(nr)) = routing_table.lookup_node_ref(node_id) else {
return None;
return false;
};
// ensure we have some dial info for the entry already,
// and that the node is still alive
// if not, we should keep looking for better info
if nr.state(Timestamp::now()).is_alive() && nr.has_any_dial_info() {
return Some(nr);
*result2.lock() = Some(nr);
return true;
}
None
false
});
// Call the fanout
@ -422,13 +439,23 @@ impl RPCProcessor {
node_id,
count,
fanout,
0,
timeout_us,
empty_fanout_node_info_filter(),
call_routine,
check_done,
);
fanout_call.run(vec![]).await
match fanout_call.run(vec![]).await {
Ok(fanout_result) => {
if matches!(fanout_result.kind, FanoutResultKind::Timeout) {
TimeoutOr::timeout()
} else {
TimeoutOr::value(Ok(result.lock().take()))
}
}
Err(e) => TimeoutOr::value(Err(e)),
}
}
/// Search the DHT for a specific node corresponding to a key unless we

2
veilid-core/src/rpc_processor/rpc_app_call.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
// Sends a high level app request and wait for response
// Can be sent via all methods including relays and routes

2
veilid-core/src/rpc_processor/rpc_app_message.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
// Sends a high level app message
// Can be sent via all methods including relays and routes

2
veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
#[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
pub(super) async fn process_cancel_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

2
veilid-core/src/rpc_processor/rpc_complete_tunnel.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
#[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
pub(super) async fn process_complete_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

2
veilid-core/src/rpc_processor/rpc_find_block.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
#[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
pub(super) async fn process_find_block_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

2
veilid-core/src/rpc_processor/rpc_find_node.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
/// Send FindNodeQ RPC request, receive FindNodeA answer
/// Can be sent via all methods including relays

12
veilid-core/src/rpc_processor/rpc_get_value.rs
View file

@ -1,6 +1,8 @@
use super::*;
use crate::storage_manager::{SignedValueData, SignedValueDescriptor};
impl_veilid_log_facility!("rpc");
#[derive(Clone, Debug)]
pub struct GetValueAnswer {
pub value: Option<SignedValueData>,
@ -78,7 +80,7 @@ impl RPCProcessor {
crypto_kind: vcrypto.kind(),
});
veilid_log!(self debug "{}", debug_string);
veilid_log!(self debug target: "dht", "{}", debug_string);
let waitable_reply = network_result_try!(
self.question(dest.clone(), question, Some(question_context))
@ -128,13 +130,13 @@ impl RPCProcessor {
dest
);
veilid_log!(self debug "{}", debug_string_answer);
veilid_log!(self debug target: "dht", "{}", debug_string_answer);
let peer_ids: Vec<String> = peers
.iter()
.filter_map(|p| p.node_ids().get(key.kind).map(|k| k.to_string()))
.collect();
veilid_log!(self debug "Peers: {:#?}", peer_ids);
veilid_log!(self debug target: "dht", "Peers: {:#?}", peer_ids);
}
// Validate peers returned are, in fact, closer to the key than the node we sent this to
@ -228,7 +230,7 @@ impl RPCProcessor {
msg.header.direct_sender_node_id()
);
veilid_log!(self debug "{}", debug_string);
veilid_log!(self debug target: "dht", "{}", debug_string);
}
// See if we would have accepted this as a set
@ -278,7 +280,7 @@ impl RPCProcessor {
msg.header.direct_sender_node_id()
);
veilid_log!(self debug "{}", debug_string_answer);
veilid_log!(self debug target: "dht", "{}", debug_string_answer);
}
// Make GetValue answer

12
veilid-core/src/rpc_processor/rpc_inspect_value.rs
View file

@ -1,6 +1,8 @@
use super::*;
use crate::storage_manager::SignedValueDescriptor;
impl_veilid_log_facility!("rpc");
#[derive(Clone, Debug)]
pub struct InspectValueAnswer {
pub seqs: Vec<ValueSeqNum>,
@ -81,7 +83,7 @@ impl RPCProcessor {
crypto_kind: vcrypto.kind(),
});
veilid_log!(self debug "{}", debug_string);
veilid_log!(self debug target: "dht", "{}", debug_string);
let waitable_reply = network_result_try!(
self.question(dest.clone(), question, Some(question_context))
@ -118,13 +120,13 @@ impl RPCProcessor {
debug_seqs(&seqs)
);
veilid_log!(self debug "{}", debug_string_answer);
veilid_log!(self debug target: "dht", "{}", debug_string_answer);
let peer_ids: Vec<String> = peers
.iter()
.filter_map(|p| p.node_ids().get(key.kind).map(|k| k.to_string()))
.collect();
veilid_log!(self debug "Peers: {:#?}", peer_ids);
veilid_log!(self debug target: "dht", "Peers: {:#?}", peer_ids);
}
// Validate peers returned are, in fact, closer to the key than the node we sent this to
@ -209,7 +211,7 @@ impl RPCProcessor {
msg.header.direct_sender_node_id()
);
veilid_log!(self debug "{}", debug_string);
veilid_log!(self debug target: "dht", "{}", debug_string);
}
// See if we would have accepted this as a set
@ -247,7 +249,7 @@ impl RPCProcessor {
msg.header.direct_sender_node_id()
);
veilid_log!(self debug "{}", debug_string_answer);
veilid_log!(self debug target: "dht", "{}", debug_string_answer);
}
// Make InspectValue answer

2
veilid-core/src/rpc_processor/rpc_return_receipt.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
// Sends a unidirectional in-band return receipt
// Can be sent via all methods including relays and routes

2
veilid-core/src/rpc_processor/rpc_route.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
#[instrument(level = "trace", target = "rpc", skip_all, err)]
async fn process_route_safety_route_hop(

18
veilid-core/src/rpc_processor/rpc_set_value.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
#[derive(Clone, Debug)]
pub struct SetValueAnswer {
pub set: bool,
@ -58,10 +60,11 @@ impl RPCProcessor {
};
let debug_string = format!(
"OUT ==> SetValueQ({} #{} len={} writer={}{}) => {}",
"OUT ==> SetValueQ({} #{} len={} seq={} writer={}{}) => {}",
key,
subkey,
value.value_data().data().len(),
value.value_data().seq(),
value.value_data().writer(),
if send_descriptor { " +senddesc" } else { "" },
dest
@ -89,7 +92,7 @@ impl RPCProcessor {
});
if debug_target_enabled!("dht") {
veilid_log!(self debug "{}", debug_string);
veilid_log!(self debug target: "dht", "{}", debug_string);
}
let waitable_reply = network_result_try!(
@ -123,8 +126,9 @@ impl RPCProcessor {
.as_ref()
.map(|v| {
format!(
" len={} writer={}",
" len={} seq={} writer={}",
v.value_data().data().len(),
v.value_data().seq(),
v.value_data().writer(),
)
})
@ -140,13 +144,13 @@ impl RPCProcessor {
dest,
);
veilid_log!(self debug "{}", debug_string_answer);
veilid_log!(self debug target: "dht", "{}", debug_string_answer);
let peer_ids: Vec<String> = peers
.iter()
.filter_map(|p| p.node_ids().get(key.kind).map(|k| k.to_string()))
.collect();
veilid_log!(self debug "Peers: {:#?}", peer_ids);
veilid_log!(self debug target: "dht", "Peers: {:#?}", peer_ids);
}
// Validate peers returned are, in fact, closer to the key than the node we sent this to
@ -244,7 +248,7 @@ impl RPCProcessor {
msg.header.direct_sender_node_id()
);
veilid_log!(self debug "{}", debug_string);
veilid_log!(self debug target: "dht", "{}", debug_string);
// If there are less than 'set_value_count' peers that are closer, then store here too
let set_value_count = self
@ -296,7 +300,7 @@ impl RPCProcessor {
msg.header.direct_sender_node_id()
);
veilid_log!(self debug "{}", debug_string_answer);
veilid_log!(self debug target: "dht", "{}", debug_string_answer);
}
// Make SetValue answer

2
veilid-core/src/rpc_processor/rpc_signal.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
// Sends a unidirectional signal to a node
// Can be sent via relays but not routes. For routed 'signal' like capabilities, use AppMessage.

2
veilid-core/src/rpc_processor/rpc_start_tunnel.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
#[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
pub(super) async fn process_start_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

2
veilid-core/src/rpc_processor/rpc_status.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Default)]
pub struct StatusResult {
pub opt_sender_info: Option<SenderInfo>,

2
veilid-core/src/rpc_processor/rpc_supply_block.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
#[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)]
pub(super) async fn process_supply_block_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {

2
veilid-core/src/rpc_processor/rpc_validate_dial_info.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
// Can only be sent directly, not via relays or routes
#[instrument(level = "trace", target = "rpc", skip(self), ret, err)]

4
veilid-core/src/rpc_processor/rpc_value_changed.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
impl RPCProcessor {
// Sends a dht value change notification
// Can be sent via all methods including relays and routes but never over a safety route
@ -88,7 +90,7 @@ impl RPCProcessor {
msg.header.direct_sender_node_id(),
);
veilid_log!(self debug "{}", debug_string_stmt);
veilid_log!(self debug target: "dht", "{}", debug_string_stmt);
}
// Save the subkey, creating a new record if necessary

25
veilid-core/src/rpc_processor/rpc_watch_value.rs
View file

@ -1,5 +1,7 @@
use super::*;
impl_veilid_log_facility!("rpc");
#[derive(Clone, Debug)]
pub struct WatchValueAnswer {
pub accepted: bool,
@ -85,7 +87,7 @@ impl RPCProcessor {
RPCQuestionDetail::WatchValueQ(Box::new(watch_value_q)),
);
veilid_log!(self debug "{}", debug_string);
veilid_log!(self debug target: "dht", "{}", debug_string);
let waitable_reply =
network_result_try!(self.question(dest.clone(), question, None).await?);
@ -122,13 +124,13 @@ impl RPCProcessor {
dest
);
veilid_log!(self debug "{}", debug_string_answer);
veilid_log!(self debug target: "dht", "{}", debug_string_answer);
let peer_ids: Vec<String> = peers
.iter()
.filter_map(|p| p.node_ids().get(key.kind).map(|k| k.to_string()))
.collect();
veilid_log!(self debug "Peers: {:#?}", peer_ids);
veilid_log!(self debug target: "dht", "Peers: {:#?}", peer_ids);
}
// Validate accepted requests
@ -249,7 +251,7 @@ impl RPCProcessor {
watcher
);
veilid_log!(self debug "{}", debug_string);
veilid_log!(self debug target: "dht", "{}", debug_string);
}
// Get the nodes that we know about that are closer to the the key than our own node
@ -268,8 +270,7 @@ impl RPCProcessor {
(false, 0, watch_id.unwrap_or_default())
} else {
// Accepted, lets try to watch or cancel it
let params = WatchParameters {
let params = InboundWatchParameters {
subkeys: subkeys.clone(),
expiration: Timestamp::new(expiration),
count,
@ -280,19 +281,19 @@ impl RPCProcessor {
// See if we have this record ourselves, if so, accept the watch
let storage_manager = self.storage_manager();
let watch_result = network_result_try!(storage_manager
.inbound_watch_value(key, params, watch_id,)
.inbound_watch_value(key, params, watch_id)
.await
.map_err(RPCError::internal)?);
// Encode the watch result
// Rejections and cancellations are treated the same way by clients
let (ret_expiration, ret_watch_id) = match watch_result {
WatchResult::Created { id, expiration } => (expiration.as_u64(), id),
WatchResult::Changed { expiration } => {
InboundWatchResult::Created { id, expiration } => (expiration.as_u64(), id),
InboundWatchResult::Changed { expiration } => {
(expiration.as_u64(), watch_id.unwrap_or_default())
}
WatchResult::Cancelled => (0, watch_id.unwrap_or_default()),
WatchResult::Rejected => (0, watch_id.unwrap_or_default()),
InboundWatchResult::Cancelled => (0, watch_id.unwrap_or_default()),
InboundWatchResult::Rejected => (0, watch_id.unwrap_or_default()),
};
(true, ret_expiration, ret_watch_id)
};
@ -309,7 +310,7 @@ impl RPCProcessor {
msg.header.direct_sender_node_id()
);
veilid_log!(self debug "{}", debug_string_answer);
veilid_log!(self debug target: "dht", "{}", debug_string_answer);
}
// Make WatchValue answer

2
veilid-core/src/rpc_processor/rpc_worker.rs
View file

@ -3,6 +3,8 @@ use stop_token::future::FutureExt as _;
use super::*;
impl_veilid_log_facility!("rpc");
#[derive(Debug)]
pub(super) enum RPCWorkerRequestKind {
Message { message_encoded: MessageEncoded },

19
veilid-core/src/storage_manager/debug.rs
View file

@ -24,16 +24,14 @@ impl StorageManager {
} else {
"".to_owned()
};
let watch = if let Some(w) = v.active_watch() {
format!(" watch: {:?}\n", w)
} else {
"".to_owned()
};
out += &format!(" {} {}{}\n", k, writer, watch);
out += &format!(" {} {}\n", k, writer);
}
format!("{}]\n", out)
}
pub async fn debug_watched_records(&self) -> String {
let inner = self.inner.lock().await;
inner.outbound_watch_manager.to_string()
}
pub async fn debug_offline_records(&self) -> String {
let inner = self.inner.lock().await;
let Some(local_record_store) = &inner.local_record_store else {
@ -53,6 +51,9 @@ impl StorageManager {
pub async fn purge_local_records(&self, reclaim: Option<usize>) -> String {
let mut inner = self.inner.lock().await;
if !inner.opened_records.is_empty() {
return "records still opened".to_owned();
}
let Some(local_record_store) = &mut inner.local_record_store else {
return "not initialized".to_owned();
};
@ -64,6 +65,9 @@ impl StorageManager {
}
pub async fn purge_remote_records(&self, reclaim: Option<usize>) -> String {
let mut inner = self.inner.lock().await;
if !inner.opened_records.is_empty() {
return "records still opened".to_owned();
}
let Some(remote_record_store) = &mut inner.remote_record_store else {
return "not initialized".to_owned();
};
@ -72,6 +76,7 @@ impl StorageManager {
.await;
format!("Remote records purged: reclaimed {} bytes", reclaimed)
}
pub async fn debug_local_record_subkey_info(
&self,
key: TypedKey,

174
veilid-core/src/storage_manager/get_value.rs
View file

@ -6,8 +6,6 @@ impl_veilid_log_facility!("stor");
struct OutboundGetValueContext {
/// The latest value of the subkey, may be the value passed in
pub value: Option<Arc<SignedValueData>>,
/// The nodes that have returned the value so far (up to the consensus count)
pub value_nodes: Vec<NodeRef>,
/// The descriptor if we got a fresh one or empty if no descriptor was needed
pub descriptor: Option<Arc<SignedValueDescriptor>>,
/// The parsed schema from the descriptor if we have one
@ -17,7 +15,7 @@ struct OutboundGetValueContext {
}
/// The result of the outbound_get_value operation
#[derive(Clone, Debug)]
#[derive(Debug)]
pub(super) struct OutboundGetValueResult {
/// Fanout result
pub fanout_result: FanoutResult,
@ -74,23 +72,23 @@ impl StorageManager {
// Make do-get-value answer context
let context = Arc::new(Mutex::new(OutboundGetValueContext {
value: last_get_result.opt_value,
value_nodes: vec![],
descriptor: last_get_result.opt_descriptor.clone(),
schema,
send_partial_update: false,
send_partial_update: true,
}));
// Routine to call to generate fanout
let call_routine = {
let context = context.clone();
let registry = self.registry();
Arc::new(move |next_node: NodeRef| {
Arc::new(
move |next_node: NodeRef| -> PinBoxFutureStatic<FanoutCallResult> {
let context = context.clone();
let registry = registry.clone();
let last_descriptor = last_get_result.opt_descriptor.clone();
Box::pin(async move {
let rpc_processor = registry.rpc_processor();
let gva = network_result_try!(
let gva = match
rpc_processor
.rpc_call_get_value(
Destination::direct(next_node.routing_domain_filtered(routing_domain))
@ -99,8 +97,18 @@ impl StorageManager {
subkey,
last_descriptor.map(|x| (*x).clone()),
)
.await?
);
.await? {
NetworkResult::Timeout => {
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Timeout});
}
NetworkResult::ServiceUnavailable(_) |
NetworkResult::NoConnection(_) |
NetworkResult::AlreadyExists(_) |
NetworkResult::InvalidMessage(_) => {
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
NetworkResult::Value(v) => v
};
let mut ctx = context.lock();
// Keep the descriptor if we got one. If we had a last_descriptor it will
@ -110,7 +118,8 @@ impl StorageManager {
let schema = match descriptor.schema() {
Ok(v) => v,
Err(e) => {
return Ok(NetworkResult::invalid_message(e));
veilid_log!(registry debug target:"network_result", "GetValue returned an invalid descriptor: {}", e);
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
};
ctx.schema = Some(schema);
@ -122,8 +131,7 @@ impl StorageManager {
let Some(value) = gva.answer.value else {
// Return peers if we have some
veilid_log!(registry debug target:"network_result", "GetValue returned no value, fanout call returned peers {}", gva.answer.peers.len());
return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list: gva.answer.peers}))
return Ok(FanoutCallOutput{peer_info_list: gva.answer.peers, disposition: FanoutCallDisposition::Rejected});
};
veilid_log!(registry debug "GetValue got value back: len={} seq={}", value.value_data().data().len(), value.value_data().seq());
@ -133,9 +141,7 @@ impl StorageManager {
else {
// Got a value but no descriptor for it
// Move to the next node
return Ok(NetworkResult::invalid_message(
"Got value with no descriptor",
));
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
};
// Validate with schema
@ -146,51 +152,52 @@ impl StorageManager {
) {
// Validation failed, ignore this value
// Move to the next node
return Ok(NetworkResult::invalid_message(format!(
"Schema validation failed on subkey {}",
subkey
)));
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
// If we have a prior value, see if this is a newer sequence number
if let Some(prior_value) = &ctx.value {
let disposition = if let Some(prior_value) = &ctx.value {
let prior_seq = prior_value.value_data().seq();
let new_seq = value.value_data().seq();
if new_seq == prior_seq {
// If sequence number is the same, the data should be the same
if prior_value.value_data() != value.value_data() {
// Move to the next node
return Ok(NetworkResult::invalid_message(
"value data mismatch",
));
}
// Value data mismatch means skip this node
// This is okay because even the conflicting value is signed,
// so the application just needs to push a newer value
FanoutCallDisposition::Stale
} else {
// Increase the consensus count for the existing value
ctx.value_nodes.push(next_node);
FanoutCallDisposition::Accepted
}
} else if new_seq > prior_seq {
// If the sequence number is greater, start over with the new value
ctx.value = Some(Arc::new(value));
// One node has shown us this value so far
ctx.value_nodes = vec![next_node];
// Send an update since the value changed
ctx.send_partial_update = true;
// Restart the consensus since we have a new value, but
// don't retry nodes we've already seen because they will return
// the same answer
FanoutCallDisposition::AcceptedNewer
} else {
// If the sequence number is older, ignore it
FanoutCallDisposition::Stale
}
} else {
// If we have no prior value, keep it
ctx.value = Some(Arc::new(value));
// One node has shown us this value so far
ctx.value_nodes = vec![next_node];
// Send an update since the value changed
ctx.send_partial_update = true;
}
// No value was returned
FanoutCallDisposition::Accepted
};
// Return peers if we have some
veilid_log!(registry debug target:"network_result", "GetValue fanout call returned peers {}", gva.answer.peers.len());
Ok(NetworkResult::value(FanoutCallOutput{peer_info_list: gva.answer.peers}))
Ok(FanoutCallOutput{peer_info_list: gva.answer.peers, disposition})
}.instrument(tracing::trace_span!("outbound_get_value fanout routine"))) as PinBoxFuture<FanoutCallResult>
})
},
)
};
// Routine to call to check if we're done at each step
@ -198,37 +205,44 @@ impl StorageManager {
let context = context.clone();
let out_tx = out_tx.clone();
let registry = self.registry();
Arc::new(move |_closest_nodes: &[NodeRef]| {
Arc::new(move |fanout_result: &FanoutResult| -> bool {
let mut ctx = context.lock();
// send partial update if desired
if ctx.send_partial_update {
match fanout_result.kind {
FanoutResultKind::Incomplete => {
// Send partial update if desired, if we've gotten at least one consensus node
if ctx.send_partial_update && !fanout_result.consensus_nodes.is_empty() {
ctx.send_partial_update = false;
// return partial result
let fanout_result = FanoutResult {
kind: FanoutResultKind::Partial,
value_nodes: ctx.value_nodes.clone(),
};
if let Err(e) = out_tx.send(Ok(OutboundGetValueResult {
fanout_result,
// Return partial result
let out = OutboundGetValueResult {
fanout_result: fanout_result.clone(),
get_result: GetResult {
opt_value: ctx.value.clone(),
opt_descriptor: ctx.descriptor.clone(),
},
})) {
};
veilid_log!(registry debug "Sending partial GetValue result: {:?}", out);
if let Err(e) = out_tx.send(Ok(out)) {
veilid_log!(registry debug "Sending partial GetValue result failed: {}", e);
}
}
// If we have reached sufficient consensus, return done
if ctx.value.is_some()
&& ctx.descriptor.is_some()
&& ctx.value_nodes.len() >= consensus_count
{
return Some(());
// Keep going
false
}
FanoutResultKind::Timeout | FanoutResultKind::Exhausted => {
// Signal we're done
true
}
FanoutResultKind::Consensus => {
assert!(
ctx.value.is_some() && ctx.descriptor.is_some(),
"should have gotten a value if we got consensus"
);
// Signal we're done
true
}
}
None
})
};
@ -244,21 +258,16 @@ impl StorageManager {
key,
key_count,
fanout,
consensus_count,
timeout_us,
capability_fanout_node_info_filter(vec![CAP_DHT]),
call_routine,
check_done,
);
let kind = match fanout_call.run(init_fanout_queue).await {
// If we don't finish in the timeout (too much time passed checking for consensus)
TimeoutOr::Timeout => FanoutResultKind::Timeout,
// If we finished with or without consensus (enough nodes returning the same value)
TimeoutOr::Value(Ok(Some(()))) => FanoutResultKind::Finished,
// If we ran out of nodes before getting consensus)
TimeoutOr::Value(Ok(None)) => FanoutResultKind::Exhausted,
// Failed
TimeoutOr::Value(Err(e)) => {
let fanout_result = match fanout_call.run(init_fanout_queue).await {
Ok(v) => v,
Err(e) => {
// If we finished with an error, return that
veilid_log!(registry debug "GetValue fanout error: {}", e);
if let Err(e) = out_tx.send(Err(e.into())) {
@ -268,20 +277,20 @@ impl StorageManager {
}
};
let ctx = context.lock();
let fanout_result = FanoutResult {
kind,
value_nodes: ctx.value_nodes.clone(),
};
veilid_log!(registry debug "GetValue Fanout: {:?}", fanout_result);
veilid_log!(registry debug "GetValue Fanout: {:#}", fanout_result);
if let Err(e) = out_tx.send(Ok(OutboundGetValueResult {
let out = {
let ctx = context.lock();
OutboundGetValueResult {
fanout_result,
get_result: GetResult {
opt_value: ctx.value.clone(),
opt_descriptor: ctx.descriptor.clone(),
},
})) {
}
};
if let Err(e) = out_tx.send(Ok(out)) {
veilid_log!(registry debug "Sending GetValue result failed: {}", e);
}
}
@ -316,18 +325,18 @@ impl StorageManager {
return false;
}
};
let is_partial = result.fanout_result.kind.is_partial();
let is_incomplete = result.fanout_result.kind.is_incomplete();
let value_data = match this.process_outbound_get_value_result(key, subkey, Some(last_seq), result).await {
Ok(Some(v)) => v,
Ok(None) => {
return is_partial;
return is_incomplete;
}
Err(e) => {
veilid_log!(this debug "Deferred fanout error: {}", e);
return false;
}
};
if is_partial {
if is_incomplete {
// If more partial results show up, don't send an update until we're done
return true;
}
@ -349,7 +358,7 @@ impl StorageManager {
#[instrument(level = "trace", target = "dht", skip_all)]
pub(super) async fn process_outbound_get_value_result(
&self,
key: TypedKey,
record_key: TypedKey,
subkey: ValueSubkey,
opt_last_seq: Option<u32>,
result: get_value::OutboundGetValueResult,
@ -360,13 +369,20 @@ impl StorageManager {
return Ok(None);
};
// Get cryptosystem
let crypto = self.crypto();
let Some(vcrypto) = crypto.get(record_key.kind) else {
apibail_generic!("unsupported cryptosystem");
};
// Keep the list of nodes that returned a value for later reference
let mut inner = self.inner.lock().await;
Self::process_fanout_results_inner(
&mut inner,
key,
core::iter::once((subkey, &result.fanout_result)),
&vcrypto,
record_key,
core::iter::once((ValueSubkeyRangeSet::single(subkey), result.fanout_result)),
false,
self.config()
.with(|c| c.network.dht.set_value_count as usize),
@ -376,10 +392,10 @@ impl StorageManager {
if Some(get_result_value.value_data().seq()) != opt_last_seq {
Self::handle_set_local_value_inner(
&mut inner,
key,
record_key,
subkey,
get_result_value.clone(),
WatchUpdateMode::UpdateAll,
InboundWatchUpdateMode::UpdateAll,
)
.await?;
}

133
veilid-core/src/storage_manager/inspect_value.rs
View file

@ -29,7 +29,9 @@ impl DescriptorInfo {
struct SubkeySeqCount {
/// The newest sequence number found for a subkey
pub seq: ValueSeqNum,
/// The nodes that have returned the value so far (up to the consensus count)
/// The set of nodes that had the most recent value for this subkey
pub consensus_nodes: Vec<NodeRef>,
/// The set of nodes that had any value for this subkey
pub value_nodes: Vec<NodeRef>,
}
@ -44,7 +46,7 @@ struct OutboundInspectValueContext {
/// The result of the outbound_get_value operation
pub(super) struct OutboundInspectValueResult {
/// Fanout results for each subkey
pub fanout_results: Vec<FanoutResult>,
pub subkey_fanout_results: Vec<FanoutResult>,
/// The inspection that was retrieved
pub inspect_result: InspectResult,
}
@ -110,6 +112,7 @@ impl StorageManager {
.iter()
.map(|s| SubkeySeqCount {
seq: *s,
consensus_nodes: vec![],
value_nodes: vec![],
})
.collect(),
@ -120,7 +123,8 @@ impl StorageManager {
let call_routine = {
let context = context.clone();
let registry = self.registry();
Arc::new(move |next_node: NodeRef| {
Arc::new(
move |next_node: NodeRef| -> PinBoxFutureStatic<FanoutCallResult> {
let context = context.clone();
let registry = registry.clone();
let opt_descriptor = local_inspect_result.opt_descriptor.clone();
@ -128,7 +132,7 @@ impl StorageManager {
Box::pin(async move {
let rpc_processor = registry.rpc_processor();
let iva = network_result_try!(
let iva = match
rpc_processor
.rpc_call_inspect_value(
Destination::direct(next_node.routing_domain_filtered(routing_domain)).with_safety(safety_selection),
@ -136,8 +140,19 @@ impl StorageManager {
subkeys.clone(),
opt_descriptor.map(|x| (*x).clone()),
)
.await?
);
.await? {
NetworkResult::Timeout => {
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Timeout});
}
NetworkResult::ServiceUnavailable(_) |
NetworkResult::NoConnection(_) |
NetworkResult::AlreadyExists(_) |
NetworkResult::InvalidMessage(_) => {
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
NetworkResult::Value(v) => v
};
let answer = iva.answer;
// Keep the descriptor if we got one. If we had a last_descriptor it will
@ -150,7 +165,8 @@ impl StorageManager {
match DescriptorInfo::new(Arc::new(descriptor.clone()), &subkeys) {
Ok(v) => v,
Err(e) => {
return Ok(NetworkResult::invalid_message(e));
veilid_log!(registry debug target:"network_result", "InspectValue returned an invalid descriptor: {}", e);
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
};
ctx.opt_descriptor_info = Some(descriptor_info);
@ -158,17 +174,20 @@ impl StorageManager {
}
// Keep the value if we got one and it is newer and it passes schema validation
if !answer.seqs.is_empty() {
veilid_log!(registry debug "Got seqs back: len={}", answer.seqs.len());
if answer.seqs.is_empty() {
veilid_log!(registry debug target:"network_result", "InspectValue returned no seq, fanout call returned peers {}", answer.peers.len());
return Ok(FanoutCallOutput{peer_info_list: answer.peers, disposition: FanoutCallDisposition::Rejected});
}
veilid_log!(registry debug target:"network_result", "Got seqs back: len={}", answer.seqs.len());
let mut ctx = context.lock();
// Ensure we have a schema and descriptor etc
let Some(descriptor_info) = &ctx.opt_descriptor_info else {
// Got a value but no descriptor for it
// Move to the next node
return Ok(NetworkResult::invalid_message(
"Got inspection with no descriptor",
));
veilid_log!(registry debug target:"network_result", "InspectValue returned a value with no descriptor invalid descriptor");
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
};
// Get number of subkeys from schema and ensure we are getting the
@ -177,11 +196,10 @@ impl StorageManager {
if answer.seqs.len() as u64 != descriptor_info.subkeys.len() as u64 {
// Not the right number of sequence numbers
// Move to the next node
return Ok(NetworkResult::invalid_message(format!(
"wrong number of seqs returned {} (wanted {})",
veilid_log!(registry debug target:"network_result", "wrong number of seqs returned {} (wanted {})",
answer.seqs.len(),
descriptor_info.subkeys.len()
)));
descriptor_info.subkeys.len());
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
// If we have a prior seqs list, merge in the new seqs
@ -192,18 +210,17 @@ impl StorageManager {
.map(|s| SubkeySeqCount {
seq: *s,
// One node has shown us the newest sequence numbers so far
value_nodes: if *s == ValueSeqNum::MAX {
vec![]
} else {
vec![next_node.clone()]
},
consensus_nodes: vec![next_node.clone()],
value_nodes: vec![next_node.clone()],
})
.collect();
} else {
if ctx.seqcounts.len() != answer.seqs.len() {
return Err(RPCError::internal(
"seqs list length should always be equal by now",
));
veilid_log!(registry debug target:"network_result", "seqs list length should always be equal by now: {} (wanted {})",
answer.seqs.len(),
ctx.seqcounts.len());
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
for pair in ctx.seqcounts.iter_mut().zip(answer.seqs.iter()) {
let ctx_seqcnt = pair.0;
@ -212,7 +229,7 @@ impl StorageManager {
// If we already have consensus for this subkey, don't bother updating it any more
// While we may find a better sequence number if we keep looking, this does not mimic the behavior
// of get and set unless we stop here
if ctx_seqcnt.value_nodes.len() >= consensus_count {
if ctx_seqcnt.consensus_nodes.len() >= consensus_count {
continue;
}
@ -225,42 +242,45 @@ impl StorageManager {
{
// One node has shown us the latest sequence numbers so far
ctx_seqcnt.seq = answer_seq;
ctx_seqcnt.value_nodes = vec![next_node.clone()];
ctx_seqcnt.consensus_nodes = vec![next_node.clone()];
} else if answer_seq == ctx_seqcnt.seq {
// Keep the nodes that showed us the latest values
ctx_seqcnt.consensus_nodes.push(next_node.clone());
}
}
ctx_seqcnt.value_nodes.push(next_node.clone());
}
}
}
}
}
// Return peers if we have some
veilid_log!(registry debug target:"network_result", "InspectValue fanout call returned peers {}", answer.peers.len());
Ok(NetworkResult::value(FanoutCallOutput { peer_info_list: answer.peers}))
// Inspect doesn't actually use the fanout queue consensus tracker
Ok(FanoutCallOutput { peer_info_list: answer.peers, disposition: FanoutCallDisposition::Accepted})
}.instrument(tracing::trace_span!("outbound_inspect_value fanout call"))) as PinBoxFuture<FanoutCallResult>
})
},
)
};
// Routine to call to check if we're done at each step
// For inspect, we are tracking consensus externally from the FanoutCall,
// for each subkey, rather than a single consensus, so the single fanoutresult
// that is passed in here is ignored in favor of our own per-subkey tracking
let check_done = {
let context = context.clone();
Arc::new(move |_closest_nodes: &[NodeRef]| {
Arc::new(move |_: &FanoutResult| {
// If we have reached sufficient consensus on all subkeys, return done
let ctx = context.lock();
let mut has_consensus = true;
for cs in ctx.seqcounts.iter() {
if cs.value_nodes.len() < consensus_count {
if cs.consensus_nodes.len() < consensus_count {
has_consensus = false;
break;
}
}
if !ctx.seqcounts.is_empty() && ctx.opt_descriptor_info.is_some() && has_consensus {
return Some(());
}
None
!ctx.seqcounts.is_empty() && ctx.opt_descriptor_info.is_some() && has_consensus
})
};
@ -271,46 +291,39 @@ impl StorageManager {
key,
key_count,
fanout,
consensus_count,
timeout_us,
capability_fanout_node_info_filter(vec![CAP_DHT]),
call_routine,
check_done,
);
let kind = match fanout_call.run(init_fanout_queue).await {
// If we don't finish in the timeout (too much time passed checking for consensus)
TimeoutOr::Timeout => FanoutResultKind::Timeout,
// If we finished with or without consensus (enough nodes returning the same value)
TimeoutOr::Value(Ok(Some(()))) => FanoutResultKind::Finished,
// If we ran out of nodes before getting consensus)
TimeoutOr::Value(Ok(None)) => FanoutResultKind::Exhausted,
// Failed
TimeoutOr::Value(Err(e)) => {
// If we finished with an error, return that
veilid_log!(self debug "InspectValue Fanout Error: {}", e);
return Err(e.into());
}
};
let fanout_result = fanout_call.run(init_fanout_queue).await?;
let ctx = context.lock();
let mut fanout_results = vec![];
let mut subkey_fanout_results = vec![];
for cs in &ctx.seqcounts {
let has_consensus = cs.value_nodes.len() >= consensus_count;
let fanout_result = FanoutResult {
let has_consensus = cs.consensus_nodes.len() >= consensus_count;
let subkey_fanout_result = FanoutResult {
kind: if has_consensus {
FanoutResultKind::Finished
FanoutResultKind::Consensus
} else {
kind
fanout_result.kind
},
consensus_nodes: cs.consensus_nodes.clone(),
value_nodes: cs.value_nodes.clone(),
};
fanout_results.push(fanout_result);
subkey_fanout_results.push(subkey_fanout_result);
}
veilid_log!(self debug "InspectValue Fanout ({:?}):\n{}", kind, debug_fanout_results(&fanout_results));
if subkey_fanout_results.len() == 1 {
veilid_log!(self debug "InspectValue Fanout: {:#}\n{:#}", fanout_result, subkey_fanout_results.first().unwrap());
} else {
veilid_log!(self debug "InspectValue Fanout: {:#}:\n{}", fanout_result, debug_fanout_results(&subkey_fanout_results));
}
Ok(OutboundInspectValueResult {
fanout_results,
subkey_fanout_results,
inspect_result: InspectResult {
subkeys: ctx
.opt_descriptor_info

467
veilid-core/src/storage_manager/mod.rs
View file

@ -1,6 +1,7 @@
mod debug;
mod get_value;
mod inspect_value;
mod outbound_watch_manager;
mod record_store;
mod set_value;
mod tasks;
@ -8,11 +9,12 @@ mod types;
mod watch_value;
use super::*;
use outbound_watch_manager::*;
use record_store::*;
use routing_table::*;
use rpc_processor::*;
pub use record_store::{WatchParameters, WatchResult};
pub use record_store::{InboundWatchParameters, InboundWatchResult};
pub use types::*;
@ -24,18 +26,26 @@ pub(crate) const MAX_SUBKEY_SIZE: usize = ValueData::MAX_LEN;
pub(crate) const MAX_RECORD_DATA_SIZE: usize = 1_048_576;
/// Frequency to flush record stores to disk
const FLUSH_RECORD_STORES_INTERVAL_SECS: u32 = 1;
/// Frequency to save metadata to disk
const SAVE_METADATA_INTERVAL_SECS: u32 = 30;
/// Frequency to check for offline subkeys writes to send to the network
const OFFLINE_SUBKEY_WRITES_INTERVAL_SECS: u32 = 5;
/// Frequency to send ValueChanged notifications to the network
const SEND_VALUE_CHANGES_INTERVAL_SECS: u32 = 1;
/// Frequency to check for dead nodes and routes for client-side active watches
const CHECK_ACTIVE_WATCHES_INTERVAL_SECS: u32 = 1;
/// Frequency to check for dead nodes and routes for client-side outbound watches
const CHECK_OUTBOUND_WATCHES_INTERVAL_SECS: u32 = 1;
/// Frequency to retry reconciliation of watches that are not at consensus
const RECONCILE_OUTBOUND_WATCHES_INTERVAL_SECS: u32 = 300;
/// How long before expiration to try to renew per-node watches
const RENEW_OUTBOUND_WATCHES_DURATION_SECS: u32 = 30;
/// Frequency to check for expired server-side watched records
const CHECK_WATCHED_RECORDS_INTERVAL_SECS: u32 = 1;
/// Table store table for storage manager metadata
const STORAGE_MANAGER_METADATA: &str = "storage_manager_metadata";
/// Storage manager metadata key name for offline subkey write persistence
const OFFLINE_SUBKEY_WRITES: &[u8] = b"offline_subkey_writes";
/// Outbound watch manager metadata key name for watch persistence
const OUTBOUND_WATCH_MANAGER: &[u8] = b"outbound_watch_manager";
#[derive(Debug, Clone)]
/// A single 'value changed' message to send
@ -61,6 +71,8 @@ struct StorageManagerInner {
pub offline_subkey_writes: HashMap<TypedKey, tasks::offline_subkey_writes::OfflineSubkeyWrite>,
/// Record subkeys that are currently being written to in the foreground
pub active_subkey_writes: HashMap<TypedKey, ValueSubkeyRangeSet>,
/// State management for outbound watches
pub outbound_watch_manager: OutboundWatchManager,
/// Storage manager metadata that is persistent, including copy of offline subkey writes
pub metadata_db: Option<TableDB>,
/// Background processing task (not part of attachment manager tick tree so it happens when detached too)
@ -76,6 +88,7 @@ impl fmt::Debug for StorageManagerInner {
.field("remote_record_store", &self.remote_record_store)
.field("offline_subkey_writes", &self.offline_subkey_writes)
.field("active_subkey_writes", &self.active_subkey_writes)
.field("outbound_watch_manager", &self.outbound_watch_manager)
//.field("metadata_db", &self.metadata_db)
//.field("tick_future", &self.tick_future)
.finish()
@ -87,17 +100,24 @@ pub(crate) struct StorageManager {
inner: AsyncMutex<StorageManagerInner>,
// Background processes
save_metadata_task: TickTask<EyreReport>,
flush_record_stores_task: TickTask<EyreReport>,
offline_subkey_writes_task: TickTask<EyreReport>,
send_value_changes_task: TickTask<EyreReport>,
check_active_watches_task: TickTask<EyreReport>,
check_watched_records_task: TickTask<EyreReport>,
check_outbound_watches_task: TickTask<EyreReport>,
check_inbound_watches_task: TickTask<EyreReport>,
// Anonymous watch keys
anonymous_watch_keys: TypedKeyPairGroup,
/// Deferred result processor
deferred_result_processor: DeferredStreamProcessor,
// Outbound watch operation lock
// Keeps changes to watches to one-at-a-time per record
outbound_watch_lock_table: AsyncTagLockTable<TypedKey>,
// Background operation processor
// for offline subkey writes, watch changes, and any other
// background operations the storage manager wants to perform
background_operation_processor: DeferredStreamProcessor,
}
impl fmt::Debug for StorageManager {
@ -116,7 +136,11 @@ impl fmt::Debug for StorageManager {
// "check_watched_records_task",
// &self.check_watched_records_task,
// )
.field("deferred_result_processor", &self.deferred_result_processor)
.field("outbound_watch_lock_table", &self.outbound_watch_lock_table)
.field(
"background_operation_processor",
&self.background_operation_processor,
)
.field("anonymous_watch_keys", &self.anonymous_watch_keys)
.finish()
}
@ -145,6 +169,7 @@ impl StorageManager {
registry,
inner: AsyncMutex::new(inner),
save_metadata_task: TickTask::new("save_metadata_task", SAVE_METADATA_INTERVAL_SECS),
flush_record_stores_task: TickTask::new(
"flush_record_stores_task",
FLUSH_RECORD_STORES_INTERVAL_SECS,
@ -157,17 +182,17 @@ impl StorageManager {
"send_value_changes_task",
SEND_VALUE_CHANGES_INTERVAL_SECS,
),
check_active_watches_task: TickTask::new(
check_outbound_watches_task: TickTask::new(
"check_active_watches_task",
CHECK_ACTIVE_WATCHES_INTERVAL_SECS,
CHECK_OUTBOUND_WATCHES_INTERVAL_SECS,
),
check_watched_records_task: TickTask::new(
check_inbound_watches_task: TickTask::new(
"check_watched_records_task",
CHECK_WATCHED_RECORDS_INTERVAL_SECS,
),
outbound_watch_lock_table: AsyncTagLockTable::new(),
anonymous_watch_keys,
deferred_result_processor: DeferredStreamProcessor::new(),
background_operation_processor: DeferredStreamProcessor::new(),
};
this.setup_tasks();
@ -240,7 +265,7 @@ impl StorageManager {
}
// Start deferred results processors
self.deferred_result_processor.init();
self.background_operation_processor.init();
Ok(())
}
@ -249,6 +274,9 @@ impl StorageManager {
async fn post_init_async(&self) -> EyreResult<()> {
let mut inner = self.inner.lock().await;
// Resolve outbound watch manager noderefs
inner.outbound_watch_manager.prepare(self.routing_table());
// Schedule tick
let registry = self.registry();
let tick_future = interval("storage manager tick", 1000, move || {
@ -286,7 +314,7 @@ impl StorageManager {
veilid_log!(self debug "starting storage manager shutdown");
// Stop deferred result processor
self.deferred_result_processor.terminate().await;
self.background_operation_processor.terminate().await;
// Terminate and release the storage manager
{
@ -320,6 +348,7 @@ impl StorageManager {
if let Some(metadata_db) = &inner.metadata_db {
let tx = metadata_db.transact();
tx.store_json(0, OFFLINE_SUBKEY_WRITES, &inner.offline_subkey_writes)?;
tx.store_json(0, OUTBOUND_WATCH_MANAGER, &inner.outbound_watch_manager)?;
tx.commit().await.wrap_err("failed to commit")?
}
Ok(())
@ -338,7 +367,19 @@ impl StorageManager {
}
Default::default()
}
};
inner.outbound_watch_manager = match metadata_db
.load_json(0, OUTBOUND_WATCH_MANAGER)
.await
{
Ok(v) => v.unwrap_or_default(),
Err(_) => {
if let Err(e) = metadata_db.delete(0, OUTBOUND_WATCH_MANAGER).await {
veilid_log!(self debug "outbound_watch_manager format changed, clearing: {}", e);
}
Default::default()
}
};
}
Ok(())
}
@ -362,21 +403,39 @@ impl StorageManager {
}
/// Get the set of nodes in our active watches
pub async fn get_active_watch_nodes(&self) -> Vec<Destination> {
pub async fn get_outbound_watch_nodes(&self) -> Vec<Destination> {
let inner = self.inner.lock().await;
inner
.opened_records
.values()
.filter_map(|v| {
v.active_watch().map(|aw| {
let mut out = vec![];
let mut node_set = HashSet::new();
for v in inner.outbound_watch_manager.outbound_watches.values() {
if let Some(current) = v.state() {
let node_refs =
current.watch_node_refs(&inner.outbound_watch_manager.per_node_states);
for node_ref in &node_refs {
let mut found = false;
for nid in node_ref.node_ids().iter() {
if node_set.contains(nid) {
found = true;
break;
}
}
if found {
continue;
}
node_set.insert(node_ref.best_node_id());
out.push(
Destination::direct(
aw.watch_node
.routing_domain_filtered(RoutingDomain::PublicInternet),
node_ref.routing_domain_filtered(RoutingDomain::PublicInternet),
)
.with_safety(v.safety_selection())
})
})
.collect()
.with_safety(current.params().safety_selection),
)
}
}
}
out
}
/// Builds the record key for a given schema and owner
@ -514,53 +573,19 @@ impl StorageManager {
#[instrument(level = "trace", target = "stor", skip_all)]
pub async fn close_record(&self, key: TypedKey) -> VeilidAPIResult<()> {
// Attempt to close the record, returning the opened record if it wasn't already closed
let opened_record = {
let mut inner = self.inner.lock().await;
let Some(opened_record) = Self::close_record_inner(&mut inner, key)? else {
return Ok(());
};
opened_record
};
Self::close_record_inner(&mut inner, key)?;
Ok(())
}
// See if we have an active watch on the closed record
let Some(active_watch) = opened_record.active_watch() else {
return Ok(());
};
// Send a one-time cancel request for the watch if we have one and we're online
if !self.dht_is_online() {
veilid_log!(self debug "skipping last-ditch watch cancel because we are offline");
return Ok(());
}
// Use the safety selection we opened the record with
// Use the writer we opened with as the 'watcher' as well
let opt_owvresult = match self
.outbound_watch_value_cancel(
key,
ValueSubkeyRangeSet::full(),
opened_record.safety_selection(),
opened_record.writer().cloned(),
active_watch.id,
active_watch.watch_node,
)
.await
{
Ok(v) => v,
Err(e) => {
veilid_log!(self debug
"close record watch cancel failed: {}", e
);
None
}
};
if let Some(owvresult) = opt_owvresult {
if owvresult.expiration_ts.as_u64() != 0 {
veilid_log!(self debug
"close record watch cancel should have zero expiration"
);
}
} else {
veilid_log!(self debug "close record watch cancel unsuccessful");
/// Close all opened records
#[instrument(level = "trace", target = "stor", skip_all)]
pub async fn close_all_records(&self) -> VeilidAPIResult<()> {
// Attempt to close the record, returning the opened record if it wasn't already closed
let mut inner = self.inner.lock().await;
let keys = inner.opened_records.keys().copied().collect::<Vec<_>>();
for key in keys {
Self::close_record_inner(&mut inner, key)?;
}
Ok(())
@ -570,10 +595,10 @@ impl StorageManager {
#[instrument(level = "trace", target = "stor", skip_all)]
pub async fn delete_record(&self, key: TypedKey) -> VeilidAPIResult<()> {
// Ensure the record is closed
self.close_record(key).await?;
let mut inner = self.inner.lock().await;
Self::close_record_inner(&mut inner, key)?;
// Get record from the local store
let mut inner = self.inner.lock().await;
let Some(local_record_store) = inner.local_record_store.as_mut() else {
apibail_not_initialized!();
};
@ -636,7 +661,7 @@ impl StorageManager {
apibail_internal!("failed to receive results");
};
let result = result?;
let partial = result.fanout_result.kind.is_partial();
let partial = result.fanout_result.kind.is_incomplete();
// Process the returned result
let out = self
@ -735,7 +760,7 @@ impl StorageManager {
key,
subkey,
signed_value_data.clone(),
WatchUpdateMode::NoUpdate,
InboundWatchUpdateMode::NoUpdate,
)
.await?;
@ -800,7 +825,7 @@ impl StorageManager {
apibail_internal!("failed to receive results");
};
let result = result?;
let partial = result.fanout_result.kind.is_partial();
let partial = result.fanout_result.kind.is_incomplete();
// Process the returned result
let out = self
@ -845,7 +870,7 @@ impl StorageManager {
out
}
/// Create,update or cancel an outbound watch to a DHT value
/// Create, update or cancel an outbound watch to a DHT value
#[instrument(level = "trace", target = "stor", skip_all)]
pub async fn watch_values(
&self,
@ -853,20 +878,37 @@ impl StorageManager {
subkeys: ValueSubkeyRangeSet,
expiration: Timestamp,
count: u32,
) -> VeilidAPIResult<Timestamp> {
let inner = self.inner.lock().await;
) -> VeilidAPIResult<bool> {
// Obtain the watch change lock
// (may need to wait for background operations to complete on the watch)
let watch_lock = self.outbound_watch_lock_table.lock_tag(key).await;
self.watch_values_inner(watch_lock, subkeys, expiration, count)
.await
}
#[instrument(level = "trace", target = "stor", skip_all)]
async fn watch_values_inner(
&self,
watch_lock: AsyncTagLockGuard<TypedKey>,
subkeys: ValueSubkeyRangeSet,
expiration: Timestamp,
count: u32,
) -> VeilidAPIResult<bool> {
let key = watch_lock.tag();
// Obtain the inner state lock
let mut inner = self.inner.lock().await;
// Get the safety selection and the writer we opened this record
// and whatever active watch id and watch node we may have in case this is a watch update
let (safety_selection, opt_writer, opt_watch_id, opt_watch_node) = {
let (safety_selection, opt_watcher) = {
let Some(opened_record) = inner.opened_records.get(&key) else {
// Record must be opened already to change watch
apibail_generic!("record not open");
};
(
opened_record.safety_selection(),
opened_record.writer().cloned(),
opened_record.active_watch().map(|aw| aw.id),
opened_record.active_watch().map(|aw| aw.watch_node.clone()),
)
};
@ -888,86 +930,67 @@ impl StorageManager {
};
let subkeys = schema.truncate_subkeys(&subkeys, None);
// Get rpc processor and drop mutex so we don't block while requesting the watch from the network
if !self.dht_is_online() {
apibail_try_again!("offline, try again later");
// Calculate desired watch parameters
let desired_params = if count == 0 {
// Cancel
None
} else {
// Get the minimum expiration timestamp we will accept
let rpc_timeout_us = self
.config()
.with(|c| TimestampDuration::from(ms_to_us(c.network.rpc.timeout_ms)));
let cur_ts = get_timestamp();
let min_expiration_ts = Timestamp::new(cur_ts + rpc_timeout_us.as_u64());
let expiration_ts = if expiration.as_u64() == 0 {
expiration
} else if expiration < min_expiration_ts {
apibail_invalid_argument!("expiration is too soon", "expiration", expiration);
} else {
expiration
};
// Create or modify
Some(OutboundWatchParameters {
expiration_ts,
count,
subkeys,
opt_watcher,
safety_selection,
})
};
// Modify the 'desired' state of the watch or add one if it does not exist
inner
.outbound_watch_manager
.set_desired_watch(key, desired_params);
// Drop the lock for network access
drop(inner);
// Use the safety selection we opened the record with
// Use the writer we opened with as the 'watcher' as well
let opt_owvresult = self
.outbound_watch_value(
key,
subkeys.clone(),
expiration,
count,
safety_selection,
opt_writer,
opt_watch_id,
opt_watch_node,
)
.await?;
// If we did not get a valid response assume nothing changed
let Some(owvresult) = opt_owvresult else {
apibail_try_again!("did not get a valid response");
};
// Clear any existing watch if the watch succeeded or got cancelled
// Process this watch's state machine operations until we are done
loop {
let opt_op_fut = {
let mut inner = self.inner.lock().await;
let Some(opened_record) = inner.opened_records.get_mut(&key) else {
apibail_generic!("record not open");
let Some(outbound_watch) =
inner.outbound_watch_manager.outbound_watches.get_mut(&key)
else {
// Watch is gone
return Ok(false);
};
opened_record.clear_active_watch();
// Get the minimum expiration timestamp we will accept
let (rpc_timeout_us, max_watch_expiration_us) = self.config().with(|c| {
(
TimestampDuration::from(ms_to_us(c.network.rpc.timeout_ms)),
TimestampDuration::from(ms_to_us(c.network.dht.max_watch_expiration_ms)),
self.get_next_outbound_watch_operation(
key,
Some(watch_lock.clone()),
Timestamp::now(),
outbound_watch,
)
});
let cur_ts = get_timestamp();
let min_expiration_ts = cur_ts + rpc_timeout_us.as_u64();
let max_expiration_ts = if expiration.as_u64() == 0 {
cur_ts + max_watch_expiration_us.as_u64()
} else {
expiration.as_u64()
};
// If the expiration time is less than our minimum expiration time (or zero) consider this watch inactive
let mut expiration_ts = owvresult.expiration_ts;
if expiration_ts.as_u64() < min_expiration_ts {
return Ok(Timestamp::new(0));
let Some(op_fut) = opt_op_fut else {
break;
};
op_fut.await;
}
// If the expiration time is greater than our maximum expiration time, clamp our local watch so we ignore extra valuechanged messages
if expiration_ts.as_u64() > max_expiration_ts {
expiration_ts = Timestamp::new(max_expiration_ts);
}
// If we requested a cancellation, then consider this watch cancelled
if count == 0 {
// Expiration returned should be zero if we requested a cancellation
if expiration_ts.as_u64() != 0 {
veilid_log!(self debug "got active watch despite asking for a cancellation");
}
return Ok(Timestamp::new(0));
}
// Keep a record of the watch
opened_record.set_active_watch(ActiveWatch {
id: owvresult.watch_id,
expiration_ts,
watch_node: owvresult.watch_node,
opt_value_changed_route: owvresult.opt_value_changed_route,
subkeys,
count,
});
Ok(owvresult.expiration_ts)
Ok(true)
}
#[instrument(level = "trace", target = "stor", skip_all)]
@ -976,18 +999,31 @@ impl StorageManager {
key: TypedKey,
subkeys: ValueSubkeyRangeSet,
) -> VeilidAPIResult<bool> {
let (subkeys, active_watch) = {
// Obtain the watch change lock
// (may need to wait for background operations to complete on the watch)
let watch_lock = self.outbound_watch_lock_table.lock_tag(key).await;
// Calculate change to existing watch
let (subkeys, count, expiration_ts) = {
let inner = self.inner.lock().await;
let Some(opened_record) = inner.opened_records.get(&key) else {
let Some(_opened_record) = inner.opened_records.get(&key) else {
apibail_generic!("record not open");
};
// See what watch we have currently if any
let Some(active_watch) = opened_record.active_watch() else {
let Some(outbound_watch) = inner.outbound_watch_manager.outbound_watches.get(&key)
else {
// If we didn't have an active watch, then we can just return false because there's nothing to do here
return Ok(false);
};
// Ensure we have a 'desired' watch state
let Some(desired) = outbound_watch.desired() else {
// If we didn't have a desired watch, then we're already cancelling
let still_active = outbound_watch.state().is_some();
return Ok(still_active);
};
// Rewrite subkey range if empty to full
let subkeys = if subkeys.is_empty() {
ValueSubkeyRangeSet::full()
@ -996,32 +1032,29 @@ impl StorageManager {
};
// Reduce the subkey range
let new_subkeys = active_watch.subkeys.difference(&subkeys);
let new_subkeys = desired.subkeys.difference(&subkeys);
(new_subkeys, active_watch)
};
// If we have no subkeys left, then set the count to zero to indicate a full cancellation
let count = if subkeys.is_empty() {
0
} else {
active_watch.count
};
// Update the watch. This just calls through to the above watch_values() function
// This will update the active_watch so we don't need to do that in this routine.
let expiration_ts =
pin_future!(self.watch_values(key, subkeys, active_watch.expiration_ts, count)).await?;
// A zero expiration time returned from watch_value() means the watch is done
// or no subkeys are left, and the watch is no longer active
if expiration_ts.as_u64() == 0 {
// Return false indicating the watch is completely gone
// If no change is happening return false
if new_subkeys == desired.subkeys {
return Ok(false);
}
// Return true because the the watch was changed
Ok(true)
// If we have no subkeys left, then set the count to zero to indicate a full cancellation
let count = if new_subkeys.is_empty() {
0
} else if let Some(state) = outbound_watch.state() {
state.remaining_count()
} else {
desired.count
};
(new_subkeys, count, desired.expiration_ts)
};
// Update the watch. This just calls through to the above watch_values_inner() function
// This will update the active_watch so we don't need to do that in this routine.
self.watch_values_inner(watch_lock, subkeys, expiration_ts, count)
.await
}
/// Inspect an opened DHT record for its subkey sequence numbers
@ -1038,6 +1071,12 @@ impl StorageManager {
subkeys
};
// Get cryptosystem
let crypto = self.crypto();
let Some(vcrypto) = crypto.get(key.kind) else {
apibail_generic!("unsupported cryptosystem");
};
let mut inner = self.inner.lock().await;
let safety_selection = {
let Some(opened_record) = inner.opened_records.get(&key) else {
@ -1122,7 +1161,7 @@ impl StorageManager {
{
assert_eq!(
result.inspect_result.subkeys.len() as u64,
result.fanout_results.len() as u64,
result.subkey_fanout_results.len() as u64,
"mismatch between subkeys returned and fanout results returned"
);
}
@ -1140,10 +1179,12 @@ impl StorageManager {
.inspect_result
.subkeys
.iter()
.zip(result.fanout_results.iter());
.map(ValueSubkeyRangeSet::single)
.zip(result.subkey_fanout_results.into_iter());
Self::process_fanout_results_inner(
&mut inner,
&vcrypto,
key,
results_iter,
false,
@ -1210,12 +1251,12 @@ impl StorageManager {
fanout_result: &FanoutResult,
) -> bool {
match fanout_result.kind {
FanoutResultKind::Partial => false,
FanoutResultKind::Incomplete => false,
FanoutResultKind::Timeout => {
let get_consensus = self
.config()
.with(|c| c.network.dht.get_value_count as usize);
let value_node_count = fanout_result.value_nodes.len();
let value_node_count = fanout_result.consensus_nodes.len();
if value_node_count < get_consensus {
veilid_log!(self debug "timeout with insufficient consensus ({}<{}), adding offline subkey: {}:{}",
value_node_count, get_consensus,
@ -1232,7 +1273,7 @@ impl StorageManager {
let get_consensus = self
.config()
.with(|c| c.network.dht.get_value_count as usize);
let value_node_count = fanout_result.value_nodes.len();
let value_node_count = fanout_result.consensus_nodes.len();
if value_node_count < get_consensus {
veilid_log!(self debug "exhausted with insufficient consensus ({}<{}), adding offline subkey: {}:{}",
value_node_count, get_consensus,
@ -1245,7 +1286,7 @@ impl StorageManager {
false
}
}
FanoutResultKind::Finished => false,
FanoutResultKind::Consensus => false,
}
}
@ -1361,7 +1402,7 @@ impl StorageManager {
continue;
};
local_record_store
.set_subkey(key, subkey, subkey_data, WatchUpdateMode::NoUpdate)
.set_subkey(key, subkey, subkey_data, InboundWatchUpdateMode::NoUpdate)
.await?;
}
@ -1495,7 +1536,12 @@ impl StorageManager {
if let Some(signed_value_data) = get_result.opt_value {
// Write subkey to local store
local_record_store
.set_subkey(key, subkey, signed_value_data, WatchUpdateMode::NoUpdate)
.set_subkey(
key,
subkey,
signed_value_data,
InboundWatchUpdateMode::NoUpdate,
)
.await?;
}
@ -1539,11 +1585,11 @@ impl StorageManager {
#[instrument(level = "trace", target = "stor", skip_all)]
pub(super) fn process_fanout_results_inner<
'a,
I: IntoIterator<Item = (ValueSubkey, &'a FanoutResult)>,
I: IntoIterator<Item = (ValueSubkeyRangeSet, FanoutResult)>,
>(
inner: &mut StorageManagerInner,
key: TypedKey,
vcrypto: &CryptoSystemGuard<'_>,
record_key: TypedKey,
subkey_results_iter: I,
is_set: bool,
consensus_count: usize,
@ -1552,21 +1598,21 @@ impl StorageManager {
let local_record_store = inner.local_record_store.as_mut().unwrap();
let cur_ts = Timestamp::now();
local_record_store.with_record_mut(key, |r| {
local_record_store.with_record_mut(record_key, |r| {
let d = r.detail_mut();
for (subkey, fanout_result) in subkey_results_iter {
for (subkeys, fanout_result) in subkey_results_iter {
for node_id in fanout_result
.value_nodes
.iter()
.filter_map(|x| x.node_ids().get(key.kind).map(|k| k.value))
.filter_map(|x| x.node_ids().get(record_key.kind).map(|k| k.value))
{
let pnd = d.nodes.entry(node_id).or_default();
if is_set || pnd.last_set == Timestamp::default() {
pnd.last_set = cur_ts;
}
pnd.last_seen = cur_ts;
pnd.subkeys.insert(subkey);
pnd.subkeys = pnd.subkeys.union(&subkeys);
}
}
@ -1576,7 +1622,17 @@ impl StorageManager {
.iter()
.map(|kv| (*kv.0, kv.1.last_seen))
.collect::<Vec<_>>();
nodes_ts.sort_by(|a, b| b.1.cmp(&a.1));
nodes_ts.sort_by(|a, b| {
// Timestamp is first metric
let res = b.1.cmp(&a.1);
if res != cmp::Ordering::Equal {
return res;
}
// Distance is the next metric, closer nodes first
let da = vcrypto.distance(&a.0, &record_key.value);
let db = vcrypto.distance(&b.0, &record_key.value);
da.cmp(&db)
});
for dead_node_key in nodes_ts.iter().skip(consensus_count) {
d.nodes.remove(&dead_node_key.0);
@ -1584,18 +1640,21 @@ impl StorageManager {
});
}
fn close_record_inner(
inner: &mut StorageManagerInner,
key: TypedKey,
) -> VeilidAPIResult<Option<OpenedRecord>> {
fn close_record_inner(inner: &mut StorageManagerInner, key: TypedKey) -> VeilidAPIResult<()> {
let Some(local_record_store) = inner.local_record_store.as_mut() else {
apibail_not_initialized!();
};
if local_record_store.peek_record(key, |_| {}).is_none() {
return Err(VeilidAPIError::key_not_found(key));
apibail_key_not_found!(key);
}
Ok(inner.opened_records.remove(&key))
if inner.opened_records.remove(&key).is_some() {
// Set the watch to cancelled if we have one
// Will process cancellation in the background
inner.outbound_watch_manager.set_desired_watch(key, None);
}
Ok(())
}
#[instrument(level = "trace", target = "stor", skip_all, err)]
@ -1628,7 +1687,7 @@ impl StorageManager {
key: TypedKey,
subkey: ValueSubkey,
signed_value_data: Arc<SignedValueData>,
watch_update_mode: WatchUpdateMode,
watch_update_mode: InboundWatchUpdateMode,
) -> VeilidAPIResult<()> {
// See if it's in the local record store
let Some(local_record_store) = inner.local_record_store.as_mut() else {
@ -1699,7 +1758,7 @@ impl StorageManager {
subkey: ValueSubkey,
signed_value_data: Arc<SignedValueData>,
signed_value_descriptor: Arc<SignedValueDescriptor>,
watch_update_mode: WatchUpdateMode,
watch_update_mode: InboundWatchUpdateMode,
) -> VeilidAPIResult<()> {
// See if it's in the remote record store
let Some(remote_record_store) = inner.remote_record_store.as_mut() else {
@ -1791,7 +1850,7 @@ impl StorageManager {
receiver: flume::Receiver<T>,
handler: impl FnMut(T) -> PinBoxFutureStatic<bool> + Send + 'static,
) -> bool {
self.deferred_result_processor
.add(receiver.into_stream(), handler)
self.background_operation_processor
.add_stream(receiver.into_stream(), handler)
}
}

213
veilid-core/src/storage_manager/outbound_watch_manager/mod.rs Normal file
View file

@ -0,0 +1,213 @@
mod outbound_watch;
mod outbound_watch_parameters;
mod outbound_watch_state;
mod per_node_state;
pub(in crate::storage_manager) use outbound_watch::*;
pub(in crate::storage_manager) use outbound_watch_parameters::*;
pub(in crate::storage_manager) use outbound_watch_state::*;
pub(in crate::storage_manager) use per_node_state::*;
use super::*;
use serde_with::serde_as;
impl_veilid_log_facility!("stor");
#[serde_as]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub(in crate::storage_manager) struct OutboundWatchManager {
/// Each watch per record key
#[serde(skip)]
pub outbound_watches: HashMap<TypedKey, OutboundWatch>,
/// Last known active watch per node+record
#[serde_as(as = "Vec<(_, _)>")]
pub per_node_states: HashMap<PerNodeKey, PerNodeState>,
/// Value changed updates that need inpection to determine if they should be reported
#[serde(skip)]
pub needs_change_inspection: HashMap<TypedKey, ValueSubkeyRangeSet>,
}
impl fmt::Display for OutboundWatchManager {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut out = format!("outbound_watches({}): [\n", self.outbound_watches.len());
{
let mut keys = self.outbound_watches.keys().copied().collect::<Vec<_>>();
keys.sort();
for k in keys {
let v = self.outbound_watches.get(&k).unwrap();
out += &format!(" {}:\n{}\n", k, indent_all_by(4, v.to_string()));
}
}
out += "]\n";
out += &format!("per_node_states({}): [\n", self.per_node_states.len());
{
let mut keys = self.per_node_states.keys().copied().collect::<Vec<_>>();
keys.sort();
for k in keys {
let v = self.per_node_states.get(&k).unwrap();
out += &format!(" {}:\n{}\n", k, indent_all_by(4, v.to_string()));
}
}
out += "]\n";
out += &format!(
"needs_change_inspection({}): [\n",
self.needs_change_inspection.len()
);
{
let mut keys = self
.needs_change_inspection
.keys()
.copied()
.collect::<Vec<_>>();
keys.sort();
for k in keys {
let v = self.needs_change_inspection.get(&k).unwrap();
out += &format!(" {}: {}\n", k, v);
}
}
out += "]\n";
write!(f, "{}", out)
}
}
impl Default for OutboundWatchManager {
fn default() -> Self {
Self::new()
}
}
impl OutboundWatchManager {
pub fn new() -> Self {
Self {
outbound_watches: HashMap::new(),
per_node_states: HashMap::new(),
needs_change_inspection: HashMap::new(),
}
}
pub fn prepare(&mut self, routing_table: VeilidComponentGuard<'_, RoutingTable>) {
for (pnk, pns) in &mut self.per_node_states {
pns.watch_node_ref = match routing_table.lookup_node_ref(pnk.node_id) {
Ok(v) => v,
Err(e) => {
veilid_log!(routing_table debug "Error looking up outbound watch node ref: {}", e);
None
}
};
}
self.per_node_states
.retain(|_, v| v.watch_node_ref.is_some());
let keys = self.per_node_states.keys().copied().collect::<HashSet<_>>();
for v in self.outbound_watches.values_mut() {
if let Some(state) = v.state_mut() {
state.edit(&self.per_node_states, |editor| {
editor.retain_nodes(|n| keys.contains(n));
})
}
}
}
pub fn set_desired_watch(
&mut self,
record_key: TypedKey,
desired_watch: Option<OutboundWatchParameters>,
) {
match self.outbound_watches.get_mut(&record_key) {
Some(w) => {
// Replace desired watch
w.set_desired(desired_watch);
// Remove if the watch is done (shortcut the dead state)
if w.state().is_none() && w.state().is_none() {
self.outbound_watches.remove(&record_key);
}
}
None => {
// Watch does not exist, add one if that's what is desired
if let Some(desired) = desired_watch {
self.outbound_watches
.insert(record_key, OutboundWatch::new(desired));
}
}
}
}
pub fn set_next_reconcile_ts(&mut self, record_key: TypedKey, next_ts: Timestamp) {
if let Some(outbound_watch) = self.outbound_watches.get_mut(&record_key) {
if let Some(state) = outbound_watch.state_mut() {
state.edit(&self.per_node_states, |editor| {
editor.set_next_reconcile_ts(next_ts);
});
}
}
}
/// Iterate all per-node watches and remove ones with dead nodes from outbound watches
/// This may trigger reconciliation to increase the number of active per-node watches
/// for an outbound watch that is still alive
pub fn update_per_node_states(&mut self, cur_ts: Timestamp) {
// Node is unreachable
let mut dead_pnks = HashSet::new();
// Per-node expiration reached
let mut expired_pnks = HashSet::new();
// Count reached
let mut finished_pnks = HashSet::new();
for (pnk, pns) in &self.per_node_states {
if pns.count == 0 {
// If per-node watch is done, add to finished list
finished_pnks.insert(*pnk);
} else if !pns
.watch_node_ref
.as_ref()
.unwrap()
.state(cur_ts)
.is_alive()
{
// If node is unreachable add to dead list
dead_pnks.insert(*pnk);
} else if cur_ts >= pns.expiration_ts {
// If per-node watch has expired add to expired list
expired_pnks.insert(*pnk);
}
}
// Go through and remove nodes that are dead or finished from active states
// If an expired per-node watch is still referenced, it may be renewable
// so remove it from the expired list
for v in self.outbound_watches.values_mut() {
let Some(current) = v.state_mut() else {
continue;
};
// Don't drop expired per-node watches that could be renewed (still referenced by this watch)
for node in current.nodes() {
expired_pnks.remove(node);
}
// Remove dead and finished per-node watch nodes from this outbound watch
current.edit(&self.per_node_states, |editor| {
editor.retain_nodes(|x| !dead_pnks.contains(x) && !finished_pnks.contains(x));
});
}
// Drop finished per-node watches and unreferenced expired per-node watches
self.per_node_states
.retain(|k, _| !finished_pnks.contains(k) && !expired_pnks.contains(k));
}
/// Set a record up to be inspected for changed subkeys
pub fn enqueue_change_inspect(&mut self, record_key: TypedKey, subkeys: ValueSubkeyRangeSet) {
self.needs_change_inspection
.entry(record_key)
.and_modify(|x| *x = x.union(&subkeys))
.or_insert(subkeys);
}
}

253
veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch.rs Normal file
View file

@ -0,0 +1,253 @@
use super::*;
impl_veilid_log_facility!("stor");
#[derive(Clone, Debug, Serialize, Deserialize)]
pub(in crate::storage_manager) struct OutboundWatch {
/// Current state
/// None means inactive/cancelled
state: Option<OutboundWatchState>,
/// Desired parameters
/// None means cancelled
desired: Option<OutboundWatchParameters>,
}
impl fmt::Display for OutboundWatch {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"Desired: {}\nState:\n{}\n",
if let Some(desired) = &self.desired {
desired.to_string()
} else {
"None".to_owned()
},
if let Some(state) = &self.state {
indent_all_by(4, state.to_string())
} else {
"None".to_owned()
},
)
}
}
impl OutboundWatch {
/// Create new outbound watch with desired parameters
pub fn new(desired: OutboundWatchParameters) -> Self {
Self {
state: None,
desired: Some(desired),
}
}
/// Get current watch state if it exists
pub fn state(&self) -> Option<&OutboundWatchState> {
self.state.as_ref()
}
/// Get mutable current watch state if it exists
pub fn state_mut(&mut self) -> Option<&mut OutboundWatchState> {
self.state.as_mut()
}
/// Clear current watch state
pub fn clear_state(&mut self) {
self.state = None;
}
/// Get or create current watch state
pub fn state_mut_or_create<F: FnOnce() -> OutboundWatchParameters>(
&mut self,
make_parameters: F,
) -> &mut OutboundWatchState {
if self.state.is_none() {
self.state = Some(OutboundWatchState::new(make_parameters()));
}
self.state.as_mut().unwrap()
}
/// Get desired watch parameters if it exists
pub fn desired(&self) -> Option<OutboundWatchParameters> {
self.desired.clone()
}
/// Set desired watch parameters
pub fn set_desired(&mut self, desired: Option<OutboundWatchParameters>) {
self.desired = desired;
}
/// Check for desired state changes
pub fn try_expire_desired_state(&mut self, cur_ts: Timestamp) {
let Some(desired) = self.desired.as_ref() else {
// No desired parameters means this is already done
return;
};
// Check if desired parameters have expired
if desired.expiration_ts.as_u64() != 0 && desired.expiration_ts <= cur_ts {
// Expired
self.set_desired(None);
return;
}
// Check if the existing state has no remaining count
if let Some(state) = self.state.as_ref() {
if state.remaining_count() == 0 {
// No remaining count
self.set_desired(None);
}
}
}
/// Returns true if this outbound watch can be removed from the table
pub fn is_dead(&self) -> bool {
self.desired.is_none() && self.state.is_none()
}
/// Returns true if this outbound watch needs to be cancelled
pub fn needs_cancel(&self, registry: &VeilidComponentRegistry) -> bool {
if self.is_dead() {
veilid_log!(registry warn "should have checked for is_dead first");
return false;
}
// If there is no current watch then there is nothing to cancel
let Some(_state) = self.state.as_ref() else {
return false;
};
// If the desired parameters is None then cancel
let Some(_desired) = self.desired.as_ref() else {
return true;
};
false
}
/// Returns true if this outbound watch can be renewed
pub fn needs_renew(
&self,
registry: &VeilidComponentRegistry,
consensus_count: usize,
cur_ts: Timestamp,
) -> bool {
if self.is_dead() || self.needs_cancel(registry) {
veilid_log!(registry warn "should have checked for is_dead and needs_cancel first");
return false;
}
// If there is no current watch then there is nothing to renew
let Some(state) = self.state.as_ref() else {
return false;
};
// Should have desired parameters here
let Some(desired) = self.desired.as_ref() else {
veilid_log!(registry warn "needs_cancel should have returned true");
return false;
};
// If we have a consensus, we can avoid fanout by renewing rather than reconciling
// but if we don't have a consensus, we should defer to fanout to try to improve it
if state.nodes().len() < consensus_count {
return false;
}
// If we have a consensus but need to renew because some per-node watches
// either expired or had their routes die, do it
if self.wants_per_node_watch_update(registry, state, cur_ts) {
return true;
}
// If the desired parameters have changed, then we should renew with them
state.params() != desired
}
/// Returns true if there is work to be done on getting the outbound
/// watch to its desired state
pub fn needs_reconcile(
&self,
registry: &VeilidComponentRegistry,
consensus_count: usize,
cur_ts: Timestamp,
) -> bool {
if self.is_dead()
|| self.needs_cancel(registry)
|| self.needs_renew(registry, consensus_count, cur_ts)
{
veilid_log!(registry warn "should have checked for is_dead, needs_cancel, needs_renew first");
return false;
}
// If desired is none, then is_dead() or needs_cancel() should have been true
let Some(desired) = self.desired.as_ref() else {
veilid_log!(registry warn "is_dead() or needs_cancel() should have been true");
return false;
};
// If there is a desired watch but no current state, then reconcile
let Some(state) = self.state() else {
return true;
};
// If we are still working on getting the 'current' state to match
// the 'desired' state, then do the reconcile if we are within the timeframe for it
if state.nodes().len() < consensus_count
&& cur_ts >= state.next_reconcile_ts().unwrap_or_default()
{
return true;
}
// Try to reconcile if our number of nodes currently is less than what we got from
// the previous reconciliation attempt
if let Some(last_consensus_node_count) = state.last_consensus_node_count() {
if state.nodes().len() < last_consensus_node_count {
return true;
}
}
// If we have a consensus, or are not attempting consensus at this time,
// but need to reconcile because some per-node watches either expired or had their routes die, do it
if self.wants_per_node_watch_update(registry, state, cur_ts) {
return true;
}
// If the desired parameters have changed, then we should reconcile with them
state.params() != desired
}
/// Returns true if we need to update our per-node watches due to expiration,
/// or if they are all dead because the route died and needs to be updated
fn wants_per_node_watch_update(
&self,
registry: &VeilidComponentRegistry,
state: &OutboundWatchState,
cur_ts: Timestamp,
) -> bool {
// If the watch has per node watches that have expired, but we can extend our watch then renew.
// Do this only within RENEW_OUTBOUND_WATCHES_DURATION_SECS of the actual expiration.
// If we're looking at this after the actual expiration, don't try because the watch id will have died.
let renew_ts = cur_ts + TimestampDuration::new_secs(RENEW_OUTBOUND_WATCHES_DURATION_SECS);
if renew_ts >= state.min_expiration_ts()
&& cur_ts < state.min_expiration_ts()
&& (state.params().expiration_ts.as_u64() == 0
|| renew_ts < state.params().expiration_ts)
{
return true;
}
let routing_table = registry.routing_table();
let rss = routing_table.route_spec_store();
// See if any of our per node watches have a dead value changed route
// if so, speculatively renew them
for vcr in state.value_changed_routes() {
if rss.get_route_id_for_key(vcr).is_none() {
// Route we would receive value changes on is dead
return true;
}
}
false
}
}

37
veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch_parameters.rs Normal file
View file

@ -0,0 +1,37 @@
use super::*;
impl_veilid_log_facility!("stor");
/// Requested parameters for watch
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct OutboundWatchParameters {
/// Requested expiration timestamp. A zero timestamp here indicates
/// that the watch it to be renewed indefinitely
pub expiration_ts: Timestamp,
/// How many notifications the requestor asked for
pub count: u32,
/// Subkeys requested for this watch
pub subkeys: ValueSubkeyRangeSet,
/// What key to use to perform the watch
pub opt_watcher: Option<KeyPair>,
/// What safety selection to use on the network
pub safety_selection: SafetySelection,
}
impl fmt::Display for OutboundWatchParameters {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{{ expiration={}, count={}, subkeys={}, opt_watcher={}, safety_selection={:?} }}",
self.expiration_ts,
self.count,
self.subkeys,
if let Some(watcher) = &self.opt_watcher {
watcher.to_string()
} else {
"None".to_owned()
},
self.safety_selection
)
}
}

188
veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch_state.rs Normal file
View file

@ -0,0 +1,188 @@
use super::*;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub(in crate::storage_manager) struct OutboundWatchState {
/// Requested parameters
params: OutboundWatchParameters,
/// Nodes that have an active watch on our behalf
nodes: Vec<PerNodeKey>,
/// How many value change updates remain
remaining_count: u32,
/// The next earliest time we are willing to try to reconcile and improve the watch
opt_next_reconcile_ts: Option<Timestamp>,
/// The number of nodes we got at our last reconciliation
opt_last_consensus_node_count: Option<usize>,
/// Calculated field: minimum expiration time for all our nodes
min_expiration_ts: Timestamp,
/// Calculated field: the set of value changed routes for this watch from all per node watches
value_changed_routes: BTreeSet<PublicKey>,
}
impl fmt::Display for OutboundWatchState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut value_changed_routes = self
.value_changed_routes
.iter()
.map(|x| x.to_string())
.collect::<Vec<_>>();
value_changed_routes.sort();
write!(
f,
r#"params: {}
nodes: [{}]
remaining_count: {}
opt_next_reconcile_ts: {}
opt_consensus_node_count: {}
min_expiration_ts: {}
value_changed_routes: [{}]"#,
self.params,
self.nodes
.iter()
.map(|x| x.node_id.to_string())
.collect::<Vec<_>>()
.join(","),
self.remaining_count,
if let Some(next_reconcile_ts) = &self.opt_next_reconcile_ts {
next_reconcile_ts.to_string()
} else {
"None".to_owned()
},
if let Some(consensus_node_count) = &self.opt_last_consensus_node_count {
consensus_node_count.to_string()
} else {
"None".to_owned()
},
self.min_expiration_ts,
value_changed_routes.join(","),
)
}
}
pub(in crate::storage_manager) struct OutboundWatchStateEditor<'a> {
state: &'a mut OutboundWatchState,
}
impl OutboundWatchStateEditor<'_> {
pub fn set_params(&mut self, params: OutboundWatchParameters) {
self.state.params = params;
}
pub fn add_nodes<I: IntoIterator<Item = PerNodeKey>>(&mut self, nodes: I) {
for node in nodes {
if !self.state.nodes.contains(&node) {
self.state.nodes.push(node);
}
}
}
pub fn retain_nodes<F: FnMut(&PerNodeKey) -> bool>(&mut self, f: F) {
self.state.nodes.retain(f);
}
pub fn set_remaining_count(&mut self, remaining_count: u32) {
self.state.remaining_count = remaining_count;
}
pub fn set_next_reconcile_ts(&mut self, next_reconcile_ts: Timestamp) {
self.state.opt_next_reconcile_ts = Some(next_reconcile_ts);
}
pub fn update_last_consensus_node_count(&mut self) {
self.state.opt_last_consensus_node_count = Some(self.state.nodes().len());
}
}
impl OutboundWatchState {
pub fn new(params: OutboundWatchParameters) -> Self {
let remaining_count = params.count;
let min_expiration_ts = params.expiration_ts;
Self {
params,
nodes: vec![],
remaining_count,
opt_next_reconcile_ts: None,
opt_last_consensus_node_count: None,
min_expiration_ts,
value_changed_routes: BTreeSet::new(),
}
}
pub fn params(&self) -> &OutboundWatchParameters {
&self.params
}
pub fn nodes(&self) -> &Vec<PerNodeKey> {
&self.nodes
}
pub fn remaining_count(&self) -> u32 {
self.remaining_count
}
pub fn next_reconcile_ts(&self) -> Option<Timestamp> {
self.opt_next_reconcile_ts
}
pub fn last_consensus_node_count(&self) -> Option<usize> {
self.opt_last_consensus_node_count
}
pub fn min_expiration_ts(&self) -> Timestamp {
self.min_expiration_ts
}
pub fn value_changed_routes(&self) -> &BTreeSet<PublicKey> {
&self.value_changed_routes
}
/// Get the parameters we use if we're updating this state's per node watches
pub fn get_per_node_params(
&self,
desired: &OutboundWatchParameters,
) -> OutboundWatchParameters {
// Change the params to update count
if self.params() != desired {
// If parameters are changing, just use the desired parameters
desired.clone()
} else {
// If this is a renewal of the same parameters,
// use the current remaining update count for the rpc
let mut renew_params = desired.clone();
renew_params.count = self.remaining_count();
renew_params
}
}
pub fn edit<R, F: FnOnce(&mut OutboundWatchStateEditor) -> R>(
&mut self,
per_node_state: &HashMap<PerNodeKey, PerNodeState>,
closure: F,
) -> R {
let mut editor = OutboundWatchStateEditor { state: self };
let res = closure(&mut editor);
// Update calculated fields
self.min_expiration_ts = self
.nodes
.iter()
.map(|x| per_node_state.get(x).unwrap().expiration_ts)
.reduce(|a, b| a.min(b))
.unwrap_or(self.params.expiration_ts);
self.value_changed_routes = self
.nodes
.iter()
.filter_map(|x| per_node_state.get(x).unwrap().opt_value_changed_route)
.collect();
res
}
pub fn watch_node_refs(
&self,
per_node_state: &HashMap<PerNodeKey, PerNodeState>,
) -> Vec<NodeRef> {
self.nodes
.iter()
.map(|x| {
per_node_state
.get(x)
.unwrap()
.watch_node_ref
.clone()
.unwrap()
})
.collect()
}
}

75
veilid-core/src/storage_manager/outbound_watch_manager/per_node_state.rs Normal file
View file

@ -0,0 +1,75 @@
use super::*;
impl_veilid_log_facility!("stor");
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub(in crate::storage_manager) struct PerNodeKey {
/// Watched record key
pub record_key: TypedKey,
/// Watching node id
pub node_id: TypedKey,
}
impl fmt::Display for PerNodeKey {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}@{}", self.record_key, self.node_id)
}
}
impl FromStr for PerNodeKey {
type Err = VeilidAPIError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let (rkey, nid) = s
.split_once('@')
.ok_or_else(|| VeilidAPIError::parse_error("invalid per-node key", s))?;
Ok(PerNodeKey {
record_key: TypedKey::from_str(rkey)?,
node_id: TypedKey::from_str(nid)?,
})
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub(in crate::storage_manager) struct PerNodeState {
/// Watch Id
pub watch_id: u64,
/// SafetySelection used to contact the node
pub safety_selection: SafetySelection,
/// What key was used to perform the watch
pub opt_watcher: Option<KeyPair>,
/// The expiration of a successful watch
pub expiration_ts: Timestamp,
/// How many value change notifications are left
pub count: u32,
/// Resolved watch node reference
#[serde(skip)]
pub watch_node_ref: Option<NodeRef>,
/// Which private route is responsible for receiving ValueChanged notifications
pub opt_value_changed_route: Option<PublicKey>,
}
impl fmt::Display for PerNodeState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{{ watch_id={}, safety_selection={:?}, opt_watcher={}, expiration_ts={}, count={}, watch_node_ref={}, opt_value_changed_route={} }}",
self.watch_id,
self.safety_selection,
if let Some(watcher) = &self.opt_watcher {
watcher.to_string()
} else {
"None".to_owned()
},
self.expiration_ts,
self.count,
if let Some(watch_node_ref) = &self.watch_node_ref {
watch_node_ref.to_string()
} else {
"None".to_string()
},
if let Some(value_changed_route)= &self.opt_value_changed_route {
value_changed_route.to_string()
} else {
"None".to_string()
}
)
}
}

14
veilid-core/src/storage_manager/record_store/watch.rs → veilid-core/src/storage_manager/record_store/inbound_watch.rs
View file

@ -2,7 +2,7 @@ use super::*;
/// Watch parameters used to configure a watch
#[derive(Debug, Clone)]
pub struct WatchParameters {
pub struct InboundWatchParameters {
/// The range of subkeys being watched, empty meaning full
pub subkeys: ValueSubkeyRangeSet,
/// When this watch will expire
@ -18,7 +18,7 @@ pub struct WatchParameters {
/// Watch result to return with answer
/// Default result is cancelled/expired/inactive/rejected
#[derive(Debug, Clone)]
pub enum WatchResult {
pub enum InboundWatchResult {
/// A new watch was created
Created {
/// The new id of the watch
@ -39,9 +39,9 @@ pub enum WatchResult {
/// An individual watch
#[derive(Debug, Clone)]
pub struct Watch {
pub struct InboundWatch {
/// The configuration of the watch
pub params: WatchParameters,
pub params: InboundWatchParameters,
/// A unique id per record assigned at watch creation time. Used to disambiguate a client's version of a watch
pub id: u64,
/// What has changed since the last update
@ -50,13 +50,13 @@ pub struct Watch {
#[derive(Debug, Default, Clone)]
/// A record being watched for changes
pub struct WatchList {
pub struct InboundWatchList {
/// The list of active watches
pub watches: Vec<Watch>,
pub watches: Vec<InboundWatch>,
}
/// How a watch gets updated when a value changes
pub enum WatchUpdateMode {
pub enum InboundWatchUpdateMode {
/// Update no watchers
NoUpdate,
/// Update all watchers

4
veilid-core/src/storage_manager/record_store/limited_size.rs
View file

@ -125,7 +125,7 @@ impl<T: PrimInt + Unsigned + fmt::Display + fmt::Debug> LimitedSize<T> {
return Err(LimitError::OverLimit);
}
}
veilid_log!(self debug "Commit ({}): {} => {}", self.description, self.value, uncommitted_value);
veilid_log!(self trace "Commit ({}): {} => {}", self.description, self.value, uncommitted_value);
self.uncommitted_value = None;
self.value = uncommitted_value;
}
@ -134,7 +134,7 @@ impl<T: PrimInt + Unsigned + fmt::Display + fmt::Debug> LimitedSize<T> {
pub fn rollback(&mut self) -> T {
if let Some(uv) = self.uncommitted_value.take() {
veilid_log!(self debug "Rollback ({}): {} (drop {})", self.description, self.value, uv);
veilid_log!(self trace "Rollback ({}): {} (drop {})", self.description, self.value, uv);
}
self.value
}

58
veilid-core/src/storage_manager/record_store/mod.rs
View file

@ -4,6 +4,7 @@
/// This store does not perform any validation on the schema, and all ValueRecordData passed in must have been previously validated.
/// Uses an in-memory store for the records, backed by the TableStore. Subkey data is LRU cached and rotated out by a limits policy,
/// and backed to the TableStore for persistence.
mod inbound_watch;
mod inspect_cache;
mod keys;
mod limited_size;
@ -13,8 +14,9 @@ mod record;
mod record_data;
mod record_store_limits;
mod remote_record_detail;
mod watch;
pub(super) use inbound_watch::*;
pub use inbound_watch::{InboundWatchParameters, InboundWatchResult};
pub(super) use inspect_cache::*;
pub(super) use keys::*;
pub(super) use limited_size::*;
@ -23,8 +25,6 @@ pub(super) use opened_record::*;
pub(super) use record::*;
pub(super) use record_store_limits::*;
pub(super) use remote_record_detail::*;
pub(super) use watch::*;
pub use watch::{WatchParameters, WatchResult};
use super::*;
use record_data::*;
@ -75,7 +75,7 @@ where
/// The list of records that have changed since last flush to disk (optimization for batched writes)
changed_records: HashSet<RecordTableKey>,
/// The list of records being watched for changes
watched_records: HashMap<RecordTableKey, WatchList>,
watched_records: HashMap<RecordTableKey, InboundWatchList>,
/// The list of watched records that have changed values since last notification
changed_watched_values: HashSet<RecordTableKey>,
/// A mutex to ensure we handle this concurrently
@ -680,12 +680,12 @@ where
&mut self,
key: TypedKey,
subkey: ValueSubkey,
watch_update_mode: WatchUpdateMode,
watch_update_mode: InboundWatchUpdateMode,
) {
let (do_update, opt_ignore_target) = match watch_update_mode {
WatchUpdateMode::NoUpdate => (false, None),
WatchUpdateMode::UpdateAll => (true, None),
WatchUpdateMode::ExcludeTarget(target) => (true, Some(target)),
InboundWatchUpdateMode::NoUpdate => (false, None),
InboundWatchUpdateMode::UpdateAll => (true, None),
InboundWatchUpdateMode::ExcludeTarget(target) => (true, Some(target)),
};
if !do_update {
return;
@ -720,7 +720,7 @@ where
key: TypedKey,
subkey: ValueSubkey,
signed_value_data: Arc<SignedValueData>,
watch_update_mode: WatchUpdateMode,
watch_update_mode: InboundWatchUpdateMode,
) -> VeilidAPIResult<()> {
// Check size limit for data
if signed_value_data.value_data().data().len() > self.limits.max_subkey_size {
@ -902,9 +902,9 @@ where
pub async fn _change_existing_watch(
&mut self,
key: TypedKey,
params: WatchParameters,
params: InboundWatchParameters,
watch_id: u64,
) -> VeilidAPIResult<WatchResult> {
) -> VeilidAPIResult<InboundWatchResult> {
if params.count == 0 {
apibail_internal!("cancel watch should not have gotten here");
}
@ -915,7 +915,7 @@ where
let rtk = RecordTableKey { key };
let Some(watch_list) = self.watched_records.get_mut(&rtk) else {
// No watches, nothing to change
return Ok(WatchResult::Rejected);
return Ok(InboundWatchResult::Rejected);
};
// Check each watch to see if we have an exact match for the id to change
@ -925,23 +925,23 @@ where
if w.id == watch_id && w.params.watcher == params.watcher {
// Updating an existing watch
w.params = params;
return Ok(WatchResult::Changed {
return Ok(InboundWatchResult::Changed {
expiration: w.params.expiration,
});
}
}
// No existing watch found
Ok(WatchResult::Rejected)
Ok(InboundWatchResult::Rejected)
}
#[instrument(level = "trace", target = "stor", skip_all, err)]
pub async fn _create_new_watch(
&mut self,
key: TypedKey,
params: WatchParameters,
params: InboundWatchParameters,
member_check: Box<dyn Fn(PublicKey) -> bool + Send>,
) -> VeilidAPIResult<WatchResult> {
) -> VeilidAPIResult<InboundWatchResult> {
// Generate a record-unique watch id > 0
let rtk = RecordTableKey { key };
let mut id = 0;
@ -1001,7 +1001,7 @@ where
// For anonymous, no more than one watch per target per record
if target_watch_count > 0 {
// Too many watches
return Ok(WatchResult::Rejected);
return Ok(InboundWatchResult::Rejected);
}
// Check watch table for limits
@ -1011,18 +1011,18 @@ where
self.limits.public_watch_limit
};
if watch_count >= watch_limit {
return Ok(WatchResult::Rejected);
return Ok(InboundWatchResult::Rejected);
}
// Ok this is an acceptable new watch, add it
let watch_list = self.watched_records.entry(rtk).or_default();
let expiration = params.expiration;
watch_list.watches.push(Watch {
watch_list.watches.push(InboundWatch {
params,
id,
changed: ValueSubkeyRangeSet::new(),
});
Ok(WatchResult::Created { id, expiration })
Ok(InboundWatchResult::Created { id, expiration })
}
/// Add or update an inbound record watch for changes
@ -1030,17 +1030,17 @@ where
pub async fn watch_record(
&mut self,
key: TypedKey,
mut params: WatchParameters,
mut params: InboundWatchParameters,
opt_watch_id: Option<u64>,
) -> VeilidAPIResult<WatchResult> {
) -> VeilidAPIResult<InboundWatchResult> {
// If count is zero then we're cancelling a watch completely
if params.count == 0 {
if let Some(watch_id) = opt_watch_id {
let cancelled = self.cancel_watch(key, watch_id, params.watcher).await?;
if cancelled {
return Ok(WatchResult::Cancelled);
return Ok(InboundWatchResult::Cancelled);
}
return Ok(WatchResult::Rejected);
return Ok(InboundWatchResult::Rejected);
}
apibail_internal!("shouldn't have let a None watch id get here");
}
@ -1058,10 +1058,10 @@ where
if let Some(watch_id) = opt_watch_id {
let cancelled = self.cancel_watch(key, watch_id, params.watcher).await?;
if cancelled {
return Ok(WatchResult::Cancelled);
return Ok(InboundWatchResult::Cancelled);
}
}
return Ok(WatchResult::Rejected);
return Ok(InboundWatchResult::Rejected);
}
// Make a closure to check for member vs anonymous
@ -1071,7 +1071,7 @@ where
Box::new(move |watcher| owner == watcher || schema.is_member(&watcher))
}) else {
// Record not found
return Ok(WatchResult::Rejected);
return Ok(InboundWatchResult::Rejected);
};
// Create or update depending on if a watch id is specified or not
@ -1128,8 +1128,8 @@ where
pub fn move_watches(
&mut self,
key: TypedKey,
in_watch: Option<(WatchList, bool)>,
) -> Option<(WatchList, bool)> {
in_watch: Option<(InboundWatchList, bool)>,
) -> Option<(InboundWatchList, bool)> {
let rtk = RecordTableKey { key };
let out = self.watched_records.remove(&rtk);
if let Some(in_watch) = in_watch {

32
veilid-core/src/storage_manager/record_store/opened_record.rs
View file

@ -1,21 +1,5 @@
use super::*;
#[derive(Clone, Debug)]
pub(in crate::storage_manager) struct ActiveWatch {
/// The watch id returned from the watch node
pub id: u64,
/// The expiration of a successful watch
pub expiration_ts: Timestamp,
/// Which node accepted the watch
pub watch_node: NodeRef,
/// Which private route is responsible for receiving ValueChanged notifications
pub opt_value_changed_route: Option<PublicKey>,
/// Which subkeys we are watching
pub subkeys: ValueSubkeyRangeSet,
/// How many notifications are left
pub count: u32,
}
/// The state associated with a local record when it is opened
/// This is not serialized to storage as it is ephemeral for the lifetime of the opened record
#[derive(Clone, Debug, Default)]
@ -27,9 +11,6 @@ pub(in crate::storage_manager) struct OpenedRecord {
/// The safety selection in current use
safety_selection: SafetySelection,
/// Active watch we have on this record
active_watch: Option<ActiveWatch>,
}
impl OpenedRecord {
@ -37,7 +18,6 @@ impl OpenedRecord {
Self {
writer,
safety_selection,
active_watch: None,
}
}
@ -54,16 +34,4 @@ impl OpenedRecord {
pub fn set_safety_selection(&mut self, safety_selection: SafetySelection) {
self.safety_selection = safety_selection;
}
pub fn set_active_watch(&mut self, active_watch: ActiveWatch) {
self.active_watch = Some(active_watch);
}
pub fn clear_active_watch(&mut self) {
self.active_watch = None;
}
pub fn active_watch(&self) -> Option<ActiveWatch> {
self.active_watch.clone()
}
}

164
veilid-core/src/storage_manager/set_value.rs
View file

@ -6,8 +6,6 @@ impl_veilid_log_facility!("stor");
struct OutboundSetValueContext {
/// The latest value of the subkey, may be the value passed in
pub value: Arc<SignedValueData>,
/// The nodes that have set the value so far (up to the consensus count)
pub value_nodes: Vec<NodeRef>,
/// The number of non-sets since the last set we have received
pub missed_since_last_set: usize,
/// The parsed schema from the descriptor if we have one
@ -39,11 +37,9 @@ impl StorageManager {
let routing_domain = RoutingDomain::PublicInternet;
// Get the DHT parameters for 'SetValue'
let (key_count, get_consensus_count, set_consensus_count, fanout, timeout_us) =
self.config().with(|c| {
let (key_count, consensus_count, fanout, timeout_us) = self.config().with(|c| {
(
c.network.dht.max_find_node_count as usize,
c.network.dht.get_value_count as usize,
c.network.dht.set_value_count as usize,
c.network.dht.set_value_fanout as usize,
TimestampDuration::from(ms_to_us(c.network.dht.set_value_timeout_ms)),
@ -71,10 +67,9 @@ impl StorageManager {
let schema = descriptor.schema()?;
let context = Arc::new(Mutex::new(OutboundSetValueContext {
value,
value_nodes: vec![],
missed_since_last_set: 0,
schema,
send_partial_update: false,
send_partial_update: true,
}));
// Routine to call to generate fanout
@ -82,7 +77,8 @@ impl StorageManager {
let context = context.clone();
let registry = self.registry();
Arc::new(move |next_node: NodeRef| {
Arc::new(
move |next_node: NodeRef| -> PinBoxFutureStatic<FanoutCallResult> {
let registry = registry.clone();
let context = context.clone();
let descriptor = descriptor.clone();
@ -98,7 +94,7 @@ impl StorageManager {
};
// send across the wire
let sva = network_result_try!(
let sva = match
rpc_processor
.rpc_call_set_value(
Destination::direct(next_node.routing_domain_filtered(routing_domain))
@ -109,8 +105,18 @@ impl StorageManager {
(*descriptor).clone(),
send_descriptor,
)
.await?
);
.await? {
NetworkResult::Timeout => {
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Timeout});
}
NetworkResult::ServiceUnavailable(_) |
NetworkResult::NoConnection(_) |
NetworkResult::AlreadyExists(_) |
NetworkResult::InvalidMessage(_) => {
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
NetworkResult::Value(v) => v
};
// If the node was close enough to possibly set the value
let mut ctx = context.lock();
@ -119,22 +125,17 @@ impl StorageManager {
// Return peers if we have some
veilid_log!(registry debug target:"network_result", "SetValue missed: {}, fanout call returned peers {}", ctx.missed_since_last_set, sva.answer.peers.len());
return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers}));
return Ok(FanoutCallOutput{peer_info_list:sva.answer.peers, disposition: FanoutCallDisposition::Rejected});
}
// See if we got a value back
// See if we got a newer value back
let Some(value) = sva.answer.value else {
// No newer value was found and returned, so increase our consensus count
ctx.value_nodes.push(next_node);
ctx.missed_since_last_set = 0;
// Send an update since it was set
if ctx.value_nodes.len() == 1 {
ctx.send_partial_update = true;
}
// Return peers if we have some
veilid_log!(registry debug target:"network_result", "SetValue returned no value, fanout call returned peers {}", sva.answer.peers.len());
return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers}));
return Ok(FanoutCallOutput{peer_info_list:sva.answer.peers, disposition: FanoutCallDisposition::Accepted});
};
// Keep the value if we got one and it is newer and it passes schema validation
@ -147,24 +148,12 @@ impl StorageManager {
value.value_data(),
) {
// Validation failed, ignore this value and pretend we never saw this node
return Ok(NetworkResult::invalid_message(format!(
"Schema validation failed on subkey {}",
subkey
)));
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
// If we got a value back it should be different than the one we are setting
// But in the case of a benign bug, we can just move to the next node
if ctx.value.value_data() == value.value_data() {
ctx.value_nodes.push(next_node);
ctx.missed_since_last_set = 0;
// Send an update since it was set
if ctx.value_nodes.len() == 1 {
ctx.send_partial_update = true;
}
return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers}));
return Ok(FanoutCallOutput{peer_info_list:sva.answer.peers, disposition: FanoutCallDisposition::Invalid});
}
// We have a prior value, ensure this is a newer sequence number
@ -174,21 +163,21 @@ impl StorageManager {
// If the sequence number is older node should have not returned a value here.
// Skip this node and its closer list because it is misbehaving
// Ignore this value and pretend we never saw this node
return Ok(NetworkResult::invalid_message("Sequence number is older"));
return Ok(FanoutCallOutput{peer_info_list: vec![], disposition: FanoutCallDisposition::Invalid});
}
// If the sequence number is greater or equal, keep it
// even if the sequence number is the same, accept all conflicts in an attempt to resolve them
ctx.value = Arc::new(value);
// One node has shown us this value so far
ctx.value_nodes = vec![next_node];
ctx.missed_since_last_set = 0;
// Send an update since the value changed
ctx.send_partial_update = true;
Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers}))
Ok(FanoutCallOutput{peer_info_list:sva.answer.peers, disposition: FanoutCallDisposition::AcceptedNewerRestart})
}.instrument(tracing::trace_span!("fanout call_routine"))) as PinBoxFuture<FanoutCallResult>
})
},
)
};
// Routine to call to check if we're done at each step
@ -196,43 +185,37 @@ impl StorageManager {
let context = context.clone();
let out_tx = out_tx.clone();
let registry = self.registry();
Arc::new(move |_closest_nodes: &[NodeRef]| {
Arc::new(move |fanout_result: &FanoutResult| -> bool {
let mut ctx = context.lock();
// send partial update if desired
if ctx.send_partial_update {
match fanout_result.kind {
FanoutResultKind::Incomplete => {
// Send partial update if desired, if we've gotten at least consensus node
if ctx.send_partial_update && !fanout_result.consensus_nodes.is_empty() {
ctx.send_partial_update = false;
// return partial result
let fanout_result = FanoutResult {
kind: FanoutResultKind::Partial,
value_nodes: ctx.value_nodes.clone(),
};
// Return partial result
let out = OutboundSetValueResult {
fanout_result,
fanout_result: fanout_result.clone(),
signed_value_data: ctx.value.clone(),
};
veilid_log!(registry debug "Sending partial SetValue result: {:?}", out);
if let Err(e) = out_tx.send(Ok(out)) {
veilid_log!(registry debug "Sending partial SetValue result failed: {}", e);
}
}
// If we have reached set consensus (the max consensus we care about), return done
if ctx.value_nodes.len() >= set_consensus_count {
return Some(());
// Keep going
false
}
FanoutResultKind::Timeout | FanoutResultKind::Exhausted => {
// Signal we're done
true
}
FanoutResultKind::Consensus => {
// Signal we're done
true
}
// If we have missed get_consensus count (the minimum consensus we care about) or more since our last set, return done
// This keeps the traversal from searching too many nodes when we aren't converging
// Only do this if we have gotten at least the get_consensus (the minimum consensus we care about)
if ctx.value_nodes.len() >= get_consensus_count
&& ctx.missed_since_last_set >= get_consensus_count
{
return Some(());
}
None
})
};
@ -248,21 +231,16 @@ impl StorageManager {
key,
key_count,
fanout,
consensus_count,
timeout_us,
capability_fanout_node_info_filter(vec![CAP_DHT]),
call_routine,
check_done,
);
let kind = match fanout_call.run(init_fanout_queue).await {
// If we don't finish in the timeout (too much time passed checking for consensus)
TimeoutOr::Timeout => FanoutResultKind::Timeout,
// If we finished with or without consensus (enough nodes returning the same value)
TimeoutOr::Value(Ok(Some(()))) => FanoutResultKind::Finished,
// If we ran out of nodes before getting consensus)
TimeoutOr::Value(Ok(None)) => FanoutResultKind::Exhausted,
// Failed
TimeoutOr::Value(Err(e)) => {
let fanout_result = match fanout_call.run(init_fanout_queue).await {
Ok(v) => v,
Err(e) => {
// If we finished with an error, return that
veilid_log!(registry debug "SetValue fanout error: {}", e);
if let Err(e) = out_tx.send(Err(e.into())) {
@ -272,19 +250,20 @@ impl StorageManager {
}
};
let ctx = context.lock();
let fanout_result = FanoutResult {
kind,
value_nodes: ctx.value_nodes.clone(),
};
veilid_log!(registry debug "SetValue Fanout: {:?}", fanout_result);
veilid_log!(registry debug "SetValue Fanout: {:#}", fanout_result);
if let Err(e) = out_tx.send(Ok(OutboundSetValueResult {
let out = {
let ctx = context.lock();
OutboundSetValueResult {
fanout_result,
signed_value_data: ctx.value.clone(),
})) {
}
};
if let Err(e) = out_tx.send(Ok(out)) {
veilid_log!(registry debug "Sending SetValue result failed: {}", e);
}
}
.instrument(tracing::trace_span!("outbound_set_value fanout routine")),
),
@ -321,19 +300,19 @@ impl StorageManager {
return false;
}
};
let is_partial = result.fanout_result.kind.is_partial();
let is_incomplete = result.fanout_result.kind.is_incomplete();
let lvd = last_value_data.lock().clone();
let value_data = match this.process_outbound_set_value_result(key, subkey, lvd, safety_selection, result).await {
Ok(Some(v)) => v,
Ok(None) => {
return is_partial;
return is_incomplete;
}
Err(e) => {
veilid_log!(registry debug "Deferred fanout error: {}", e);
return false;
}
};
if is_partial {
if is_incomplete {
// If more partial results show up, don't send an update until we're done
return true;
}
@ -364,27 +343,34 @@ impl StorageManager {
#[instrument(level = "trace", target = "stor", skip_all, err)]
pub(super) async fn process_outbound_set_value_result(
&self,
key: TypedKey,
record_key: TypedKey,
subkey: ValueSubkey,
last_value_data: ValueData,
safety_selection: SafetySelection,
result: set_value::OutboundSetValueResult,
) -> Result<Option<ValueData>, VeilidAPIError> {
// Get cryptosystem
let crypto = self.crypto();
let Some(vcrypto) = crypto.get(record_key.kind) else {
apibail_generic!("unsupported cryptosystem");
};
// Regain the lock after network access
let mut inner = self.inner.lock().await;
// Report on fanout result offline
let was_offline = self.check_fanout_set_offline(key, subkey, &result.fanout_result);
let was_offline = self.check_fanout_set_offline(record_key, subkey, &result.fanout_result);
if was_offline {
// Failed to write, try again later
Self::add_offline_subkey_write_inner(&mut inner, key, subkey, safety_selection);
Self::add_offline_subkey_write_inner(&mut inner, record_key, subkey, safety_selection);
}
// Keep the list of nodes that returned a value for later reference
Self::process_fanout_results_inner(
&mut inner,
key,
core::iter::once((subkey, &result.fanout_result)),
&vcrypto,
record_key,
core::iter::once((ValueSubkeyRangeSet::single(subkey), result.fanout_result)),
true,
self.config()
.with(|c| c.network.dht.set_value_count as usize),
@ -396,10 +382,10 @@ impl StorageManager {
Self::handle_set_local_value_inner(
&mut inner,
key,
record_key,
subkey,
result.signed_value_data.clone(),
WatchUpdateMode::UpdateAll,
InboundWatchUpdateMode::UpdateAll,
)
.await?;
@ -500,7 +486,7 @@ impl StorageManager {
key,
subkey,
value,
WatchUpdateMode::ExcludeTarget(target),
InboundWatchUpdateMode::ExcludeTarget(target),
)
.await
} else {
@ -510,7 +496,7 @@ impl StorageManager {
subkey,
value,
actual_descriptor,
WatchUpdateMode::ExcludeTarget(target),
InboundWatchUpdateMode::ExcludeTarget(target),
)
.await
};

67
veilid-core/src/storage_manager/tasks/check_active_watches.rs
View file

@ -1,67 +0,0 @@
use super::*;
impl StorageManager {
// Check if client-side watches on opened records either have dead nodes or if the watch has expired
#[instrument(level = "trace", target = "stor", skip_all, err)]
pub(super) async fn check_active_watches_task_routine(
&self,
_stop_token: StopToken,
_last_ts: Timestamp,
_cur_ts: Timestamp,
) -> EyreResult<()> {
{
let mut inner = self.inner.lock().await;
let routing_table = self.routing_table();
let update_callback = self.update_callback();
let cur_ts = Timestamp::now();
for (k, v) in inner.opened_records.iter_mut() {
// If no active watch, then skip this
let Some(active_watch) = v.active_watch() else {
continue;
};
// See if the active watch's node is dead
let mut is_dead = false;
if !active_watch.watch_node.state(cur_ts).is_alive() {
// Watched node is dead
is_dead = true;
}
// See if the private route we're using is dead
if !is_dead {
if let Some(value_changed_route) = active_watch.opt_value_changed_route {
if routing_table
.route_spec_store()
.get_route_id_for_key(&value_changed_route)
.is_none()
{
// Route we would receive value changes on is dead
is_dead = true;
}
}
}
// See if the watch is expired
if !is_dead && active_watch.expiration_ts <= cur_ts {
// Watch has expired
is_dead = true;
}
if is_dead {
v.clear_active_watch();
// Send valuechange with dead count and no subkeys
update_callback(VeilidUpdate::ValueChange(Box::new(VeilidValueChange {
key: *k,
subkeys: ValueSubkeyRangeSet::new(),
count: 0,
value: None,
})));
}
}
}
Ok(())
}
}

2
veilid-core/src/storage_manager/tasks/check_watched_records.rs → veilid-core/src/storage_manager/tasks/check_inbound_watches.rs
View file

@ -3,7 +3,7 @@ use super::*;
impl StorageManager {
// Check if server-side watches have expired
#[instrument(level = "trace", target = "stor", skip_all, err)]
pub(super) async fn check_watched_records_task_routine(
pub(super) async fn check_inbound_watches_task_routine(
&self,
_stop_token: StopToken,
_last_ts: Timestamp,

37
veilid-core/src/storage_manager/tasks/check_outbound_watches.rs Normal file
View file

@ -0,0 +1,37 @@
use super::*;
impl StorageManager {
// Check if client-side watches on opened records either have dead nodes or if the watch has expired
//#[instrument(level = "trace", target = "stor", skip_all, err)]
pub(super) async fn check_outbound_watches_task_routine(
&self,
_stop_token: StopToken,
_last_ts: Timestamp,
_cur_ts: Timestamp,
) -> EyreResult<()> {
let inner = &mut *self.inner.lock().await;
let cur_ts = Timestamp::now();
// Update per-node watch states
// Desired state updates are performed by get_next_outbound_watch_operation
inner.outbound_watch_manager.update_per_node_states(cur_ts);
// Iterate all outbound watches and determine what work needs doing if any
for (k, v) in &mut inner.outbound_watch_manager.outbound_watches {
// Get next work on watch and queue it if we have something to do
if let Some(op_fut) = self.get_next_outbound_watch_operation(*k, None, cur_ts, v) {
self.background_operation_processor.add_future(op_fut);
};
}
// Iterate all queued change inspections and do them
for (k, v) in inner.outbound_watch_manager.needs_change_inspection.drain() {
// Get next work on watch and queue it if we have something to do
let op_fut = self.get_change_inspection_operation(k, v);
self.background_operation_processor.add_future(op_fut);
}
Ok(())
}
}

28
veilid-core/src/storage_manager/tasks/mod.rs
View file

@ -1,7 +1,8 @@
pub mod check_active_watches;
pub mod check_watched_records;
pub mod check_inbound_watches;
pub mod check_outbound_watches;
pub mod flush_record_stores;
pub mod offline_subkey_writes;
pub mod save_metadata;
pub mod send_value_changes;
use super::*;
@ -16,7 +17,9 @@ impl StorageManager {
flush_record_stores_task,
flush_record_stores_task_routine
);
// Set save metadata task
veilid_log!(self debug "starting save metadata task");
impl_setup_task!(self, Self, save_metadata_task, save_metadata_task_routine);
// Set offline subkey writes tick task
veilid_log!(self debug "starting offline subkey writes task");
impl_setup_task!(
@ -40,8 +43,8 @@ impl StorageManager {
impl_setup_task!(
self,
Self,
check_active_watches_task,
check_active_watches_task_routine
check_outbound_watches_task,
check_outbound_watches_task_routine
);
// Set check watched records tick task
@ -49,8 +52,8 @@ impl StorageManager {
impl_setup_task!(
self,
Self,
check_watched_records_task,
check_watched_records_task_routine
check_inbound_watches_task,
check_inbound_watches_task_routine
);
}
@ -59,11 +62,14 @@ impl StorageManager {
// Run the flush stores task
self.flush_record_stores_task.tick().await?;
// Run the flush stores task
self.save_metadata_task.tick().await?;
// Check active watches
self.check_active_watches_task.tick().await?;
self.check_outbound_watches_task.tick().await?;
// Check watched records
self.check_watched_records_task.tick().await?;
self.check_inbound_watches_task.tick().await?;
// Run online-only tasks
if self.dht_is_online() {
@ -81,11 +87,11 @@ impl StorageManager {
#[instrument(level = "trace", target = "stor", skip_all)]
pub(super) async fn cancel_tasks(&self) {
veilid_log!(self debug "stopping check watched records task");
if let Err(e) = self.check_watched_records_task.stop().await {
if let Err(e) = self.check_inbound_watches_task.stop().await {
veilid_log!(self warn "check_watched_records_task not stopped: {}", e);
}
veilid_log!(self debug "stopping check active watches task");
if let Err(e) = self.check_active_watches_task.stop().await {
if let Err(e) = self.check_outbound_watches_task.stop().await {
veilid_log!(self warn "check_active_watches_task not stopped: {}", e);
}
veilid_log!(self debug "stopping send value changes task");

28
veilid-core/src/storage_manager/tasks/offline_subkey_writes.rs
View file

@ -28,9 +28,9 @@ struct WorkItem {
#[derive(Debug)]
struct WorkItemResult {
key: TypedKey,
record_key: TypedKey,
written_subkeys: ValueSubkeyRangeSet,
fanout_results: Vec<(ValueSubkey, FanoutResult)>,
fanout_results: Vec<(ValueSubkeyRangeSet, FanoutResult)>,
}
impl StorageManager {
@ -74,7 +74,7 @@ impl StorageManager {
while let Ok(Ok(res)) = res_rx.recv_async().timeout_at(stop_token.clone()).await {
match res {
Ok(result) => {
let partial = result.fanout_result.kind.is_partial();
let partial = result.fanout_result.kind.is_incomplete();
// Skip partial results in offline subkey write mode
if partial {
continue;
@ -90,7 +90,7 @@ impl StorageManager {
key,
subkey,
result.signed_value_data.clone(),
WatchUpdateMode::UpdateAll,
InboundWatchUpdateMode::UpdateAll,
)
.await?;
}
@ -121,7 +121,7 @@ impl StorageManager {
work_item: WorkItem,
) -> EyreResult<WorkItemResult> {
let mut written_subkeys = ValueSubkeyRangeSet::new();
let mut fanout_results = Vec::<(ValueSubkey, FanoutResult)>::new();
let mut fanout_results = Vec::<(ValueSubkeyRangeSet, FanoutResult)>::new();
for subkey in work_item.subkeys.iter() {
if poll!(stop_token.clone()).is_ready() {
@ -155,11 +155,11 @@ impl StorageManager {
if !was_offline {
written_subkeys.insert(subkey);
}
fanout_results.push((subkey, result.fanout_result));
fanout_results.push((ValueSubkeyRangeSet::single(subkey), result.fanout_result));
}
Ok(WorkItemResult {
key: work_item.key,
record_key: work_item.key,
written_subkeys,
fanout_results,
})
@ -192,7 +192,7 @@ impl StorageManager {
veilid_log!(self debug "Offline write result: {:?}", result);
// Get the offline subkey write record
match inner.offline_subkey_writes.entry(result.key) {
match inner.offline_subkey_writes.entry(result.record_key) {
std::collections::hash_map::Entry::Occupied(mut o) => {
let finished = {
let osw = o.get_mut();
@ -208,20 +208,24 @@ impl StorageManager {
osw.subkeys.is_empty()
};
if finished {
veilid_log!(self debug "Offline write finished key {}", result.key);
veilid_log!(self debug "Offline write finished key {}", result.record_key);
o.remove();
}
}
std::collections::hash_map::Entry::Vacant(_) => {
veilid_log!(self warn "offline write work items should always be on offline_subkey_writes entries that exist: ignoring key {}", result.key);
veilid_log!(self warn "offline write work items should always be on offline_subkey_writes entries that exist: ignoring key {}", result.record_key);
}
}
// Keep the list of nodes that returned a value for later reference
let crypto = self.crypto();
let vcrypto = crypto.get(result.record_key.kind).unwrap();
Self::process_fanout_results_inner(
&mut inner,
result.key,
result.fanout_results.iter().map(|x| (x.0, &x.1)),
&vcrypto,
result.record_key,
result.fanout_results.into_iter().map(|x| (x.0, x.1)),
true,
consensus_count,
);

16
veilid-core/src/storage_manager/tasks/save_metadata.rs Normal file
View file

@ -0,0 +1,16 @@
use super::*;
impl StorageManager {
// Save metadata to disk
#[instrument(level = "trace", target = "stor", skip_all, err)]
pub(super) async fn save_metadata_task_routine(
&self,
_stop_token: StopToken,
_last_ts: Timestamp,
_cur_ts: Timestamp,
) -> EyreResult<()> {
let mut inner = self.inner.lock().await;
self.save_metadata_inner(&mut inner).await?;
Ok(())
}
}

1278
veilid-core/src/storage_manager/watch_value.rs
View file

File diff suppressed because it is too large Load diff

55
veilid-core/src/veilid_api/debug.rs
View file

@ -1474,6 +1474,11 @@ impl VeilidAPI {
out += &storage_manager.debug_opened_records().await;
out
}
"watched" => {
let mut out = "Watched Records:\n".to_string();
out += &storage_manager.debug_watched_records().await;
out
}
"offline" => {
let mut out = "Offline Records:\n".to_string();
out += &storage_manager.debug_offline_records().await;
@ -1489,6 +1494,11 @@ impl VeilidAPI {
let registry = self.core_context()?.registry();
let storage_manager = registry.storage_manager();
self.with_debug_cache(|dc| {
dc.opened_record_contexts.clear();
});
storage_manager.close_all_records().await?;
let scope = get_debug_argument_at(&args, 1, "debug_record_purge", "scope", get_string)?;
let bytes = get_debug_argument_at(&args, 2, "debug_record_purge", "bytes", get_number).ok();
let out = match scope.as_str() {
@ -1786,13 +1796,14 @@ impl VeilidAPI {
get_subkeys,
)
.ok()
.map(Some)
.unwrap_or_else(|| {
rest_defaults = true;
Default::default()
None
});
let expiration = if rest_defaults {
Default::default()
let opt_expiration = if rest_defaults {
None
} else {
get_debug_argument_at(
&args,
@ -1802,14 +1813,20 @@ impl VeilidAPI {
parse_duration,
)
.ok()
.map(|dur| dur + get_timestamp())
.map(|dur| {
if dur == 0 {
None
} else {
Some(Timestamp::new(dur + get_timestamp()))
}
})
.unwrap_or_else(|| {
rest_defaults = true;
Default::default()
None
})
};
let count = if rest_defaults {
u32::MAX
None
} else {
get_debug_argument_at(
&args,
@ -1819,15 +1836,16 @@ impl VeilidAPI {
get_number,
)
.ok()
.map(Some)
.unwrap_or_else(|| {
rest_defaults = true;
u32::MAX
Some(u32::MAX)
})
};
// Do a record watch
let ts = match rc
.watch_dht_values(key, subkeys, Timestamp::new(expiration), count)
let active = match rc
.watch_dht_values(key, subkeys, opt_expiration, count)
.await
{
Err(e) => {
@ -1835,10 +1853,10 @@ impl VeilidAPI {
}
Ok(v) => v,
};
if ts.as_u64() == 0 {
if !active {
return Ok("Failed to watch value".to_owned());
}
Ok(format!("Success: expiration={:?}", display_ts(ts.as_u64())))
Ok("Success".to_owned())
}
async fn debug_record_cancel(&self, args: Vec<String>) -> VeilidAPIResult<String> {
@ -1858,8 +1876,7 @@ impl VeilidAPI {
"subkeys",
get_subkeys,
)
.ok()
.unwrap_or_default();
.ok();
// Do a record watch cancel
let still_active = match rc.cancel_dht_watch(key, subkeys).await {
@ -1906,7 +1923,10 @@ impl VeilidAPI {
})
};
let subkeys = get_debug_argument_at(
let subkeys = if rest_defaults {
None
} else {
get_debug_argument_at(
&args,
2 + opt_arg_add,
"debug_record_inspect",
@ -1914,10 +1934,7 @@ impl VeilidAPI {
get_subkeys,
)
.ok()
.unwrap_or_else(|| {
rest_defaults = true;
Default::default()
});
};
// Do a record inspect
let report = match rc.inspect_dht_record(key, subkeys, scope).await {
@ -2115,7 +2132,7 @@ RPC Operations:
appreply [#id] <data> - Reply to an 'App Call' RPC received by this node
DHT Operations:
record list <local|remote|opened|offline> - display the dht records in the store
record list <local|remote|opened|offline|watched> - display the dht records in the store
purge <local|remote> [bytes] - clear all dht records optionally down to some total size
create <dhtschema> [<cryptokind> [<safety>]] - create a new dht record
open <key>[+<safety>] [<writer>] - open an existing dht record

13
veilid-core/src/veilid_api/json_api/routing_context.rs
View file

@ -78,19 +78,20 @@ pub enum RoutingContextRequestOp {
WatchDhtValues {
#[schemars(with = "String")]
key: TypedKey,
subkeys: ValueSubkeyRangeSet,
expiration: Timestamp,
count: u32,
subkeys: Option<ValueSubkeyRangeSet>,
expiration: Option<Timestamp>,
count: Option<u32>,
},
CancelDhtWatch {
#[schemars(with = "String")]
key: TypedKey,
subkeys: ValueSubkeyRangeSet,
subkeys: Option<ValueSubkeyRangeSet>,
},
InspectDhtRecord {
#[schemars(with = "String")]
key: TypedKey,
subkeys: ValueSubkeyRangeSet,
subkeys: Option<ValueSubkeyRangeSet>,
#[schemars(default)]
scope: DHTReportScope,
},
}
@ -149,7 +150,7 @@ pub enum RoutingContextResponseOp {
},
WatchDhtValues {
#[serde(flatten)]
result: ApiResult<Timestamp>,
result: ApiResult<bool>,
},
CancelDhtWatch {
#[serde(flatten)]

48
veilid-core/src/veilid_api/routing_context.rs
View file

@ -398,13 +398,18 @@ impl RoutingContext {
///
/// There is only one watch permitted per record. If a change to a watch is desired, the previous one will be overwritten.
/// * `key` is the record key to watch. it must first be opened for reading or writing.
/// * `subkeys` is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
/// * `expiration` is the desired timestamp of when to automatically terminate the watch, in microseconds. If this value is less than `network.rpc.timeout_ms` milliseconds in the future, this function will return an error immediately.
/// * `count` is the number of times the watch will be sent, maximum. A zero value here is equivalent to a cancellation.
/// * `subkeys`:
/// - None: specifies watching the entire range of subkeys.
/// - Some(range): is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
/// * `expiration`:
/// - None: specifies a watch with no expiration
/// - Some(timestamp): the desired timestamp of when to automatically terminate the watch, in microseconds. If this value is less than `network.rpc.timeout_ms` milliseconds in the future, this function will return an error immediately.
/// * `count:
/// - None: specifies a watch count of u32::MAX
/// - Some(count): is the number of times the watch will be sent, maximum. A zero value here is equivalent to a cancellation.
///
/// Returns a timestamp of when the watch will expire. All watches are guaranteed to expire at some point in the future,
/// and the returned timestamp will be no later than the requested expiration, but -may- be before the requested expiration.
/// If the returned timestamp is zero it indicates that the watch creation or update has failed. In the case of a faild update, the watch is considered cancelled.
/// Returns Ok(true) if a watch is active for this record.
/// Returns Ok(false) if the entire watch has been cancelled.
///
/// DHT watches are accepted with the following conditions:
/// * First-come first-served basis for arbitrary unauthenticated readers, up to network.dht.public_watch_limit per record.
@ -415,12 +420,15 @@ impl RoutingContext {
pub async fn watch_dht_values(
&self,
key: TypedKey,
subkeys: ValueSubkeyRangeSet,
expiration: Timestamp,
count: u32,
) -> VeilidAPIResult<Timestamp> {
subkeys: Option<ValueSubkeyRangeSet>,
expiration: Option<Timestamp>,
count: Option<u32>,
) -> VeilidAPIResult<bool> {
veilid_log!(self debug
"RoutingContext::watch_dht_values(self: {:?}, key: {:?}, subkeys: {:?}, expiration: {}, count: {})", self, key, subkeys, expiration, count);
"RoutingContext::watch_dht_values(self: {:?}, key: {:?}, subkeys: {:?}, expiration: {:?}, count: {:?})", self, key, subkeys, expiration, count);
let subkeys = subkeys.unwrap_or_default();
let expiration = expiration.unwrap_or_default();
let count = count.unwrap_or(u32::MAX);
Crypto::validate_crypto_kind(key.kind)?;
@ -431,20 +439,24 @@ impl RoutingContext {
/// Cancels a watch early.
///
/// This is a convenience function that cancels watching all subkeys in a range. The subkeys specified here
/// are subtracted from the watched subkey range. If no range is specified, this is equivalent to cancelling the entire range of subkeys.
/// are subtracted from the currently-watched subkey range.
/// * `subkeys`:
/// - None: specifies watching the entire range of subkeys.
/// - Some(range): is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
/// Only the subkey range is changed, the expiration and count remain the same.
/// If no subkeys remain, the watch is entirely cancelled and will receive no more updates.
///
/// Returns Ok(true) if there is any remaining watch for this record.
/// Returns Ok(true) if a watch is active for this record.
/// Returns Ok(false) if the entire watch has been cancelled.
#[instrument(target = "veilid_api", level = "debug", fields(__VEILID_LOG_KEY = self.log_key()), ret, err)]
pub async fn cancel_dht_watch(
&self,
key: TypedKey,
subkeys: ValueSubkeyRangeSet,
subkeys: Option<ValueSubkeyRangeSet>,
) -> VeilidAPIResult<bool> {
veilid_log!(self debug
"RoutingContext::cancel_dht_watch(self: {:?}, key: {:?}, subkeys: {:?}", self, key, subkeys);
let subkeys = subkeys.unwrap_or_default();
Crypto::validate_crypto_kind(key.kind)?;
@ -457,8 +469,9 @@ impl RoutingContext {
/// to see what needs updating locally.
///
/// * `key` is the record key to inspect. it must first be opened for reading or writing.
/// * `subkeys` is the the range of subkeys to inspect. The range must not exceed 512 discrete non-overlapping or adjacent subranges.
/// If no range is specified, this is equivalent to inspecting the entire range of subkeys. In total, the list of subkeys returned will be truncated at 512 elements.
/// * `subkeys`:
/// - None: specifies inspecting the entire range of subkeys.
/// - Some(range): is the the range of subkeys to inspect. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
/// * `scope` is what kind of range the inspection has:
///
/// - DHTReportScope::Local
@ -495,11 +508,12 @@ impl RoutingContext {
pub async fn inspect_dht_record(
&self,
key: TypedKey,
subkeys: ValueSubkeyRangeSet,
subkeys: Option<ValueSubkeyRangeSet>,
scope: DHTReportScope,
) -> VeilidAPIResult<DHTRecordReport> {
veilid_log!(self debug
"RoutingContext::inspect_dht_record(self: {:?}, key: {:?}, subkeys: {:?}, scope: {:?})", self, key, subkeys, scope);
let subkeys = subkeys.unwrap_or_default();
Crypto::validate_crypto_kind(key.kind)?;

34
veilid-core/src/veilid_api/types/dht/dht_record_report.rs
View file

@ -51,6 +51,20 @@ impl DHTRecordReport {
pub fn network_seqs(&self) -> &[ValueSeqNum] {
&self.network_seqs
}
pub fn changed_subkeys(&self) -> ValueSubkeyRangeSet {
let mut changed = ValueSubkeyRangeSet::new();
for ((sk, lseq), nseq) in self
.subkeys
.iter()
.zip(self.local_seqs.iter())
.zip(self.network_seqs.iter())
{
if nseq > lseq {
changed.insert(sk);
}
}
changed
}
}
impl fmt::Debug for DHTRecordReport {
@ -65,9 +79,21 @@ impl fmt::Debug for DHTRecordReport {
)
}
}
/// DHT Record Report Scope
#[derive(
Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
Copy,
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
JsonSchema,
Default,
)]
#[cfg_attr(
all(target_arch = "wasm32", target_os = "unknown"),
@ -77,6 +103,7 @@ impl fmt::Debug for DHTRecordReport {
pub enum DHTReportScope {
/// Return only the local copy sequence numbers
/// Useful for seeing what subkeys you have locally and which ones have not been retrieved
#[default]
Local = 0,
/// Return the local sequence numbers and the network sequence numbers with GetValue fanout parameters
/// Provides an independent view of both the local sequence numbers and the network sequence numbers for nodes that
@ -100,8 +127,3 @@ pub enum DHTReportScope {
/// Useful for determine which subkeys would change on an SetValue operation
UpdateSet = 4,
}
impl Default for DHTReportScope {
fn default() -> Self {
Self::Local
}
}

1
veilid-flutter/example/android/app/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
.cxx

2
veilid-flutter/example/integration_test/app_test.dart
View file

@ -1,6 +1,6 @@
@Timeout(Duration(seconds: 120))
library veilid_flutter_integration_test;
library;
import 'package:flutter_test/flutter_test.dart';
import 'package:integration_test/integration_test.dart';

5
veilid-flutter/example/lib/app.dart
View file

@ -76,19 +76,14 @@ class _MyAppState extends State<MyApp> with UiLoggy {
switch (log.logLevel) {
case VeilidLogLevel.error:
loggy.error(log.message, error, stackTrace);
break;
case VeilidLogLevel.warn:
loggy.warning(log.message, error, stackTrace);
break;
case VeilidLogLevel.info:
loggy.info(log.message, error, stackTrace);
break;
case VeilidLogLevel.debug:
loggy.debug(log.message, error, stackTrace);
break;
case VeilidLogLevel.trace:
loggy.trace(log.message, error, stackTrace);
break;
}
}

2
veilid-flutter/example/pubspec.lock
View file

@ -450,7 +450,7 @@ packages:
path: ".."
relative: true
source: path
version: "0.4.3"
version: "0.4.4"
veilid_test:
dependency: "direct dev"
description:

4
veilid-flutter/example/test/widget_test.dart
View file

@ -11,14 +11,14 @@ import 'package:flutter_test/flutter_test.dart';
import 'package:veilid_example/app.dart';
void main() {
testWidgets('Verify Platform version', (WidgetTester tester) async {
testWidgets('Verify Platform version', (tester) async {
// Build our app and trigger a frame.
await tester.pumpWidget(const MyApp());
// Verify that platform version is retrieved.
expect(
find.byWidgetPredicate(
(Widget widget) =>
(widget) =>
widget is Text && widget.data!.startsWith('Running on:'),
),
findsOneWidget,

2
veilid-flutter/lib/routing_context.dart
View file

@ -301,7 +301,7 @@ abstract class VeilidRoutingContext {
{bool forceRefresh = false});
Future<ValueData?> setDHTValue(TypedKey key, int subkey, Uint8List data,
{KeyPair? writer});
Future<Timestamp> watchDHTValues(TypedKey key,
Future<bool> watchDHTValues(TypedKey key,
{List<ValueSubkeyRange>? subkeys, Timestamp? expiration, int? count});
Future<bool> cancelDHTWatch(TypedKey key, {List<ValueSubkeyRange>? subkeys});
Future<DHTRecordReport> inspectDHTRecord(TypedKey key,

36
veilid-flutter/lib/veilid_ffi.dart
View file

@ -294,10 +294,11 @@ Future<T> processFuturePlain<T>(Future<dynamic> future) async =>
'Unexpected async return message type: ${list[0]}');
}
}
// Any errors at all from Veilid need to be caught
// ignore: inference_failure_on_untyped_parameter
}).catchError((e) {
}).catchError((e, s) {
// Wrap all other errors in VeilidAPIExceptionInternal
throw VeilidAPIExceptionInternal(e.toString());
throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
}, test: (e) => e is! VeilidAPIException);
Future<T> processFutureJson<T>(
@ -332,10 +333,11 @@ Future<T> processFutureJson<T>(
'Unexpected async return message type: ${list[0]}');
}
}
// Any errors at all from Veilid need to be caught
// ignore: inference_failure_on_untyped_parameter
}).catchError((e) {
}).catchError((e, s) {
// Wrap all other errors in VeilidAPIExceptionInternal
throw VeilidAPIExceptionInternal(e.toString());
throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
}, test: (e) => e is! VeilidAPIException);
Future<T?> processFutureOptJson<T>(
@ -372,10 +374,11 @@ Future<T?> processFutureOptJson<T>(
'Unexpected async return message type: ${list[0]}');
}
}
// Any errors at all from Veilid need to be caught
// ignore: inference_failure_on_untyped_parameter
}).catchError((e) {
}).catchError((e, s) {
// Wrap all other errors in VeilidAPIExceptionInternal
throw VeilidAPIExceptionInternal(e.toString());
throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
}, test: (e) => e is! VeilidAPIException);
Future<void> processFutureVoid(Future<dynamic> future) async =>
@ -414,10 +417,11 @@ Future<void> processFutureVoid(Future<dynamic> future) async =>
'Unexpected async return message type: ${list[0]}');
}
}
// Any errors at all from Veilid need to be caught
// ignore: inference_failure_on_untyped_parameter
}).catchError((e) {
}).catchError((e, s) {
// Wrap all other errors in VeilidAPIExceptionInternal
throw VeilidAPIExceptionInternal(e.toString());
throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
}, test: (e) => e is! VeilidAPIException);
Future<Stream<T>> processFutureStream<T>(
@ -457,10 +461,11 @@ Future<Stream<T>> processFutureStream<T>(
'Unexpected async return message type: ${list[0]}');
}
}
// Any errors at all from Veilid need to be caught
// ignore: inference_failure_on_untyped_parameter
}).catchError((e) {
}).catchError((e, s) {
// Wrap all other errors in VeilidAPIExceptionInternal
throw VeilidAPIExceptionInternal(e.toString());
throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
}, test: (e) => e is! VeilidAPIException);
Stream<T> processStreamJson<T>(
@ -703,7 +708,7 @@ class VeilidRoutingContextFFI extends VeilidRoutingContext {
}
@override
Future<Timestamp> watchDHTValues(TypedKey key,
Future<bool> watchDHTValues(TypedKey key,
{List<ValueSubkeyRange>? subkeys,
Timestamp? expiration,
int? count}) async {
@ -720,9 +725,8 @@ class VeilidRoutingContextFFI extends VeilidRoutingContext {
final sendPort = recvPort.sendPort;
_ctx.ffi._routingContextWatchDHTValues(sendPort.nativePort, _ctx.id!,
nativeKey, nativeSubkeys, nativeExpiration, count);
final actualExpiration = Timestamp(
value: BigInt.from(await processFuturePlain<int>(recvPort.first)));
return actualExpiration;
final active = await processFuturePlain<bool>(recvPort.first);
return active;
}
@override
@ -738,8 +742,8 @@ class VeilidRoutingContextFFI extends VeilidRoutingContext {
final sendPort = recvPort.sendPort;
_ctx.ffi._routingContextCancelDHTWatch(
sendPort.nativePort, _ctx.id!, nativeKey, nativeSubkeys);
final cancelled = await processFuturePlain<bool>(recvPort.first);
return cancelled;
final active = await processFuturePlain<bool>(recvPort.first);
return active;
}
@override

10
veilid-flutter/lib/veilid_js.dart
View file

@ -24,14 +24,15 @@ dynamic convertUint8ListToJson(Uint8List data) => data.toList().jsify();
Future<T> _wrapApiPromise<T>(Object p) => js_util
.promiseToFuture<T>(p)
.then((value) => value)
// Any errors at all from Veilid need to be caught
// ignore: inference_failure_on_untyped_parameter
.catchError((e) {
.catchError((e, s) {
try {
final ex = VeilidAPIException.fromJson(jsonDecode(e as String));
throw ex;
} on Exception catch (_) {
// Wrap all other errors in VeilidAPIExceptionInternal
throw VeilidAPIExceptionInternal(e.toString());
throw VeilidAPIExceptionInternal('$e\nStack Trace:\n$s');
}
});
@ -206,7 +207,7 @@ class VeilidRoutingContextJS extends VeilidRoutingContext {
}
@override
Future<Timestamp> watchDHTValues(TypedKey key,
Future<bool> watchDHTValues(TypedKey key,
{List<ValueSubkeyRange>? subkeys,
Timestamp? expiration,
int? count}) async {
@ -215,7 +216,7 @@ class VeilidRoutingContextJS extends VeilidRoutingContext {
count ??= 0xFFFFFFFF;
final id = _ctx.requireId();
final ts = await _wrapApiPromise<String>(js_util.callMethod(
return _wrapApiPromise<bool>(js_util.callMethod(
wasm, 'routing_context_watch_dht_values', [
id,
jsonEncode(key),
@ -223,7 +224,6 @@ class VeilidRoutingContextJS extends VeilidRoutingContext {
expiration.toString(),
count
]));
return Timestamp.fromString(ts);
}
@override

8
veilid-flutter/packages/veilid_test/.flutter-plugins Normal file
View file

@ -0,0 +1,8 @@
# This is a generated file; do not edit or check into version control.
path_provider=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider-2.1.5/
path_provider_android=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_android-2.2.16/
path_provider_foundation=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.1/
path_provider_linux=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_linux-2.2.1/
path_provider_windows=/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_windows-2.3.0/
system_info_plus=/Users/dildog/.pub-cache/hosted/pub.dev/system_info_plus-0.0.6/
veilid=/Users/dildog/code/veilid/veilid-flutter/

1
veilid-flutter/packages/veilid_test/.flutter-plugins-dependencies Normal file
View file

@ -0,0 +1 @@
{"info":"This is a generated file; do not edit or check into version control.","plugins":{"ios":[{"name":"path_provider_foundation","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.1/","shared_darwin_source":true,"native_build":true,"dependencies":[],"dev_dependency":false},{"name":"system_info_plus","path":"/Users/dildog/.pub-cache/hosted/pub.dev/system_info_plus-0.0.6/","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":["system_info_plus"],"dev_dependency":false}],"android":[{"name":"path_provider_android","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_android-2.2.16/","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"system_info_plus","path":"/Users/dildog/.pub-cache/hosted/pub.dev/system_info_plus-0.0.6/","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":["system_info_plus"],"dev_dependency":false}],"macos":[{"name":"path_provider_foundation","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.1/","shared_darwin_source":true,"native_build":true,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":[],"dev_dependency":false}],"linux":[{"name":"path_provider_linux","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_linux-2.2.1/","native_build":false,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":[],"dev_dependency":false}],"windows":[{"name":"path_provider_windows","path":"/Users/dildog/.pub-cache/hosted/pub.dev/path_provider_windows-2.3.0/","native_build":false,"dependencies":[],"dev_dependency":false},{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","native_build":true,"dependencies":[],"dev_dependency":false}],"web":[{"name":"veilid","path":"/Users/dildog/code/veilid/veilid-flutter/","dependencies":[],"dev_dependency":false}]},"dependencyGraph":[{"name":"path_provider","dependencies":["path_provider_android","path_provider_foundation","path_provider_linux","path_provider_windows"]},{"name":"path_provider_android","dependencies":[]},{"name":"path_provider_foundation","dependencies":[]},{"name":"path_provider_linux","dependencies":[]},{"name":"path_provider_windows","dependencies":[]},{"name":"system_info_plus","dependencies":[]},{"name":"veilid","dependencies":["path_provider","system_info_plus"]}],"date_created":"2025-03-21 12:28:02.057123","version":"3.29.2","swift_package_manager_enabled":{"ios":false,"macos":false}}

1
veilid-flutter/packages/veilid_test/lib/src/veilid_fixture.dart
View file

@ -149,7 +149,6 @@ class DefaultVeilidFixture implements VeilidFixture {
case AttachmentState.overAttached:
case AttachmentState.fullyAttached:
done = true;
break;
}
}
if (done) {

8
veilid-flutter/rust/src/dart_ffi.rs
View file

@ -821,9 +821,9 @@ pub extern "C" fn routing_context_watch_dht_values(
let routing_context = get_routing_context(id, "routing_context_watch_dht_values")?;
let res = routing_context
.watch_dht_values(key, subkeys, expiration, count)
.watch_dht_values(key, Some(subkeys), Some(expiration), Some(count))
.await?;
APIResult::Ok(res.as_u64())
APIResult::Ok(res)
}
.in_current_span(),
);
@ -846,7 +846,7 @@ pub extern "C" fn routing_context_cancel_dht_watch(
async move {
let routing_context = get_routing_context(id, "routing_context_cancel_dht_watch")?;
let res = routing_context.cancel_dht_watch(key, subkeys).await?;
let res = routing_context.cancel_dht_watch(key, Some(subkeys)).await?;
APIResult::Ok(res)
}
.in_current_span(),
@ -874,7 +874,7 @@ pub extern "C" fn routing_context_inspect_dht_record(
let routing_context = get_routing_context(id, "routing_context_inspect_dht_record")?;
let res = routing_context
.inspect_dht_record(key, subkeys, scope)
.inspect_dht_record(key, Some(subkeys), scope)
.await?;
APIResult::Ok(res)
}

333
veilid-python/tests/test_dht.py
View file

@ -1,5 +1,4 @@
# Routing context veilid tests
from typing import Any, Awaitable, Callable, Optional
import pytest
import asyncio
@ -7,7 +6,8 @@ import time
import os
import veilid
from veilid import ValueSubkey
from veilid import ValueSubkey, Timestamp, SafetySelection
from veilid.types import VeilidJSONEncoder
##################################################################
BOGUS_KEY = veilid.TypedKey.from_value(
@ -86,8 +86,8 @@ async def test_set_get_dht_value(api_connection: veilid.VeilidAPI):
vd4 = await rc.get_dht_value(rec.key, ValueSubkey(1), False)
assert vd4 is None
print("vd2: {}", vd2.__dict__)
print("vd3: {}", vd3.__dict__)
#print("vd2: {}", vd2.__dict__)
#print("vd3: {}", vd3.__dict__)
assert vd2 == vd3
@ -245,8 +245,7 @@ async def test_open_writer_dht_value(api_connection: veilid.VeilidAPI):
await rc.delete_dht_record(key)
# @pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running")
@pytest.mark.skip(reason = "don't work yet")
@pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running")
@pytest.mark.asyncio
async def test_watch_dht_values():
@ -256,113 +255,257 @@ async def test_watch_dht_values():
if update.kind == veilid.VeilidUpdateKind.VALUE_CHANGE:
await value_change_queue.put(update)
async def null_update_callback(update: veilid.VeilidUpdate):
pass
try:
api = await veilid.api_connector(value_change_update_callback)
api0 = await veilid.api_connector(value_change_update_callback, 0)
except veilid.VeilidConnectionError:
pytest.skip("Unable to connect to veilid-server.")
pytest.skip("Unable to connect to veilid-server 0.")
# Make two routing contexts, one with and one without safety
# So we can pretend to be a different node and get the watch updates
# Normally they would not get sent if the set comes from the same target
# as the watch's target
# XXX: this logic doesn't work because our node still suppresses updates
# XXX: if the value hasn't changed in the local record store
rcWatch = await api.new_routing_context()
rcSet = await (await api.new_routing_context()).with_safety(veilid.SafetySelection.unsafe())
async with rcWatch, rcSet:
# Make a DHT record
rec = await rcWatch.create_dht_record(veilid.DHTSchema.dflt(10))
# Set some subkey we care about
vd = await rcWatch.set_dht_value(rec.key, ValueSubkey(3), b"BLAH BLAH BLAH")
assert vd is None
# Make a watch on that subkey
ts = await rcWatch.watch_dht_values(rec.key, [], 0, 0xFFFFFFFF)
assert ts != 0
# Reopen without closing to change routing context and not lose watch
rec = await rcSet.open_dht_record(rec.key, rec.owner_key_pair())
# Now set the subkey and trigger an update
vd = await rcSet.set_dht_value(rec.key, ValueSubkey(3), b"BLAH")
assert vd is None
# Now we should NOT get an update because the update is the same as our local copy
update = None
try:
update = await asyncio.wait_for(value_change_queue.get(), timeout=5)
api1 = await veilid.api_connector(null_update_callback, 1)
except veilid.VeilidConnectionError:
pytest.skip("Unable to connect to veilid-server 1.")
async with api0, api1:
# purge local and remote record stores to ensure we start fresh
await api0.debug("record purge local")
await api0.debug("record purge remote")
await api1.debug("record purge local")
await api1.debug("record purge remote")
# Clear the change queue if record purge cancels old watches
while True:
try:
upd = await asyncio.wait_for(value_change_queue.get(), timeout=3)
except asyncio.TimeoutError:
break
# make routing contexts
rc0 = await api0.new_routing_context()
rc1 = await api1.new_routing_context()
async with rc0, rc1:
# Server 0: Make a DHT record
rec0 = await rc0.create_dht_record(veilid.DHTSchema.dflt(10))
# Server 0: Set some subkey we care about
vd = await rc0.set_dht_value(rec0.key, ValueSubkey(3), b"BLAH")
assert vd is None
await sync(rc0, [rec0])
# Server 0: Make a watch on all the subkeys
active = await rc0.watch_dht_values(rec0.key, [], Timestamp(0), 0xFFFFFFFF)
assert active
# Server 1: Open the subkey
rec1 = await rc1.open_dht_record(rec0.key, rec0.owner_key_pair())
# Server 1: Now set the subkey and trigger an update
vd = await rc1.set_dht_value(rec1.key, ValueSubkey(3), b"BLAH")
assert vd is None
await sync(rc1, [rec1])
# Server 0: Now we should NOT get an update because the update is the same as our local copy
upd = None
try:
upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
except asyncio.TimeoutError:
pass
assert update is None
assert upd is None
# Now set multiple subkeys and trigger an update
vd = await asyncio.gather(*[rcSet.set_dht_value(rec.key, ValueSubkey(3), b"BLAH BLAH"), rcSet.set_dht_value(rec.key, ValueSubkey(4), b"BZORT")])
assert vd == [None, None]
# Server 1: Now set subkey and trigger an update
vd = await rc1.set_dht_value(rec1.key, ValueSubkey(3), b"BLAH BLAH")
assert vd is None
await sync(rc1, [rec1])
# Wait for the update
upd = await asyncio.wait_for(value_change_queue.get(), timeout=5)
# Verify the update came back but we don't get a new value because the sequence number is the same
assert upd.detail.key == rec.key
assert upd.detail.count == 0xFFFFFFFD
assert upd.detail.subkeys == [(3, 4)]
assert upd.detail.value is None
# Reopen without closing to change routing context and not lose watch
rec = await rcWatch.open_dht_record(rec.key, rec.owner_key_pair())
# Cancel some subkeys we don't care about
still_active = await rcWatch.cancel_dht_watch(rec.key, [(ValueSubkey(0), ValueSubkey(2))])
assert still_active
# Reopen without closing to change routing context and not lose watch
rec = await rcSet.open_dht_record(rec.key, rec.owner_key_pair())
# Now set multiple subkeys and trigger an update
vd = await asyncio.gather(*[rcSet.set_dht_value(rec.key, ValueSubkey(3), b"BLAH BLAH BLAH"), rcSet.set_dht_value(rec.key, ValueSubkey(5), b"BZORT BZORT")])
assert vd == [None, None]
# Wait for the update, this longer timeout seems to help the flaky check below
# Server 0: Wait for the update
upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
# Verify the update came back but we don't get a new value because the sequence number is the same
assert upd.detail.key == rec.key
# Server 0: Verify the update came back with the first changed subkey's data
assert upd.detail.key == rec0.key
assert upd.detail.count == 0xFFFFFFFE
assert upd.detail.subkeys == [(3, 3)]
assert upd.detail.value.data == b"BLAH BLAH"
# This check is flaky on slow connections and often fails with different counts
assert upd.detail.count == 0xFFFFFFFC
assert upd.detail.subkeys == [(3, 3), (5, 5)]
assert upd.detail.value is None
# Server 1: Now set subkey and trigger an update
vd = await rc1.set_dht_value(rec1.key, ValueSubkey(4), b"BZORT")
assert vd is None
await sync(rc1, [rec1])
# Reopen without closing to change routing context and not lose watch
rec = await rcWatch.open_dht_record(rec.key, rec.owner_key_pair())
# Server 0: Wait for the update
upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
# Now cancel the update
still_active = await rcWatch.cancel_dht_watch(rec.key, [(ValueSubkey(3), ValueSubkey(9))])
assert not still_active
# Server 0: Verify the update came back with the first changed subkey's data
assert upd.detail.key == rec0.key
assert upd.detail.count == 0xFFFFFFFD
assert upd.detail.subkeys == [(4, 4)]
assert upd.detail.value.data == b"BZORT"
# Reopen without closing to change routing context and not lose watch
rec = await rcSet.open_dht_record(rec.key, rec.owner_key_pair())
# Server 0: Cancel some subkeys we don't care about
active = await rc0.cancel_dht_watch(rec0.key, [(ValueSubkey(0), ValueSubkey(3))])
assert active
# Now set multiple subkeys
vd = await asyncio.gather(*[rcSet.set_dht_value(rec.key, ValueSubkey(3), b"BLAH BLAH BLAH BLAH"), rcSet.set_dht_value(rec.key, ValueSubkey(5), b"BZORT BZORT BZORT")])
# Server 1: Now set multiple subkeys and trigger an update
vd = await asyncio.gather(*[rc1.set_dht_value(rec1.key, ValueSubkey(3), b"BLAH BLAH BLAH"), rc1.set_dht_value(rec1.key, ValueSubkey(4), b"BZORT BZORT")])
assert vd == [None, None]
await sync(rc1, [rec1])
# Now we should NOT get an update
update = None
# Server 0: Wait for the update
upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
# Server 0: Verify only one update came back
assert upd.detail.key == rec0.key
assert upd.detail.count == 0xFFFFFFFC
assert upd.detail.subkeys == [(4, 4)]
assert upd.detail.value.data == b"BZORT BZORT"
# Server 0: Now we should NOT get any other update
upd = None
try:
update = await asyncio.wait_for(value_change_queue.get(), timeout=5)
upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
except asyncio.TimeoutError:
pass
assert update is None
if upd is not None:
print(f"bad update: {VeilidJSONEncoder.dumps(upd)}")
assert upd is None
# Now cancel the update
active = await rc0.cancel_dht_watch(rec0.key, [(ValueSubkey(3), ValueSubkey(9))])
assert not active
# Server 0: Wait for the cancellation update
upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
# Server 0: Verify only one update came back
assert upd.detail.key == rec0.key
assert upd.detail.count == 0
assert upd.detail.subkeys == []
assert upd.detail.value is None
# Now set multiple subkeys
vd = await asyncio.gather(*[rc1.set_dht_value(rec1.key, ValueSubkey(3), b"BLAH BLAH BLAH BLAH"), rc1.set_dht_value(rec1.key, ValueSubkey(5), b"BZORT BZORT BZORT")])
assert vd == [None, None]
await sync(rc1, [rec1])
# Now we should NOT get an update
upd = None
try:
upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
except asyncio.TimeoutError:
pass
if upd is not None:
print(f"bad update: {VeilidJSONEncoder.dumps(upd)}")
assert upd is None
# Clean up
await rcSet.close_dht_record(rec.key)
await rcSet.delete_dht_record(rec.key)
await rc1.close_dht_record(rec1.key)
await rc1.delete_dht_record(rec1.key)
await rc0.close_dht_record(rec0.key)
await rc0.delete_dht_record(rec0.key)
@pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running")
@pytest.mark.skipif(os.getenv("STRESS") != "1", reason="stress test takes a long time")
@pytest.mark.asyncio
async def test_watch_many_dht_values():
value_change_queue: asyncio.Queue[veilid.VeilidUpdate] = asyncio.Queue()
async def value_change_update_callback(update: veilid.VeilidUpdate):
if update.kind == veilid.VeilidUpdateKind.VALUE_CHANGE:
await value_change_queue.put(update)
async def null_update_callback(update: veilid.VeilidUpdate):
pass
try:
api0 = await veilid.api_connector(value_change_update_callback, 0)
except veilid.VeilidConnectionError:
pytest.skip("Unable to connect to veilid-server 0.")
try:
api1 = await veilid.api_connector(null_update_callback, 1)
except veilid.VeilidConnectionError:
pytest.skip("Unable to connect to veilid-server 1.")
async with api0, api1:
# purge local and remote record stores to ensure we start fresh
await api0.debug("record purge local")
await api0.debug("record purge remote")
await api1.debug("record purge local")
await api1.debug("record purge remote")
# make routing contexts
# unsafe version for debugging
rc0 = await (await api0.new_routing_context()).with_safety(SafetySelection.unsafe())
rc1 = await (await api1.new_routing_context()).with_safety(SafetySelection.unsafe())
# safe default version
# rc0 = await api0.new_routing_context()
# rc1 = await api1.new_routing_context()
async with rc0, rc1:
COUNT = 10
records = []
# Make and watch all records
for n in range(COUNT):
print(f"making record {n}")
# Server 0: Make a DHT record
records.append(await rc0.create_dht_record(veilid.DHTSchema.dflt(1)))
# Server 0: Set some subkey we care about
vd = await rc0.set_dht_value(records[n].key, ValueSubkey(0), b"BLAH")
assert vd is None
# Server 0: Make a watch on all the subkeys
active = await rc0.watch_dht_values(records[n].key, [], Timestamp(0), 0xFFFFFFFF)
assert active
# Open and set all records
missing_records = set()
for (n, record) in enumerate(records):
print(f"setting record {n}")
# Server 1: Open the subkey
_ignore = await rc1.open_dht_record(record.key, record.owner_key_pair())
# Server 1: Now set the subkey and trigger an update
vd = await rc1.set_dht_value(record.key, ValueSubkey(0), b"BLAH BLAH")
assert vd is None
missing_records.add(record.key)
# Server 0: Now we should get an update for every change
for n in range(len(records)):
print(f"waiting for change {n}")
# Server 0: Wait for the update
try:
upd = await asyncio.wait_for(value_change_queue.get(), timeout=10)
missing_records.remove(upd.detail.key)
except:
# Dump which records didn't get updates
for (m, record) in enumerate(records):
if record.key not in missing_records:
continue
print(f"missing update for record {m}: {record}")
info0 = await api0.debug(f"record info {record.key}")
info1 = await api1.debug(f"record info {record.key}")
print(f"from rc0: {info0}")
print(f"from rc1: {info1}")
raise
# Clean up
for record in records:
await rc1.close_dht_record(record.key)
await rc1.delete_dht_record(record.key)
await rc0.close_dht_record(record.key)
await rc0.delete_dht_record(record.key)
@pytest.mark.asyncio
async def test_inspect_dht_record(api_connection: veilid.VeilidAPI):
rc = await api_connection.new_routing_context()
@ -486,8 +629,6 @@ async def test_schema_limit_smpl(api_connection: veilid.VeilidAPI):
@pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running")
@pytest.mark.asyncio
async def test_dht_integration_writer_reader():
@ -702,21 +843,23 @@ async def test_dht_write_read_full_subkeys_local():
async def sync(rc: veilid.RoutingContext, records: list[veilid.DHTRecordDescriptor]):
print('syncing records to the network')
syncrecords = records.copy()
while len(syncrecords) > 0:
if len(syncrecords) == 0:
return
while True:
donerecords = set()
subkeysleft = 0
for desc in records:
rr = await rc.inspect_dht_record(desc.key, [])
left = 0; [left := left + (x[1]-x[0]+1) for x in rr.offline_subkeys]
if left == 0:
if veilid.ValueSeqNum.NONE not in rr.local_seqs:
donerecords.add(desc)
else:
subkeysleft += left
syncrecords = [x for x in syncrecords if x not in donerecords]
print(f' {len(syncrecords)} records {subkeysleft} subkeys left')
if len(syncrecords) == 0:
break
print(f' syncing {len(syncrecords)} records {subkeysleft} subkeys left')
time.sleep(1)

8
veilid-python/veilid/api.py
View file

@ -92,17 +92,17 @@ class RoutingContext(ABC):
async def watch_dht_values(
self,
key: types.TypedKey,
subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]],
expiration: types.Timestamp = 0,
subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]] = [],
expiration: types.Timestamp = types.Timestamp(0),
count: int = 0xFFFFFFFF,
) -> types.Timestamp:
) -> bool:
pass
@abstractmethod
async def cancel_dht_watch(
self,
key: types.TypedKey,
subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]],
subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]] = [],
) -> bool:
pass

13
veilid-python/veilid/json_api.py
View file

@ -740,10 +740,10 @@ class _JsonRoutingContext(RoutingContext):
async def watch_dht_values(
self,
key: TypedKey,
subkeys: list[tuple[ValueSubkey, ValueSubkey]],
expiration: Timestamp = 0,
subkeys: list[tuple[ValueSubkey, ValueSubkey]] = [],
expiration: Timestamp = Timestamp(0),
count: int = 0xFFFFFFFF,
) -> Timestamp:
) -> bool:
assert isinstance(key, TypedKey)
assert isinstance(subkeys, list)
for s in subkeys:
@ -753,8 +753,7 @@ class _JsonRoutingContext(RoutingContext):
assert isinstance(expiration, Timestamp)
assert isinstance(count, int)
return Timestamp(
raise_api_result(
return raise_api_result(
await self.api.send_ndjson_request(
Operation.ROUTING_CONTEXT,
validate=validate_rc_op,
@ -766,10 +765,10 @@ class _JsonRoutingContext(RoutingContext):
count=count,
)
)
)
async def cancel_dht_watch(
self, key: TypedKey, subkeys: list[tuple[ValueSubkey, ValueSubkey]]
self, key: TypedKey, subkeys: list[tuple[ValueSubkey, ValueSubkey]] = []
) -> bool:
assert isinstance(key, TypedKey)
assert isinstance(subkeys, list)

2
veilid-python/veilid/schema/RecvMessage.json
View file

@ -858,7 +858,7 @@
],
"properties": {
"value": {
"type": "string"
"type": "boolean"
}
}
},

42
veilid-python/veilid/schema/Request.json
View file

@ -469,20 +469,23 @@
{
"type": "object",
"required": [
"count",
"expiration",
"key",
"rc_op",
"subkeys"
"rc_op"
],
"properties": {
"count": {
"type": "integer",
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"expiration": {
"type": "string"
"type": [
"string",
"null"
]
},
"key": {
"type": "string"
@ -494,7 +497,10 @@
]
},
"subkeys": {
"type": "array",
"type": [
"array",
"null"
],
"items": {
"type": "array",
"items": [
@ -519,8 +525,7 @@
"type": "object",
"required": [
"key",
"rc_op",
"subkeys"
"rc_op"
],
"properties": {
"key": {
@ -533,7 +538,10 @@
]
},
"subkeys": {
"type": "array",
"type": [
"array",
"null"
],
"items": {
"type": "array",
"items": [
@ -558,9 +566,7 @@
"type": "object",
"required": [
"key",
"rc_op",
"scope",
"subkeys"
"rc_op"
],
"properties": {
"key": {
@ -573,10 +579,18 @@
]
},
"scope": {
"default": "Local",
"allOf": [
{
"$ref": "#/definitions/DHTReportScope"
}
]
},
"subkeys": {
"type": "array",
"type": [
"array",
"null"
],
"items": {
"type": "array",
"items": [

44
veilid-python/veilid/state.py
View file

@ -59,6 +59,8 @@ class VeilidStateAttachment:
j["attached_uptime"],
)
def to_json(self) -> dict:
return self.__dict__
class AnswerStats:
@ -114,6 +116,9 @@ class AnswerStats:
j["consecutive_lost_answers_minimum"],
)
def to_json(self) -> dict:
return self.__dict__
class RPCStats:
messages_sent: int
messages_rcvd: int
@ -172,6 +177,9 @@ class RPCStats:
AnswerStats.from_json(j["answer_ordered"]),
)
def to_json(self) -> dict:
return self.__dict__
class LatencyStats:
fastest: TimestampDuration
@ -213,6 +221,9 @@ class LatencyStats:
TimestampDuration(j["p75"]),
)
def to_json(self) -> dict:
return self.__dict__
class TransferStats:
total: ByteCount
@ -365,6 +376,9 @@ class PeerStats:
StateStats.from_json(j["state"]),
)
def to_json(self) -> dict:
return self.__dict__
class PeerTableData:
node_ids: list[str]
@ -381,6 +395,9 @@ class PeerTableData:
"""JSON object hook"""
return cls(j["node_ids"], j["peer_address"], PeerStats.from_json(j["peer_stats"]))
def to_json(self) -> dict:
return self.__dict__
class VeilidStateNetwork:
started: bool
@ -410,6 +427,9 @@ class VeilidStateNetwork:
[PeerTableData.from_json(peer) for peer in j["peers"]],
)
def to_json(self) -> dict:
return self.__dict__
class VeilidStateConfig:
config: VeilidConfig
@ -422,6 +442,9 @@ class VeilidStateConfig:
"""JSON object hook"""
return cls(VeilidConfig.from_json(j["config"]))
def to_json(self) -> dict:
return self.__dict__
class VeilidState:
attachment: VeilidStateAttachment
@ -447,6 +470,9 @@ class VeilidState:
VeilidStateConfig.from_json(j["config"]),
)
def to_json(self) -> dict:
return self.__dict__
class VeilidLog:
log_level: VeilidLogLevel
@ -463,6 +489,9 @@ class VeilidLog:
"""JSON object hook"""
return cls(VeilidLogLevel(j["log_level"]), j["message"], j["backtrace"])
def to_json(self) -> dict:
return self.__dict__
class VeilidAppMessage:
sender: Optional[TypedKey]
@ -483,6 +512,9 @@ class VeilidAppMessage:
urlsafe_b64decode_no_pad(j["message"]),
)
def to_json(self) -> dict:
return self.__dict__
class VeilidAppCall:
sender: Optional[TypedKey]
@ -506,6 +538,9 @@ class VeilidAppCall:
OperationId(j["call_id"]),
)
def to_json(self) -> dict:
return self.__dict__
class VeilidRouteChange:
dead_routes: list[RouteId]
@ -523,6 +558,9 @@ class VeilidRouteChange:
[RouteId(route) for route in j["dead_remote_routes"]],
)
def to_json(self) -> dict:
return self.__dict__
class VeilidValueChange:
key: TypedKey
@ -546,6 +584,9 @@ class VeilidValueChange:
None if j["value"] is None else ValueData.from_json(j["value"]),
)
def to_json(self) -> dict:
return self.__dict__
class VeilidUpdateKind(StrEnum):
LOG = "Log"
@ -610,3 +651,6 @@ class VeilidUpdate:
case _:
raise ValueError("Unknown VeilidUpdateKind")
return cls(kind, detail)
def to_json(self) -> dict:
return self.__dict__

22
veilid-server/src/client_api.rs
View file

@ -395,7 +395,7 @@ impl ClientApi {
// Request receive processor future
// Receives from socket and enqueues RequestLines
// Completes when the connection is closed or there is a failure
unord.push(system_boxed(self.clone().receive_requests(
unord.push(pin_dyn_future!(self.clone().receive_requests(
reader,
requests_tx,
responses_tx,
@ -404,12 +404,14 @@ impl ClientApi {
// Response send processor
// Sends finished response strings out the socket
// Completes when the responses channel is closed
unord.push(system_boxed(
self.clone().send_responses(responses_rx, writer),
));
unord.push(pin_dyn_future!(self
.clone()
.send_responses(responses_rx, writer)));
// Add future to process first request
unord.push(system_boxed(Self::next_request_line(requests_rx.clone())));
unord.push(pin_dyn_future!(Self::next_request_line(
requests_rx.clone()
)));
// Send and receive until we're done or a stop is requested
while let Ok(Some(r)) = unord.next().timeout_at(stop_token.clone()).await {
@ -417,7 +419,9 @@ impl ClientApi {
let request_line = match r {
Ok(Some(request_line)) => {
// Add future to process next request
unord.push(system_boxed(Self::next_request_line(requests_rx.clone())));
unord.push(pin_dyn_future!(Self::next_request_line(
requests_rx.clone()
)));
// Socket receive future returned something to process
request_line
@ -434,9 +438,9 @@ impl ClientApi {
};
// Enqueue unordered future to process request line in parallel
unord.push(system_boxed(
self.clone().process_request_line(jrp.clone(), request_line),
));
unord.push(pin_dyn_future!(self
.clone()
.process_request_line(jrp.clone(), request_line)));
}
// Stop sending updates

36
veilid-tools/src/async_tag_lock.rs
View file

@ -3,8 +3,26 @@ use super::*;
use core::fmt::Debug;
use core::hash::Hash;
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct AsyncTagLockGuard<T>
where
T: Hash + Eq + Clone + Debug,
{
inner: Arc<AsyncTagLockGuardInner<T>>,
}
impl<T> AsyncTagLockGuard<T>
where
T: Hash + Eq + Clone + Debug,
{
#[must_use]
pub fn tag(&self) -> T {
self.inner.tag()
}
}
#[derive(Debug)]
struct AsyncTagLockGuardInner<T>
where
T: Hash + Eq + Clone + Debug,
{
@ -13,7 +31,7 @@ where
guard: Option<AsyncMutexGuardArc<()>>,
}
impl<T> AsyncTagLockGuard<T>
impl<T> AsyncTagLockGuardInner<T>
where
T: Hash + Eq + Clone + Debug,
{
@ -24,9 +42,13 @@ where
guard: Some(guard),
}
}
fn tag(&self) -> T {
self.tag.clone()
}
}
impl<T> Drop for AsyncTagLockGuard<T>
impl<T> Drop for AsyncTagLockGuardInner<T>
where
T: Hash + Eq + Clone + Debug,
{
@ -133,7 +155,9 @@ where
let guard = asyncmutex_lock_arc!(mutex);
// Return the locked guard
AsyncTagLockGuard::new(self.clone(), tag, guard)
AsyncTagLockGuard {
inner: Arc::new(AsyncTagLockGuardInner::new(self.clone(), tag, guard)),
}
}
pub fn try_lock_tag(&self, tag: T) -> Option<AsyncTagLockGuard<T>> {
@ -160,7 +184,9 @@ where
}
};
// Return guard
Some(AsyncTagLockGuard::new(self.clone(), tag, guard))
Some(AsyncTagLockGuard {
inner: Arc::new(AsyncTagLockGuardInner::new(self.clone(), tag, guard)),
})
}
}

23
veilid-tools/src/deferred_stream_processor.rs
View file

@ -113,7 +113,10 @@ impl DeferredStreamProcessor {
/// * 'handler' is the callback to handle each item from the stream
///
/// Returns 'true' if the stream was added for processing, and 'false' if the stream could not be added, possibly due to not being initialized.
pub fn add<T: Send + 'static, S: futures_util::Stream<Item = T> + Unpin + Send + 'static>(
pub fn add_stream<
T: Send + 'static,
S: futures_util::Stream<Item = T> + Unpin + Send + 'static,
>(
&self,
mut receiver: S,
mut handler: impl FnMut(T) -> PinBoxFutureStatic<bool> + Send + 'static,
@ -140,6 +143,24 @@ impl DeferredStreamProcessor {
}
true
}
/// Queue a single future to process in the background
pub fn add_future<F>(&self, fut: F) -> bool
where
F: Future<Output = ()> + Send + 'static,
{
let dsc_tx = {
let inner = self.inner.lock();
let Some(dsc_tx) = inner.opt_deferred_stream_channel.clone() else {
return false;
};
dsc_tx
};
if dsc_tx.send(Box::pin(fut)).is_err() {
return false;
}
true
}
}
impl Default for DeferredStreamProcessor {

8
veilid-tools/src/tools.rs
View file

@ -119,14 +119,6 @@ macro_rules! asyncrwlock_try_write_arc {
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
pub fn system_boxed<'a, Out>(
future: impl Future<Output = Out> + Send + 'a,
) -> PinBoxFuture<'a, Out> {
Box::pin(future)
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
cfg_if! {
if #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] {
#[must_use]

6
veilid-tools/src/virtual_network/router_client.rs
View file

@ -589,7 +589,7 @@ impl RouterClient {
let framed_reader = FramedRead::new(reader, BytesCodec);
let framed_writer = FramedWrite::new(writer, BytesCodec);
let framed_writer_fut = system_boxed(async move {
let framed_writer_fut = Box::pin(async move {
if let Err(e) = receiver
.into_stream()
.map(|command| {
@ -603,7 +603,7 @@ impl RouterClient {
error!("{}", e);
}
});
let framed_reader_fut = system_boxed(async move {
let framed_reader_fut = Box::pin(async move {
let fut = framed_reader.try_for_each(|x| async {
let x = x;
let evt = from_bytes::<ServerProcessorEvent>(&x)
@ -631,7 +631,7 @@ impl RouterClient {
.into_stream()
.map(io::Result::<ServerProcessorEvent>::Ok);
let receiver_fut = system_boxed(async move {
let receiver_fut = Box::pin(async move {
let fut =
receiver.try_for_each(|evt| Self::process_event(evt, router_op_waiter.clone()));
if let Err(e) = fut.await {

14
veilid-tools/src/virtual_network/router_server/mod.rs
View file

@ -117,7 +117,7 @@ impl RouterServer {
let stop_token = stop_source.token();
let this = self.clone();
let listener_fut = system_boxed(async move {
let listener_fut = Box::pin(async move {
loop {
// Wait for a new connection
match listener.accept().timeout_at(stop_token.clone()).await {
@ -125,7 +125,7 @@ impl RouterServer {
let conn = conn.compat();
// Register a connection processing inbound receiver
let this2 = this.clone();
let inbound_receiver_fut = system_boxed(async move {
let inbound_receiver_fut = Box::pin(async move {
let (reader, writer) = conn.split();
this2.process_connection(reader, writer).await
@ -178,7 +178,7 @@ impl RouterServer {
let stop_token = stop_source.token();
let this = self.clone();
let listener_fut = system_boxed(async move {
let listener_fut = Box::pin(async move {
loop {
// Wait for a new connection
match listener.accept().timeout_at(stop_token.clone()).await {
@ -188,7 +188,7 @@ impl RouterServer {
let ws = WsStream::new(s);
// Register a connection processing inbound receiver
let this2 = this.clone();
let inbound_receiver_fut = system_boxed(async move {
let inbound_receiver_fut = Box::pin(async move {
let (reader, writer) = ws.split();
this2.process_connection(reader, writer).await
});
@ -233,7 +233,7 @@ impl RouterServer {
let (local_outbound_sender, local_outbound_receiver) = flume::unbounded();
let this = self.clone();
let inbound_receiver_fut = system_boxed(async move {
let inbound_receiver_fut = Box::pin(async move {
local_inbound_receiver
.into_stream()
.for_each(|cmd| async {
@ -316,7 +316,7 @@ impl RouterServer {
let framed_writer = FramedWrite::new(writer, BytesCodec);
let (outbound_sender, outbound_receiver) = flume::unbounded();
let outbound_fut = system_boxed(
let outbound_fut = Box::pin(
outbound_receiver
.into_stream()
.map(|command| {
@ -327,7 +327,7 @@ impl RouterServer {
.forward(framed_writer),
);
let inbound_fut = system_boxed(framed_reader.try_for_each(|x| async {
let inbound_fut = Box::pin(framed_reader.try_for_each(|x| async {
let x = x;
let cmd = from_bytes::<ServerProcessorCommand>(&x).map_err(io::Error::other)?;

8
veilid-wasm/src/lib.rs
View file

@ -626,9 +626,9 @@ pub fn routing_context_watch_dht_values(
let routing_context = get_routing_context(id, "routing_context_watch_dht_values")?;
let res = routing_context
.watch_dht_values(key, subkeys, expiration, count)
.watch_dht_values(key, Some(subkeys), Some(expiration), Some(count))
.await?;
APIResult::Ok(res.as_u64().to_string())
APIResult::Ok(res)
})
}
@ -642,7 +642,7 @@ pub fn routing_context_cancel_dht_watch(id: u32, key: String, subkeys: String) -
let routing_context = get_routing_context(id, "routing_context_cancel_dht_watch")?;
let res = routing_context.cancel_dht_watch(key, subkeys).await?;
let res = routing_context.cancel_dht_watch(key, Some(subkeys)).await?;
APIResult::Ok(res)
})
}
@ -665,7 +665,7 @@ pub fn routing_context_inspect_dht_record(
let routing_context = get_routing_context(id, "routing_context_inspect_dht_record")?;
let res = routing_context
.inspect_dht_record(key, subkeys, scope)
.inspect_dht_record(key, Some(subkeys), scope)
.await?;
APIResult::Ok(res)

46
veilid-wasm/src/veilid_routing_context_js.rs
View file

@ -344,17 +344,22 @@ impl VeilidRoutingContext {
///
/// There is only one watch permitted per record. If a change to a watch is desired, the previous one will be overwritten.
/// * `key` is the record key to watch. it must first be opened for reading or writing.
/// * `subkeys` is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
/// * `expiration` is the desired timestamp of when to automatically terminate the watch, in microseconds. If this value is less than `network.rpc.timeout_ms` milliseconds in the future, this function will return an error immediately.
/// * `count` is the number of times the watch will be sent, maximum. A zero value here is equivalent to a cancellation.
/// * `subkeys`:
/// - None: specifies watching the entire range of subkeys.
/// - Some(range): is the the range of subkeys to watch. The range must not exceed 512 discrete non-overlapping or adjacent subranges. If no range is specified, this is equivalent to watching the entire range of subkeys.
/// * `expiration`:
/// - None: specifies a watch with no expiration
/// - Some(timestamp): the desired timestamp of when to automatically terminate the watch, in microseconds. If this value is less than `network.rpc.timeout_ms` milliseconds in the future, this function will return an error immediately.
/// * `count:
/// - None: specifies a watch count of u32::MAX
/// - Some(count): is the number of times the watch will be sent, maximum. A zero value here is equivalent to a cancellation.
///
/// Returns a timestamp of when the watch will expire. All watches are guaranteed to expire at some point in the future,
/// and the returned timestamp will be no later than the requested expiration, but -may- be before the requested expiration.
/// If the returned timestamp is zero it indicates that the watch creation or update has failed. In the case of a faild update, the watch is considered cancelled.
/// Returns Ok(true) if a watch is active for this record.
/// Returns Ok(false) if the entire watch has been cancelled.
///
/// DHT watches are accepted with the following conditions:
/// * First-come first-served basis for arbitrary unauthenticated readers, up to network.dht.public_watch_limit per record
/// * If a member (either the owner or a SMPL schema member) has opened the key for writing (even if no writing is performed) then the watch will be signed and guaranteed network.dht.member_watch_limit per writer
/// * First-come first-served basis for arbitrary unauthenticated readers, up to network.dht.public_watch_limit per record.
/// * If a member (either the owner or a SMPL schema member) has opened the key for writing (even if no writing is performed) then the watch will be signed and guaranteed network.dht.member_watch_limit per writer.
///
/// Members can be specified via the SMPL schema and do not need to allocate writable subkeys in order to offer a member watch capability.
pub async fn watchDhtValues(
@ -363,42 +368,38 @@ impl VeilidRoutingContext {
subkeys: Option<ValueSubkeyRangeSet>,
expiration: Option<String>,
count: Option<u32>,
) -> APIResult<String> {
) -> APIResult<bool> {
let key = TypedKey::from_str(&key)?;
let subkeys = subkeys.unwrap_or_default();
let expiration = if let Some(expiration) = expiration {
veilid_core::Timestamp::new(
Some(veilid_core::Timestamp::new(
u64::from_str(&expiration).map_err(VeilidAPIError::generic)?,
)
))
} else {
veilid_core::Timestamp::default()
None
};
let count = count.unwrap_or(u32::MAX);
let routing_context = self.getRoutingContext()?;
let res = routing_context
.watch_dht_values(key, subkeys, expiration, count)
.await?;
APIResult::Ok(res.as_u64().to_string())
APIResult::Ok(res)
}
/// Cancels a watch early
/// Cancels a watch early.
///
/// This is a convenience function that cancels watching all subkeys in a range. The subkeys specified here
/// are subtracted from the watched subkey range. If no range is specified, this is equivalent to cancelling the entire range of subkeys.
/// are subtracted from the currently-watched subkey range.
/// If no range is specified, this is equivalent to cancelling the entire range of subkeys.
/// Only the subkey range is changed, the expiration and count remain the same.
/// If no subkeys remain, the watch is entirely cancelled and will receive no more updates.
///
/// Returns true if there is any remaining watch for this record
/// Returns false if the entire watch has been cancelled
/// Returns Ok(true) if a watch is active for this record.
/// Returns Ok(false) if the entire watch has been cancelled.
pub async fn cancelDhtWatch(
&self,
key: String,
subkeys: Option<ValueSubkeyRangeSet>,
) -> APIResult<bool> {
let key = TypedKey::from_str(&key)?;
let subkeys = subkeys.unwrap_or_default();
let routing_context = self.getRoutingContext()?;
let res = routing_context.cancel_dht_watch(key, subkeys).await?;
APIResult::Ok(res)
@ -450,7 +451,6 @@ impl VeilidRoutingContext {
scope: Option<DHTReportScope>,
) -> APIResult<DHTRecordReport> {
let key = TypedKey::from_str(&key)?;
let subkeys = subkeys.unwrap_or_default();
let scope = scope.unwrap_or_default();
let routing_context = self.getRoutingContext()?;

2
veilid-wasm/tests/package-lock.json generated
View file

@ -21,7 +21,7 @@
},
"../pkg": {
"name": "veilid-wasm",
"version": "0.4.3",
"version": "0.4.4",
"dev": true,
"license": "MPL-2.0"
},

8
veilid-wasm/tests/src/VeilidRoutingContext.test.ts
View file

@ -236,9 +236,7 @@ describe('VeilidRoutingContext', () => {
"0",
0xFFFFFFFF,
);
expect(watchValueRes).toBeDefined();
expect(watchValueRes).not.toEqual("");
expect(watchValueRes).not.toEqual("0");
expect(watchValueRes).toEqual(true);
const cancelValueRes = await routingContext.cancelDhtWatch(
dhtRecord.key,
@ -261,9 +259,7 @@ describe('VeilidRoutingContext', () => {
const watchValueRes = await routingContext.watchDhtValues(
dhtRecord.key,
);
expect(watchValueRes).toBeDefined();
expect(watchValueRes).not.toEqual("");
expect(watchValueRes).not.toEqual("0");
expect(watchValueRes).toEqual(true);
const cancelValueRes = await routingContext.cancelDhtWatch(
dhtRecord.key,

1887656
veilid-wasm/tests/test_results/profile-dht-stress-test.json Normal file
View file

File diff suppressed because one or more lines are too long