diff --git a/CHANGELOG.md b/CHANGELOG.md index 81480736..fa845266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -**Changed in Veilid 0.3.3** +**Changed in Veilid 0.3.4** - Crates updates - Update crates to newer versions - Remove veilid-async-tungstenite and veilid-async-tls crates as they are no longer needed diff --git a/Cargo.lock b/Cargo.lock index 8f7a6569..e2ba3560 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5886,6 +5886,7 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", + "time", "tracing", "tracing-core", "tracing-log 0.2.0", @@ -6328,6 +6329,7 @@ dependencies = [ "async-tungstenite 0.27.0", "backtrace", "cfg-if 1.0.0", + "chrono", "clap 4.5.15", "color-eyre", "config 0.14.0", @@ -6355,6 +6357,7 @@ dependencies = [ "signal-hook-async-std", "stop-token", "sysinfo", + "time", "tokio", "tokio-stream", "tokio-util", diff --git a/Earthfile b/Earthfile index db4ef40e..31ca7651 100644 --- a/Earthfile +++ b/Earthfile @@ -15,10 +15,11 @@ VERSION 0.7 # Ensure we are using an amd64 platform because some of these targets use cross-platform tooling FROM ubuntu:18.04 -ENV ZIG_VERSION=0.13.0-dev.46+3648d7df1 +ENV ZIG_VERSION=0.13.0 ENV CMAKE_VERSION_MINOR=3.30 ENV CMAKE_VERSION_PATCH=3.30.1 ENV WASM_BINDGEN_CLI_VERSION=0.2.93 +ENV RUST_VERSION=1.81.0 ENV RUSTUP_HOME=/usr/local/rustup ENV RUSTUP_DIST_SERVER=https://static.rust-lang.org ENV CARGO_HOME=/usr/local/cargo @@ -40,7 +41,7 @@ deps-base: # Install Rust deps-rust: FROM +deps-base - RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal + RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain=$RUST_VERSION -y -c clippy --no-modify-path --profile minimal RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \ rustup --version; \ cargo --version; \ @@ -60,7 +61,7 @@ deps-rust: # Caching tool RUN cargo install cargo-chef # Install Linux cross-platform tooling - RUN curl -O https://ziglang.org/builds/zig-linux-$(arch)-$ZIG_VERSION.tar.xz + RUN curl -O https://ziglang.org/download/$ZIG_VERSION/zig-linux-$(arch)-$ZIG_VERSION.tar.xz RUN tar -C /usr/local -xJf zig-linux-$(arch)-$ZIG_VERSION.tar.xz RUN mv /usr/local/zig-linux-$(arch)-$ZIG_VERSION /usr/local/zig RUN cargo install cargo-zigbuild diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index ef78f6ff..ce11f562 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Veilid Team "] edition = "2021" license = "MPL-2.0" resolver = "2" +rust-version = "1.81.0" [[bin]] name = "veilid-cli" diff --git a/veilid-cli/src/cached_text_view.rs b/veilid-cli/src/cached_text_view.rs index d6137c88..67eaefa3 100644 --- a/veilid-cli/src/cached_text_view.rs +++ b/veilid-cli/src/cached_text_view.rs @@ -62,8 +62,6 @@ pub struct TextContent { content: Arc>, } -#[allow(dead_code)] - impl TextContent { /// Creates a new text content around the given value. /// @@ -129,6 +127,7 @@ impl TextContent { } /// Remove lines from the end until we have no more than 'count' from the beginning + #[expect(dead_code)] pub fn resize_front(&self, count: usize) { if self.get_content().len() <= count { return; @@ -238,7 +237,6 @@ pub struct CachedTextView { width: Option, } -#[allow(dead_code)] impl CachedTextView { /// Creates a new TextView with the given content. pub fn new(content: S, cache_size: usize, max_lines: Option) -> Self @@ -281,6 +279,7 @@ impl CachedTextView { } /// Creates a new empty `TextView`. + #[expect(dead_code)] pub fn empty(cache_size: usize, max_lines: Option) -> Self { CachedTextView::new(ContentType::default(), cache_size, max_lines) } @@ -295,6 +294,7 @@ impl CachedTextView { /// /// Chainable variant. #[must_use] + #[expect(dead_code)] pub fn style>(self, style: S) -> Self { self.with(|s| s.set_style(style)) } @@ -303,6 +303,7 @@ impl CachedTextView { /// /// This may be useful if you want horizontal scrolling. #[must_use] + #[expect(dead_code)] pub fn no_wrap(self) -> Self { self.with(|s| s.set_content_wrap(false)) } @@ -317,6 +318,7 @@ impl CachedTextView { /// Sets the horizontal alignment for this view. #[must_use] + #[expect(dead_code)] pub fn h_align(mut self, h: HAlign) -> Self { self.align.h = h; @@ -325,6 +327,7 @@ impl CachedTextView { /// Sets the vertical alignment for this view. #[must_use] + #[expect(dead_code)] pub fn v_align(mut self, v: VAlign) -> Self { self.align.v = v; @@ -333,6 +336,7 @@ impl CachedTextView { /// Sets the alignment for this view. #[must_use] + #[expect(dead_code)] pub fn align(mut self, a: Align) -> Self { self.align = a; @@ -341,6 +345,7 @@ impl CachedTextView { /// Center the text horizontally and vertically inside the view. #[must_use] + #[expect(dead_code)] pub fn center(mut self) -> Self { self.align = Align::center(); self @@ -350,6 +355,7 @@ impl CachedTextView { /// /// Chainable variant. #[must_use] + #[expect(dead_code)] pub fn content(self, content: S) -> Self where S: Into, @@ -392,11 +398,13 @@ impl CachedTextView { } /// Returns the current text in this view. + #[cfg_attr(not(test), expect(dead_code))] pub fn get_content(&self) -> TextContentRef { TextContentInner::get_content(&self.content.content) } /// Returns a shared reference to the content, allowing content mutation. + #[expect(dead_code)] pub fn get_shared_content(&mut self) -> TextContent { // We take &mut here without really needing it, // because it sort of "makes sense". diff --git a/veilid-cli/src/interactive_ui.rs b/veilid-cli/src/interactive_ui.rs index c6c17bcf..c076c40f 100644 --- a/veilid-cli/src/interactive_ui.rs +++ b/veilid-cli/src/interactive_ui.rs @@ -103,9 +103,15 @@ impl InteractiveUI { println!("Error: {:?}", e); break; } + match readline.readline().timeout_at(done.clone()).await { Ok(Ok(ReadlineEvent::Line(line))) => { let line = line.trim(); + + if !line.is_empty() { + readline.add_history_entry(line.to_string()); + } + if line == "clear" { if let Err(e) = readline.clear() { println!("Error: {:?}", e); @@ -183,7 +189,6 @@ impl InteractiveUI { self.inner.lock().log_enabled = false; } } else if !line.is_empty() { - readline.add_history_entry(line.to_string()); let cmdproc = self.inner.lock().cmdproc.clone(); if let Some(cmdproc) = &cmdproc { if let Err(e) = cmdproc.run_command( diff --git a/veilid-cli/src/main.rs b/veilid-cli/src/main.rs index 1698cfa0..5ab103fd 100644 --- a/veilid-cli/src/main.rs +++ b/veilid-cli/src/main.rs @@ -36,7 +36,7 @@ struct CmdlineArgs { #[arg(long, short = 'p')] ipc_path: Option, /// Subnode index to use when connecting - #[arg(long, default_value = "0")] + #[arg(short('n'), long, default_value = "0")] subnode_index: usize, /// Address to connect to #[arg(long, short = 'a')] diff --git a/veilid-cli/src/settings.rs b/veilid-cli/src/settings.rs index 550dc123..e83cd915 100644 --- a/veilid-cli/src/settings.rs +++ b/veilid-cli/src/settings.rs @@ -229,7 +229,7 @@ pub struct Settings { } impl Settings { - #[allow(dead_code)] + #[cfg_attr(windows, expect(dead_code))] fn get_server_default_directory(subpath: &str) -> PathBuf { #[cfg(unix)] { diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index e4c06b99..25f26e86 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -10,6 +10,7 @@ edition = "2021" build = "build.rs" license = "MPL-2.0" resolver = "2" +rust-version = "1.81.0" [lib] crate-type = ["cdylib", "staticlib", "rlib"] diff --git a/veilid-core/src/crypto/envelope.rs b/veilid-core/src/crypto/envelope.rs index 8a4f3933..22ab3743 100644 --- a/veilid-core/src/crypto/envelope.rs +++ b/veilid-core/src/crypto/envelope.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] #![allow(clippy::absurd_extreme_comparisons)] use super::*; use crate::*; diff --git a/veilid-core/src/crypto/receipt.rs b/veilid-core/src/crypto/receipt.rs index ab8af8b2..65462c45 100644 --- a/veilid-core/src/crypto/receipt.rs +++ b/veilid-core/src/crypto/receipt.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] #![allow(clippy::absurd_extreme_comparisons)] use super::*; use crate::*; diff --git a/veilid-core/src/intf/native/system.rs b/veilid-core/src/intf/native/system.rs index ed8dba81..3bab9ad9 100644 --- a/veilid-core/src/intf/native/system.rs +++ b/veilid-core/src/intf/native/system.rs @@ -1,8 +1,8 @@ -#![allow(dead_code)] +use super::*; -use crate::*; - -pub async fn get_outbound_relay_peer() -> Option { +pub async fn get_outbound_relay_peer( + _routing_domain: routing_table::RoutingDomain, +) -> Option> { panic!("Native Veilid should never require an outbound relay"); } diff --git a/veilid-core/src/intf/wasm/system.rs b/veilid-core/src/intf/wasm/system.rs index 727244e3..49b48c4d 100644 --- a/veilid-core/src/intf/wasm/system.rs +++ b/veilid-core/src/intf/wasm/system.rs @@ -1,8 +1,10 @@ -use crate::*; +use super::*; //use js_sys::*; -pub async fn get_outbound_relay_peer() -> Option { +pub async fn get_outbound_relay_peer( + _routing_domain: routing_table::RoutingDomain, +) -> Option> { // unimplemented! None } diff --git a/veilid-core/src/logging/facilities.rs b/veilid-core/src/logging/facilities.rs index 7e351b68..bf984edf 100644 --- a/veilid-core/src/logging/facilities.rs +++ b/veilid-core/src/logging/facilities.rs @@ -1,4 +1,4 @@ -pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 28] = [ +pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 29] = [ "mio", "h2", "hyper", @@ -27,6 +27,7 @@ pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 28] = [ "dht", "fanout", "receipt", + "rpc_message", ]; pub static FLAME_LOG_FACILITIES_IGNORE_LIST: [&str; 22] = [ @@ -425,3 +426,31 @@ macro_rules! log_crypto { trace!(target:"crypto", $fmt, $($arg),+); } } + +#[macro_export] +macro_rules! log_rpc_message { + (error $text:expr) => { error!( + target: "rpc_message", + "{}", + $text, + )}; + (error $fmt:literal, $($arg:expr),+) => { + error!(target:"crypto", $fmt, $($arg),+); + }; + (warn $text:expr) => { warn!( + target: "rpc_message", + "{}", + $text, + )}; + (warn $fmt:literal, $($arg:expr),+) => { + warn!(target:"crypto", $fmt, $($arg),+); + }; + ($text:expr) => {trace!( + target: "rpc_message", + "{}", + $text, + )}; + ($fmt:literal, $($arg:expr),+) => { + trace!(target:"rpc_message", $fmt, $($arg),+); + } +} diff --git a/veilid-core/src/network_manager/address_filter.rs b/veilid-core/src/network_manager/address_filter.rs index e084165e..76772bad 100644 --- a/veilid-core/src/network_manager/address_filter.rs +++ b/veilid-core/src/network_manager/address_filter.rs @@ -158,7 +158,7 @@ impl AddressFilter { } } for key in dead_keys { - log_net!(debug ">>> FORGIVING: {}", key); + warn!("Forgiving: {}", key); inner.punishments_by_ip4.remove(&key); } } @@ -174,7 +174,7 @@ impl AddressFilter { } } for key in dead_keys { - log_net!(debug ">>> FORGIVING: {}", key); + warn!("Forgiving: {}", key); inner.punishments_by_ip6_prefix.remove(&key); } } @@ -190,7 +190,7 @@ impl AddressFilter { } } for key in dead_keys { - log_net!(debug ">>> FORGIVING: {}", key); + warn!("Forgiving: {}", key); inner.punishments_by_node_id.remove(&key); // make the entry alive again if it's still here if let Ok(Some(nr)) = self.unlocked_inner.routing_table.lookup_node_ref(key) { @@ -210,7 +210,7 @@ impl AddressFilter { } } for key in dead_keys { - log_net!(debug ">>> DIALINFO PERMIT: {}", key); + log_net!(debug "DialInfo Permit: {}", key); inner.dial_info_failures.remove(&key); } } @@ -259,10 +259,10 @@ impl AddressFilter { let mut inner = self.inner.lock(); if inner.dial_info_failures.len() >= MAX_DIAL_INFO_FAILURES { - log_net!(debug ">>> DIALINFO FAILURE TABLE FULL: {}", dial_info); + warn!("DialInfo failure table full: {}", dial_info); return; } - log_net!(debug ">>> DIALINFO FAILURE: {:?}", dial_info); + log_net!(debug "DialInfo failure: {:?}", dial_info); inner .dial_info_failures .entry(dial_info) @@ -279,7 +279,7 @@ impl AddressFilter { } pub fn punish_ip_addr(&self, addr: IpAddr, reason: PunishmentReason) { - log_net!(debug ">>> PUNISHED: {} for {:?}", addr, reason); + warn!("Punished: {} for {:?}", addr, reason); let timestamp = Timestamp::now(); let punishment = Punishment { reason, timestamp }; @@ -326,10 +326,10 @@ impl AddressFilter { let mut inner = self.inner.lock(); if inner.punishments_by_node_id.len() >= MAX_PUNISHMENTS_BY_NODE_ID { - log_net!(debug ">>> PUNISHMENT TABLE FULL: {}", node_id); + warn!("Punishment table full: {}", node_id); return; } - log_net!(debug ">>> PUNISHED: {} for {:?}", node_id, reason); + warn!("Punished: {} for {:?}", node_id, reason); inner .punishments_by_node_id .entry(node_id) @@ -372,7 +372,7 @@ impl AddressFilter { let cnt = inner.conn_count_by_ip4.entry(v4).or_default(); assert!(*cnt <= self.unlocked_inner.max_connections_per_ip4); if *cnt == self.unlocked_inner.max_connections_per_ip4 { - warn!("address filter count exceeded: {:?}", v4); + warn!("Address filter count exceeded: {:?}", v4); return Err(AddressFilterError::CountExceeded); } // See if this ip block has connected too frequently @@ -383,7 +383,7 @@ impl AddressFilter { }); assert!(tstamps.len() <= self.unlocked_inner.max_connection_frequency_per_min); if tstamps.len() == self.unlocked_inner.max_connection_frequency_per_min { - warn!("address filter rate exceeded: {:?}", v4); + warn!("Address filter rate exceeded: {:?}", v4); return Err(AddressFilterError::RateExceeded); } @@ -396,14 +396,14 @@ impl AddressFilter { let cnt = inner.conn_count_by_ip6_prefix.entry(v6).or_default(); assert!(*cnt <= self.unlocked_inner.max_connections_per_ip6_prefix); if *cnt == self.unlocked_inner.max_connections_per_ip6_prefix { - warn!("address filter count exceeded: {:?}", v6); + warn!("Address filter count exceeded: {:?}", v6); return Err(AddressFilterError::CountExceeded); } // See if this ip block has connected too frequently let tstamps = inner.conn_timestamps_by_ip6_prefix.entry(v6).or_default(); assert!(tstamps.len() <= self.unlocked_inner.max_connection_frequency_per_min); if tstamps.len() == self.unlocked_inner.max_connection_frequency_per_min { - warn!("address filter rate exceeded: {:?}", v6); + warn!("Address filter rate exceeded: {:?}", v6); return Err(AddressFilterError::RateExceeded); } diff --git a/veilid-core/src/network_manager/connection_handle.rs b/veilid-core/src/network_manager/connection_handle.rs index c5606360..29e77127 100644 --- a/veilid-core/src/network_manager/connection_handle.rs +++ b/veilid-core/src/network_manager/connection_handle.rs @@ -26,12 +26,12 @@ impl ConnectionHandle { } } - #[allow(dead_code)] + #[expect(dead_code)] pub fn connection_id(&self) -> NetworkConnectionId { self.connection_id } - #[allow(dead_code)] + #[expect(dead_code)] pub fn flow(&self) -> Flow { self.flow } diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index eb6501ac..682dd898 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -9,7 +9,6 @@ use stop_token::future::FutureExt; #[derive(Debug)] enum ConnectionManagerEvent { - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] Accepted(ProtocolNetworkConnection), Dead(NetworkConnection), } @@ -45,6 +44,7 @@ struct ConnectionManagerInner { sender: flume::Sender, async_processor_jh: Option>, stop_source: Option, + protected_addresses: HashMap, } struct ConnectionManagerArc { @@ -80,6 +80,7 @@ impl ConnectionManager { stop_source: Some(stop_source), sender, async_processor_jh: Some(async_processor_jh), + protected_addresses: HashMap::new(), } } fn new_arc(network_manager: NetworkManager) -> ConnectionManagerArc { @@ -182,25 +183,61 @@ impl ConnectionManager { // Internal routine to see if we should keep this connection // from being LRU removed. Used on our initiated relay connections. - fn should_protect_connection(&self, conn: &NetworkConnection) -> Option { - let netman = self.network_manager(); - let routing_table = netman.routing_table(); - let remote_address = conn.flow().remote_address().address(); - let routing_domain = routing_table.routing_domain_for_address(remote_address)?; - let relay_node = routing_table.relay_node(routing_domain)?; - let relay_nr = relay_node.filtered_clone( - NodeRefFilter::new() - .with_routing_domain(routing_domain) - .with_address_type(conn.flow().address_type()) - .with_protocol_type(conn.flow().protocol_type()), - ); - let dids = relay_nr.all_filtered_dial_info_details(); - for did in dids { - if did.dial_info.address() == remote_address { - return Some(relay_nr); + fn should_protect_connection( + &self, + inner: &mut ConnectionManagerInner, + conn: &NetworkConnection, + ) -> Option { + inner + .protected_addresses + .get(conn.flow().remote_address()) + .cloned() + } + + // Update connection protections if things change, like a node becomes a relay + pub fn update_protections(&self) { + let Ok(_guard) = self.arc.startup_lock.enter() else { + return; + }; + + let mut lock = self.arc.inner.lock(); + let Some(inner) = lock.as_mut() else { + return; + }; + + // Get addresses for relays in all routing domains + inner.protected_addresses.clear(); + for routing_domain in RoutingDomainSet::all() { + let Some(relay_node) = self + .network_manager() + .routing_table() + .relay_node(routing_domain) + else { + continue; + }; + for did in relay_node.dial_info_details() { + // SocketAddress are distinct per routing domain, so they should not collide + // and two nodes should never have the same SocketAddress + inner + .protected_addresses + .insert(did.dial_info.socket_address(), relay_node.unfiltered()); } } - None + + self.arc + .connection_table + .with_all_connections_mut(|conn| { + if let Some(protect_nr) = conn.protected_node_ref() { + if self.should_protect_connection(inner, conn).is_none() { + log_net!(debug "== Unprotecting connection: {} -> {} for node {}", conn.connection_id(), conn.debug_print(Timestamp::now()), protect_nr); + conn.unprotect(); + } + } else if let Some(protect_nr) = self.should_protect_connection(inner, conn) { + log_net!(debug "== Protecting existing connection: {} -> {} for node {}", conn.connection_id(), conn.debug_print(Timestamp::now()), protect_nr); + conn.protect(protect_nr); + } + Option::<()>::None + }); } // Internal routine to register new connection atomically. @@ -231,8 +268,8 @@ impl ConnectionManager { let handle = conn.get_handle(); // See if this should be a protected connection - if let Some(protect_nr) = self.should_protect_connection(&conn) { - log_net!(debug "== PROTECTING connection: {} -> {} for node {}", id, conn.debug_print(Timestamp::now()), protect_nr); + if let Some(protect_nr) = self.should_protect_connection(inner, &conn) { + log_net!(debug "== Protecting new connection: {} -> {} for node {}", id, conn.debug_print(Timestamp::now()), protect_nr); conn.protect(protect_nr); } @@ -472,7 +509,7 @@ impl ConnectionManager { // Called by low-level network when any connection-oriented protocol connection appears // either from incoming connections. - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub(super) async fn on_accepted_protocol_network_connection( &self, protocol_connection: ProtocolNetworkConnection, diff --git a/veilid-core/src/network_manager/connection_table.rs b/veilid-core/src/network_manager/connection_table.rs index dd3b29b1..b10794d3 100644 --- a/veilid-core/src/network_manager/connection_table.rs +++ b/veilid-core/src/network_manager/connection_table.rs @@ -246,6 +246,58 @@ impl ConnectionTable { let _ = inner.conn_by_id[protocol_index].get(&id).unwrap(); } + #[expect(dead_code)] + pub fn with_connection_by_flow R>( + &self, + flow: Flow, + closure: F, + ) -> Option { + if flow.protocol_type() == ProtocolType::UDP { + return None; + } + + let inner = self.inner.lock(); + + let id = *inner.id_by_flow.get(&flow)?; + let protocol_index = Self::protocol_to_index(flow.protocol_type()); + let out = inner.conn_by_id[protocol_index].peek(&id).unwrap(); + Some(closure(out)) + } + + #[expect(dead_code)] + pub fn with_connection_by_flow_mut R>( + &self, + flow: Flow, + closure: F, + ) -> Option { + if flow.protocol_type() == ProtocolType::UDP { + return None; + } + + let mut inner = self.inner.lock(); + + let id = *inner.id_by_flow.get(&flow)?; + let protocol_index = Self::protocol_to_index(flow.protocol_type()); + let out = inner.conn_by_id[protocol_index].peek_mut(&id).unwrap(); + Some(closure(out)) + } + + pub fn with_all_connections_mut Option>( + &self, + mut closure: F, + ) -> Option { + let mut inner_lock = self.inner.lock(); + let inner = &mut *inner_lock; + for (id, idx) in inner.protocol_index_by_id.iter() { + if let Some(conn) = inner.conn_by_id[*idx].peek_mut(id) { + if let Some(out) = closure(conn) { + return Some(out); + } + } + } + None + } + //#[instrument(level = "trace", skip(self), ret)] pub fn ref_connection_by_id( &self, @@ -304,7 +356,7 @@ impl ConnectionTable { } //#[instrument(level = "trace", skip(self), ret)] - #[allow(dead_code)] + #[expect(dead_code)] pub fn get_connection_ids_by_remote(&self, remote: PeerAddress) -> Vec { let inner = self.inner.lock(); inner diff --git a/veilid-core/src/network_manager/direct_boot.rs b/veilid-core/src/network_manager/direct_boot.rs index 38ec60fe..554a28f7 100644 --- a/veilid-core/src/network_manager/direct_boot.rs +++ b/veilid-core/src/network_manager/direct_boot.rs @@ -34,7 +34,7 @@ impl NetworkManager { // Direct bootstrap request #[instrument(level = "trace", target = "net", err, skip(self))] - pub async fn boot_request(&self, dial_info: DialInfo) -> EyreResult> { + pub async fn boot_request(&self, dial_info: DialInfo) -> EyreResult>> { let timeout_ms = self.with_config(|c| c.network.rpc.timeout_ms); // Send boot magic to requested peer address let data = BOOT_MAGIC.to_vec(); @@ -51,6 +51,6 @@ impl NetworkManager { deserialize_json(std::str::from_utf8(&out_data).wrap_err("bad utf8 in boot peerinfo")?) .wrap_err("failed to deserialize boot peerinfo")?; - Ok(bootstrap_peerinfo) + Ok(bootstrap_peerinfo.into_iter().map(Arc::new).collect()) } } diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 1de7707b..4b1ffc11 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -54,8 +54,9 @@ pub const IPADDR_TABLE_SIZE: usize = 1024; pub const IPADDR_MAX_INACTIVE_DURATION_US: TimestampDuration = TimestampDuration::new(300_000_000u64); // 5 minutes pub const NODE_CONTACT_METHOD_CACHE_SIZE: usize = 1024; -pub const PUBLIC_ADDRESS_CHANGE_DETECTION_COUNT: usize = 5; -pub const PUBLIC_ADDRESS_CHECK_CACHE_SIZE: usize = 10; +pub const PUBLIC_ADDRESS_CHANGE_CONSISTENCY_DETECTION_COUNT: usize = 3; // Number of consistent results in the cache during outbound-only to trigger detection +pub const PUBLIC_ADDRESS_CHANGE_INCONSISTENCY_DETECTION_COUNT: usize = 7; // Number of inconsistent results in the cache during inbound-capable to trigger detection +pub const PUBLIC_ADDRESS_CHECK_CACHE_SIZE: usize = 10; // Length of consistent/inconsistent result cache for detection pub const PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS: u32 = 60; pub const PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US: TimestampDuration = TimestampDuration::new(300_000_000u64); // 5 minutes @@ -64,18 +65,6 @@ pub const PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US: TimestampDuration pub const ADDRESS_FILTER_TASK_INTERVAL_SECS: u32 = 60; pub const BOOT_MAGIC: &[u8; 4] = b"BOOT"; -#[derive(Clone, Debug, Default)] -pub struct ProtocolConfig { - pub outbound: ProtocolTypeSet, - pub inbound: ProtocolTypeSet, - pub family_global: AddressTypeSet, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - pub family_local: AddressTypeSet, - pub public_internet_capabilities: Vec, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - pub local_network_capabilities: Vec, -} - // Things we get when we start up and go away when we shut down // Routing table is not in here because we want it to survive a network shutdown/startup restart #[derive(Clone)] @@ -111,20 +100,20 @@ pub(crate) enum NodeContactMethod { /// Contact the node directly Direct(DialInfo), /// Request via signal the node connect back directly (relay, target) - SignalReverse(NodeRef, NodeRef), + SignalReverse(FilteredNodeRef, FilteredNodeRef), /// Request via signal the node negotiate a hole punch (relay, target) - SignalHolePunch(NodeRef, NodeRef), + SignalHolePunch(FilteredNodeRef, FilteredNodeRef), /// Must use an inbound relay to reach the node - InboundRelay(NodeRef), + InboundRelay(FilteredNodeRef), /// Must use outbound relay to reach the node - OutboundRelay(NodeRef), + OutboundRelay(FilteredNodeRef), } #[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Hash)] struct NodeContactMethodCacheKey { node_ids: TypedKeyGroup, own_node_info_ts: Timestamp, target_node_info_ts: Timestamp, - target_node_ref_filter: Option, + target_node_ref_filter: NodeRefFilter, target_node_ref_sequencing: Sequencing, } @@ -139,7 +128,7 @@ enum SendDataToExistingFlowResult { #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum StartupDisposition { Success, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] BindRetry, } @@ -148,9 +137,9 @@ struct NetworkManagerInner { stats: NetworkManagerStats, client_allowlist: LruCache, node_contact_method_cache: LruCache, - public_address_check_cache: + public_internet_address_check_cache: BTreeMap>, - public_address_inconsistencies_table: + public_internet_address_inconsistencies_table: BTreeMap>, } @@ -169,7 +158,7 @@ struct NetworkManagerUnlockedInner { update_callback: RwLock>, // Background processes rolling_transfers_task: TickTask, - public_address_check_task: TickTask, + public_internet_address_check_task: TickTask, address_filter_task: TickTask, // Network Key network_key: Option, @@ -189,8 +178,8 @@ impl NetworkManager { stats: NetworkManagerStats::default(), client_allowlist: LruCache::new_unbounded(), node_contact_method_cache: LruCache::new(NODE_CONTACT_METHOD_CACHE_SIZE), - public_address_check_cache: BTreeMap::new(), - public_address_inconsistencies_table: BTreeMap::new(), + public_internet_address_check_cache: BTreeMap::new(), + public_internet_address_inconsistencies_table: BTreeMap::new(), } } fn new_unlocked_inner( @@ -216,7 +205,7 @@ impl NetworkManager { "rolling_transfers_task", ROLLING_TRANSFERS_INTERVAL_SECS, ), - public_address_check_task: TickTask::new( + public_internet_address_check_task: TickTask::new( "public_address_check_task", PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS, ), @@ -525,6 +514,7 @@ impl NetworkManager { log_net!(debug "finished network manager shutdown"); } + #[expect(dead_code)] pub fn update_client_allowlist(&self, client: TypedKey) { let mut inner = self.inner.lock(); match inner.client_allowlist.entry(client) { @@ -693,7 +683,7 @@ impl NetworkManager { pub async fn handle_in_band_receipt>( &self, receipt_data: R, - inbound_noderef: NodeRef, + inbound_noderef: FilteredNodeRef, ) -> NetworkResult<()> { let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else { return NetworkResult::service_unavailable("network is not started"); @@ -779,11 +769,8 @@ impl NetworkManager { let rpc = self.rpc_processor(); // Add the peer info to our routing table - let mut peer_nr = match routing_table.register_node_with_peer_info( - RoutingDomain::PublicInternet, - peer_info, - false, - ) { + let mut peer_nr = match routing_table.register_node_with_peer_info(peer_info, false) + { Ok(nr) => nr, Err(e) => { return Ok(NetworkResult::invalid_message(format!( @@ -808,11 +795,8 @@ impl NetworkManager { let rpc = self.rpc_processor(); // Add the peer info to our routing table - let mut peer_nr = match routing_table.register_node_with_peer_info( - RoutingDomain::PublicInternet, - peer_info, - false, - ) { + let mut peer_nr = match routing_table.register_node_with_peer_info(peer_info, false) + { Ok(nr) => nr, Err(e) => { return Ok(NetworkResult::invalid_message(format!( @@ -826,9 +810,8 @@ impl NetworkManager { let outbound_nrf = routing_table .get_outbound_node_ref_filter(RoutingDomain::PublicInternet) .with_protocol_type(ProtocolType::UDP); - peer_nr.set_filter(Some(outbound_nrf)); - let Some(hole_punch_dial_info_detail) = peer_nr.first_filtered_dial_info_detail() - else { + peer_nr.set_filter(outbound_nrf); + let Some(hole_punch_dial_info_detail) = peer_nr.first_dial_info_detail() else { return Ok(NetworkResult::no_connection_other(format!( "No hole punch capable dialinfo found for node: {}", peer_nr @@ -836,10 +819,10 @@ impl NetworkManager { }; // Now that we picked a specific dialinfo, further restrict the noderef to the specific address type - let filter = peer_nr.take_filter().unwrap(); + let filter = peer_nr.take_filter(); let filter = filter.with_address_type(hole_punch_dial_info_detail.dial_info.address_type()); - peer_nr.set_filter(Some(filter)); + peer_nr.set_filter(filter); // Do our half of the hole punch by sending an empty packet // Both sides will do this and then the receipt will get sent over the punched hole @@ -912,7 +895,7 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all)] pub async fn send_envelope>( &self, - node_ref: NodeRef, + node_ref: FilteredNodeRef, destination_node_ref: Option, body: B, ) -> EyreResult> { @@ -920,7 +903,7 @@ impl NetworkManager { return Ok(NetworkResult::no_connection_other("network is not started")); }; - let destination_node_ref = destination_node_ref.as_ref().unwrap_or(&node_ref).clone(); + let destination_node_ref = destination_node_ref.unwrap_or_else(|| node_ref.unfiltered()); let best_node_id = destination_node_ref.best_node_id(); // Get node's envelope versions and see if we can send to it @@ -1110,7 +1093,7 @@ impl NetworkManager { .resolve_node(recipient_id, SafetySelection::Unsafe(Sequencing::default())) .await { - Ok(v) => v, + Ok(v) => v.map(|nr| nr.default_filtered()), Err(e) => { log_net!(debug "failed to resolve recipient node for relay, dropping outbound relayed packet: {}" ,e); return Ok(false); @@ -1125,7 +1108,7 @@ impl NetworkManager { // should be mutually in each others routing tables. The node needing the relay will be // pinging this node regularly to keep itself in the routing table match routing_table.lookup_node_ref(recipient_id) { - Ok(v) => v, + Ok(v) => v.map(|nr| nr.default_filtered()), Err(e) => { log_net!(debug "failed to look up recipient node for relay, dropping outbound relayed packet: {}" ,e); return Ok(false); @@ -1182,7 +1165,8 @@ impl NetworkManager { }; // Cache the envelope information in the routing table - let mut source_noderef = match routing_table.register_node_with_existing_connection( + let source_noderef = match routing_table.register_node_with_existing_connection( + routing_domain, envelope.get_sender_typed_id(), flow, ts, @@ -1196,9 +1180,6 @@ impl NetworkManager { }; source_noderef.add_envelope_version(envelope.get_version()); - // Enforce routing domain - source_noderef.merge_filter(NodeRefFilter::new().with_routing_domain(routing_domain)); - // Pass message to RPC system if let Err(e) = rpc.enqueue_direct_message(envelope, source_noderef, flow, routing_domain, body) diff --git a/veilid-core/src/network_manager/native/discovery_context.rs b/veilid-core/src/network_manager/native/discovery_context.rs index cb45b53c..de8bd572 100644 --- a/veilid-core/src/network_manager/native/discovery_context.rs +++ b/veilid-core/src/network_manager/native/discovery_context.rs @@ -136,7 +136,7 @@ impl DiscoveryContext { // Ask for a public address check from a particular noderef // This is done over the normal port using RPC #[instrument(level = "trace", skip(self), ret)] - async fn request_public_address(&self, node_ref: NodeRef) -> Option { + async fn request_public_address(&self, node_ref: FilteredNodeRef) -> Option { let rpc = self.unlocked_inner.routing_table.rpc_processor(); let res = network_result_value_or_log!(match rpc.rpc_call_status(Destination::direct(node_ref.clone())).await { @@ -217,7 +217,7 @@ impl DiscoveryContext { let nodes = self .unlocked_inner .routing_table - .find_fast_public_nodes_filtered(node_count, filters); + .find_fast_non_local_nodes_filtered(routing_domain, node_count, filters); if nodes.is_empty() { log_net!(debug "no external address detection peers of type {:?}:{:?}", @@ -232,7 +232,7 @@ impl DiscoveryContext { let get_public_address_func = |node: NodeRef| { let this = self.clone(); - let node = node.filtered_clone( + let node = node.custom_filtered( NodeRefFilter::new() .with_routing_domain(routing_domain) .with_dial_info_filter(dial_info_filter), @@ -246,7 +246,7 @@ impl DiscoveryContext { return Some(ExternalInfo { dial_info, address, - node, + node: node.unfiltered(), }); } None @@ -308,12 +308,7 @@ impl DiscoveryContext { ) -> bool { let rpc_processor = self.unlocked_inner.routing_table.rpc_processor(); - // asking for node validation doesn't have to use the dial info filter of the dial info we are validating - let mut node_ref = node_ref.clone(); - node_ref.set_filter(None); - // ask the node to send us a dial info validation receipt - match rpc_processor .rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect) .await diff --git a/veilid-core/src/network_manager/native/mod.rs b/veilid-core/src/network_manager/native/mod.rs index 6df4b164..c71a4c88 100644 --- a/veilid-core/src/network_manager/native/mod.rs +++ b/veilid-core/src/network_manager/native/mod.rs @@ -1,15 +1,17 @@ mod discovery_context; mod igd_manager; -mod network_class_discovery; +mod network_state; mod network_tcp; mod network_udp; mod protocol; mod start_protocols; +mod tasks; use super::*; use crate::routing_table::*; use connection_manager::*; use discovery_context::*; +use network_state::*; use network_tcp::*; use protocol::tcp::RawTcpProtocolHandler; use protocol::udp::RawUdpProtocolHandler; @@ -75,26 +77,19 @@ struct NetworkInner { /// set if the network needs to be restarted due to a low level configuration change /// such as dhcp release or change of address or interfaces being added or removed network_needs_restart: bool, - /// the calculated protocol configuration for inbound/outbound protocols - protocol_config: ProtocolConfig, - /// set of statically configured protocols with public dialinfo - static_public_dialinfo: ProtocolTypeSet, + /// join handles for all the low level network background tasks join_handles: Vec>, /// stop source for shutting down the low level network background tasks stop_source: Option, - /// does our network have ipv4 on any network? - enable_ipv4: bool, - /// does our network have ipv6 on the global internet? - enable_ipv6_global: bool, - /// does our network have ipv6 on the local network? - enable_ipv6_local: bool, /// set if we need to calculate our public dial info again needs_public_dial_info_check: bool, /// set if we have yet to clear the network during public dial info checking network_already_cleared: bool, /// the punishment closure to enax public_dial_info_check_punishment: Option>, + /// Actual bound addresses per protocol + bound_address_per_protocol: BTreeMap>, /// mapping of protocol handlers to accept messages from a set of bound socket addresses udp_protocol_handlers: BTreeMap, /// outbound udp protocol handler for udpv4 @@ -107,8 +102,10 @@ struct NetworkInner { listener_states: BTreeMap>>, /// Preferred local addresses for protocols/address combinations for outgoing connections preferred_local_addresses: BTreeMap<(ProtocolType, AddressType), SocketAddr>, - /// The list of stable interface addresses we have last seen - stable_interface_addresses_at_startup: Vec, + /// set of statically configured protocols with public dialinfo + static_public_dial_info: ProtocolTypeSet, + /// Network state + network_state: Option, } struct NetworkUnlockedInner { @@ -125,6 +122,7 @@ struct NetworkUnlockedInner { update_network_class_task: TickTask, network_interfaces_task: TickTask, upnp_task: TickTask, + network_task_lock: AsyncMutex<()>, // Managers igd_manager: igd_manager::IGDManager, @@ -144,20 +142,17 @@ impl Network { needs_public_dial_info_check: false, network_already_cleared: false, public_dial_info_check_punishment: None, - protocol_config: Default::default(), - static_public_dialinfo: ProtocolTypeSet::empty(), join_handles: Vec::new(), stop_source: None, - enable_ipv4: false, - enable_ipv6_global: false, - enable_ipv6_local: false, + bound_address_per_protocol: BTreeMap::new(), udp_protocol_handlers: BTreeMap::new(), default_udpv4_protocol_handler: None, default_udpv6_protocol_handler: None, tls_acceptor: None, listener_states: BTreeMap::new(), preferred_local_addresses: BTreeMap::new(), - stable_interface_addresses_at_startup: Vec::new(), + static_public_dial_info: ProtocolTypeSet::new(), + network_state: None, } } @@ -176,6 +171,7 @@ impl Network { update_network_class_task: TickTask::new("update_network_class_task", 1), network_interfaces_task: TickTask::new("network_interfaces_task", 1), upnp_task: TickTask::new("upnp_task", 1), + network_task_lock: AsyncMutex::new(()), igd_manager: igd_manager::IGDManager::new(config.clone()), } } @@ -195,31 +191,7 @@ impl Network { )), }; - // Set update network class tick task - { - let this2 = this.clone(); - this.unlocked_inner - .update_network_class_task - .set_routine(move |s, l, t| { - Box::pin(this2.clone().update_network_class_task_routine(s, l, t)) - }); - } - // Set network interfaces tick task - { - let this2 = this.clone(); - this.unlocked_inner - .network_interfaces_task - .set_routine(move |s, l, t| { - Box::pin(this2.clone().network_interfaces_task_routine(s, l, t)) - }); - } - // Set upnp tick task - { - let this2 = this.clone(); - this.unlocked_inner - .upnp_task - .set_routine(move |s, l, t| Box::pin(this2.clone().upnp_task_routine(s, l, t))); - } + this.setup_tasks(); this } @@ -301,11 +273,11 @@ impl Network { inner.join_handles.push(jh); } - fn translate_unspecified_address(&self, from: &SocketAddr) -> Vec { + fn translate_unspecified_address(&self, from: SocketAddr) -> Vec { if !from.ip().is_unspecified() { - vec![*from] + vec![from] } else { - let addrs = self.get_stable_interface_addresses(); + let addrs = self.last_network_state().stable_interface_addresses; addrs .iter() .filter_map(|a| { @@ -336,46 +308,6 @@ impl Network { inner.preferred_local_addresses.get(&key).copied() } - pub(crate) fn is_stable_interface_address(&self, addr: IpAddr) -> bool { - let stable_addrs = self.get_stable_interface_addresses(); - stable_addrs.contains(&addr) - } - - pub(crate) fn get_stable_interface_addresses(&self) -> Vec { - let addrs = self.unlocked_inner.interfaces.stable_addresses(); - let mut addrs: Vec = addrs - .into_iter() - .filter(|addr| { - let address = Address::from_ip_addr(*addr); - address.is_local() || address.is_global() - }) - .collect(); - addrs.sort(); - addrs.dedup(); - addrs - } - - // See if our interface addresses have changed, if so redo public dial info if necessary - async fn check_interface_addresses(&self) -> EyreResult { - if !self - .unlocked_inner - .interfaces - .refresh() - .await - .wrap_err("failed to check network interfaces")? - { - return Ok(false); - } - - let mut inner = self.inner.lock(); - let new_stable_interface_addresses = self.get_stable_interface_addresses(); - if new_stable_interface_addresses != inner.stable_interface_addresses_at_startup { - inner.network_needs_restart = true; - } - - Ok(true) - } - //////////////////////////////////////////////////////////// // Record DialInfo failures @@ -720,216 +652,155 @@ impl Network { ///////////////////////////////////////////////////////////////// pub async fn startup_internal(&self) -> EyreResult { - // initialize interfaces - self.unlocked_inner.interfaces.refresh().await?; + // Get the initial network state snapshot + // Caution: this -must- happen first because we use unwrap() in last_network_state() + let network_state = self.make_network_state().await?; - // build the set of networks we should consider for the 'LocalNetwork' routing domain - let mut local_networks: HashSet<(IpAddr, IpAddr)> = HashSet::new(); - self.unlocked_inner - .interfaces - .with_interfaces(|interfaces| { - log_net!(debug "interfaces: {:#?}", interfaces); - - for intf in interfaces.values() { - // Skip networks that we should never encounter - if intf.is_loopback() || !intf.is_running() { - continue; - } - // Add network to local networks table - for addr in &intf.addrs { - let netmask = addr.if_addr().netmask(); - let network_ip = ipaddr_apply_netmask(addr.if_addr().ip(), netmask); - local_networks.insert((network_ip, netmask)); - } - } - }); - let local_networks: Vec<(IpAddr, IpAddr)> = local_networks.into_iter().collect(); - self.unlocked_inner - .routing_table - .configure_local_network_routing_domain(local_networks); - - // determine if we have ipv4/ipv6 addresses { let mut inner = self.inner.lock(); - let stable_interface_addresses = self.get_stable_interface_addresses(); - - inner.enable_ipv4 = false; - for addr in stable_interface_addresses.iter().copied() { - if addr.is_ipv4() { - log_net!(debug "enable address {:?} as ipv4", addr); - inner.enable_ipv4 = true; - } else if addr.is_ipv6() { - let address = Address::from_ip_addr(addr); - if address.is_global() { - log_net!(debug "enable address {:?} as ipv6 global", address); - inner.enable_ipv6_global = true; - } else if address.is_local() { - log_net!(debug "enable address {:?} as ipv6 local", address); - inner.enable_ipv6_local = true; - } - } - } - inner.stable_interface_addresses_at_startup = stable_interface_addresses; - } - - // Build our protocol config to share it with other nodes - let protocol_config = { - let mut inner = self.inner.lock(); - - // Create stop source + // Create the shutdown stopper inner.stop_source = Some(StopSource::new()); - // get protocol config - let protocol_config = { - let c = self.config.get(); - let mut inbound = ProtocolTypeSet::new(); - - if c.network.protocol.udp.enabled { - inbound.insert(ProtocolType::UDP); - } - if c.network.protocol.tcp.listen { - inbound.insert(ProtocolType::TCP); - } - if c.network.protocol.ws.listen { - inbound.insert(ProtocolType::WS); - } - if c.network.protocol.wss.listen { - inbound.insert(ProtocolType::WSS); - } - - let mut outbound = ProtocolTypeSet::new(); - if c.network.protocol.udp.enabled { - outbound.insert(ProtocolType::UDP); - } - if c.network.protocol.tcp.connect { - outbound.insert(ProtocolType::TCP); - } - if c.network.protocol.ws.connect { - outbound.insert(ProtocolType::WS); - } - if c.network.protocol.wss.connect { - outbound.insert(ProtocolType::WSS); - } - - let mut family_global = AddressTypeSet::new(); - let mut family_local = AddressTypeSet::new(); - if inner.enable_ipv4 { - family_global.insert(AddressType::IPV4); - family_local.insert(AddressType::IPV4); - } - if inner.enable_ipv6_global { - family_global.insert(AddressType::IPV6); - } - if inner.enable_ipv6_local { - family_local.insert(AddressType::IPV6); - } - - // set up the routing table's network config - // if we have static public dialinfo, upgrade our network class - let public_internet_capabilities = { - PUBLIC_INTERNET_CAPABILITIES - .iter() - .copied() - .filter(|cap| !c.capabilities.disable.contains(cap)) - .collect::>() - }; - let local_network_capabilities = { - LOCAL_NETWORK_CAPABILITIES - .iter() - .copied() - .filter(|cap| !c.capabilities.disable.contains(cap)) - .collect::>() - }; - - ProtocolConfig { - outbound, - inbound, - family_global, - family_local, - public_internet_capabilities, - local_network_capabilities, - } - }; - inner.protocol_config = protocol_config.clone(); - - protocol_config - }; + // Store the first network state snapshot + inner.network_state = Some(network_state.clone()); + } // Start editing routing table let mut editor_public_internet = self .unlocked_inner .routing_table - .edit_routing_domain(RoutingDomain::PublicInternet); + .edit_public_internet_routing_domain(); let mut editor_local_network = self .unlocked_inner .routing_table - .edit_routing_domain(RoutingDomain::LocalNetwork); + .edit_local_network_routing_domain(); - // start listeners - if protocol_config.inbound.contains(ProtocolType::UDP) { - let res = self - .bind_udp_protocol_handlers(&mut editor_public_internet, &mut editor_local_network) - .await; - if !matches!(res, Ok(StartupDisposition::Success)) { - return res; - } - } - if protocol_config.inbound.contains(ProtocolType::WS) { - let res = self - .start_ws_listeners(&mut editor_public_internet, &mut editor_local_network) - .await; - if !matches!(res, Ok(StartupDisposition::Success)) { - return res; - } - } - if protocol_config.inbound.contains(ProtocolType::WSS) { - let res = self - .start_wss_listeners(&mut editor_public_internet, &mut editor_local_network) - .await; - if !matches!(res, Ok(StartupDisposition::Success)) { - return res; - } - } - if protocol_config.inbound.contains(ProtocolType::TCP) { - let res = self - .start_tcp_listeners(&mut editor_public_internet, &mut editor_local_network) - .await; - if !matches!(res, Ok(StartupDisposition::Success)) { - return res; - } - } + // Setup network + editor_local_network.set_local_networks(network_state.local_networks); + editor_local_network.setup_network( + network_state.protocol_config.outbound, + network_state.protocol_config.inbound, + network_state.protocol_config.family_local, + network_state.protocol_config.local_network_capabilities, + ); editor_public_internet.setup_network( - protocol_config.outbound, - protocol_config.inbound, - protocol_config.family_global, - protocol_config.public_internet_capabilities, - ); - editor_local_network.setup_network( - protocol_config.outbound, - protocol_config.inbound, - protocol_config.family_local, - protocol_config.local_network_capabilities, + network_state.protocol_config.outbound, + network_state.protocol_config.inbound, + network_state.protocol_config.family_global, + network_state.protocol_config.public_internet_capabilities, ); + + // Start listeners + if network_state + .protocol_config + .inbound + .contains(ProtocolType::UDP) + { + let res = self.bind_udp_protocol_handlers().await; + if !matches!(res, Ok(StartupDisposition::Success)) { + return res; + } + } + if network_state + .protocol_config + .inbound + .contains(ProtocolType::WS) + { + let res = self.start_ws_listeners().await; + if !matches!(res, Ok(StartupDisposition::Success)) { + return res; + } + } + if network_state + .protocol_config + .inbound + .contains(ProtocolType::WSS) + { + let res = self.start_wss_listeners().await; + if !matches!(res, Ok(StartupDisposition::Success)) { + return res; + } + } + if network_state + .protocol_config + .inbound + .contains(ProtocolType::TCP) + { + let res = self.start_tcp_listeners().await; + if !matches!(res, Ok(StartupDisposition::Success)) { + return res; + } + } + + // Register all dialinfo + self.register_all_dial_info(&mut editor_public_internet, &mut editor_local_network) + .await?; + + // Set network class statically if we have static public dialinfo let detect_address_changes = { let c = self.config.get(); c.network.detect_address_changes }; if !detect_address_changes { let inner = self.inner.lock(); - if !inner.static_public_dialinfo.is_empty() { + if !inner.static_public_dial_info.is_empty() { editor_public_internet.set_network_class(Some(NetworkClass::InboundCapable)); } } - // commit routing table edits - editor_public_internet.commit(true).await; - editor_local_network.commit(true).await; + // Set network class statically for local network routing domain until + // we can do some reachability analysis eventually + editor_local_network.set_network_class(Some(NetworkClass::InboundCapable)); + + // Commit routing domain edits + if editor_public_internet.commit(true).await { + editor_public_internet.publish(); + } + if editor_local_network.commit(true).await { + editor_local_network.publish(); + } Ok(StartupDisposition::Success) } + #[instrument(level = "debug", err, skip_all)] + pub(super) async fn register_all_dial_info( + &self, + editor_public_internet: &mut RoutingDomainEditorPublicInternet, + editor_local_network: &mut RoutingDomainEditorLocalNetwork, + ) -> EyreResult<()> { + let Some(protocol_config) = ({ + let inner = self.inner.lock(); + inner + .network_state + .as_ref() + .map(|ns| ns.protocol_config.clone()) + }) else { + bail!("can't register dial info without network state"); + }; + + if protocol_config.inbound.contains(ProtocolType::UDP) { + self.register_udp_dial_info(editor_public_internet, editor_local_network) + .await?; + } + if protocol_config.inbound.contains(ProtocolType::WS) { + self.register_ws_dial_info(editor_public_internet, editor_local_network) + .await?; + } + if protocol_config.inbound.contains(ProtocolType::WSS) { + self.register_wss_dial_info(editor_public_internet, editor_local_network) + .await?; + } + if protocol_config.inbound.contains(ProtocolType::TCP) { + self.register_tcp_dial_info(editor_public_internet, editor_local_network) + .await?; + } + + Ok(()) + } + #[instrument(level = "debug", err, skip_all)] pub async fn startup(&self) -> EyreResult { let guard = self.unlocked_inner.startup_lock.startup()?; @@ -994,19 +865,13 @@ impl Network { log_net!(debug "clearing dial info"); routing_table - .edit_routing_domain(RoutingDomain::PublicInternet) - .clear_dial_info_details(None, None) - .set_network_class(None) - .clear_relay_node() - .commit(true) + .edit_public_internet_routing_domain() + .shutdown() .await; routing_table - .edit_routing_domain(RoutingDomain::LocalNetwork) - .clear_dial_info_details(None, None) - .set_network_class(None) - .clear_relay_node() - .commit(true) + .edit_local_network_routing_domain() + .shutdown() .await; // Reset state including network class @@ -1051,87 +916,4 @@ impl Network { } ////////////////////////////////////////// - - #[instrument(level = "trace", target = "net", skip_all, err)] - async fn network_interfaces_task_routine( - self, - _stop_token: StopToken, - _l: u64, - _t: u64, - ) -> EyreResult<()> { - self.check_interface_addresses().await?; - - Ok(()) - } - - #[instrument(parent = None, level = "trace", target = "net", skip_all, err)] - async fn upnp_task_routine(self, _stop_token: StopToken, _l: u64, _t: u64) -> EyreResult<()> { - if !self.unlocked_inner.igd_manager.tick().await? { - info!("upnp failed, restarting local network"); - let mut inner = self.inner.lock(); - inner.network_needs_restart = true; - } - - Ok(()) - } - - #[instrument(level = "trace", target = "net", name = "Network::tick", skip_all, err)] - pub(crate) async fn tick(&self) -> EyreResult<()> { - let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else { - log_net!(debug "ignoring due to not started up"); - return Ok(()); - }; - - let (detect_address_changes, upnp) = { - let config = self.network_manager().config(); - let c = config.get(); - (c.network.detect_address_changes, c.network.upnp) - }; - - // If we need to figure out our network class, tick the task for it - if detect_address_changes { - let public_internet_network_class = self - .routing_table() - .get_network_class(RoutingDomain::PublicInternet) - .unwrap_or(NetworkClass::Invalid); - let needs_public_dial_info_check = self.needs_public_dial_info_check(); - if public_internet_network_class == NetworkClass::Invalid - || needs_public_dial_info_check - { - let routing_table = self.routing_table(); - let rth = routing_table.get_routing_table_health(); - - // We want at least two live entries per crypto kind before we start doing this (bootstrap) - let mut has_at_least_two = true; - for ck in VALID_CRYPTO_KINDS { - if rth - .live_entry_counts - .get(&(RoutingDomain::PublicInternet, ck)) - .copied() - .unwrap_or_default() - < 2 - { - has_at_least_two = false; - break; - } - } - - if has_at_least_two { - self.unlocked_inner.update_network_class_task.tick().await?; - } - } - - // Check our network interfaces to see if they have changed - if !self.needs_restart() { - self.unlocked_inner.network_interfaces_task.tick().await?; - } - } - - // If we need to tick upnp, do it - if upnp && !self.needs_restart() { - self.unlocked_inner.upnp_task.tick().await?; - } - - Ok(()) - } } diff --git a/veilid-core/src/network_manager/native/network_state.rs b/veilid-core/src/network_manager/native/network_state.rs new file mode 100644 index 00000000..7670c4ed --- /dev/null +++ b/veilid-core/src/network_manager/native/network_state.rs @@ -0,0 +1,189 @@ +use super::*; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ProtocolConfig { + pub outbound: ProtocolTypeSet, + pub inbound: ProtocolTypeSet, + pub family_global: AddressTypeSet, + pub family_local: AddressTypeSet, + pub public_internet_capabilities: Vec, + pub local_network_capabilities: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct NetworkState { + /// the calculated protocol configuration for inbound/outbound protocols + pub protocol_config: ProtocolConfig, + /// does our network have ipv4 on any network? + pub enable_ipv4: bool, + /// does our network have ipv6 on the global internet? + pub enable_ipv6_global: bool, + /// does our network have ipv6 on the local network? + pub enable_ipv6_local: bool, + /// The list of stable interface addresses we have last seen + pub stable_interface_addresses: Vec, + /// The local networks (network+mask) most recently seen + pub local_networks: Vec<(IpAddr, IpAddr)>, +} + +impl Network { + fn make_stable_interface_addresses(&self) -> Vec { + let addrs = self.unlocked_inner.interfaces.stable_addresses(); + let mut addrs: Vec = addrs + .into_iter() + .filter(|addr| { + let address = Address::from_ip_addr(*addr); + address.is_local() || address.is_global() + }) + .collect(); + addrs.sort(); + addrs.dedup(); + addrs + } + + pub(super) fn last_network_state(&self) -> NetworkState { + self.inner.lock().network_state.clone().unwrap() + } + + pub(super) fn is_stable_interface_address(&self, addr: IpAddr) -> bool { + self.inner + .lock() + .network_state + .as_ref() + .unwrap() + .stable_interface_addresses + .contains(&addr) + } + + pub(super) async fn make_network_state(&self) -> EyreResult { + // refresh network interfaces + self.unlocked_inner + .interfaces + .refresh() + .await + .wrap_err("failed to refresh network interfaces")?; + + // build the set of networks we should consider for the 'LocalNetwork' routing domain + let mut local_networks: HashSet<(IpAddr, IpAddr)> = HashSet::new(); + + self.unlocked_inner + .interfaces + .with_interfaces(|interfaces| { + for intf in interfaces.values() { + // Skip networks that we should never encounter + if intf.is_loopback() || !intf.is_running() { + continue; + } + // Add network to local networks table + for addr in &intf.addrs { + let netmask = addr.if_addr().netmask(); + let network_ip = ipaddr_apply_netmask(addr.if_addr().ip(), netmask); + local_networks.insert((network_ip, netmask)); + } + } + }); + let mut local_networks: Vec<(IpAddr, IpAddr)> = local_networks.into_iter().collect(); + local_networks.sort(); + + // determine if we have ipv4/ipv6 addresses + let mut enable_ipv4 = false; + let mut enable_ipv6_global = false; + let mut enable_ipv6_local = false; + + let stable_interface_addresses = self.make_stable_interface_addresses(); + + for addr in stable_interface_addresses.iter().copied() { + if addr.is_ipv4() { + enable_ipv4 = true; + } else if addr.is_ipv6() { + let address = Address::from_ip_addr(addr); + if address.is_global() { + enable_ipv6_global = true; + } else if address.is_local() { + enable_ipv6_local = true; + } + } + } + + // Get protocol config + let protocol_config = { + let c = self.config.get(); + let mut inbound = ProtocolTypeSet::new(); + + if c.network.protocol.udp.enabled { + inbound.insert(ProtocolType::UDP); + } + if c.network.protocol.tcp.listen { + inbound.insert(ProtocolType::TCP); + } + if c.network.protocol.ws.listen { + inbound.insert(ProtocolType::WS); + } + if c.network.protocol.wss.listen { + inbound.insert(ProtocolType::WSS); + } + + let mut outbound = ProtocolTypeSet::new(); + if c.network.protocol.udp.enabled { + outbound.insert(ProtocolType::UDP); + } + if c.network.protocol.tcp.connect { + outbound.insert(ProtocolType::TCP); + } + if c.network.protocol.ws.connect { + outbound.insert(ProtocolType::WS); + } + if c.network.protocol.wss.connect { + outbound.insert(ProtocolType::WSS); + } + + let mut family_global = AddressTypeSet::new(); + let mut family_local = AddressTypeSet::new(); + if enable_ipv4 { + family_global.insert(AddressType::IPV4); + family_local.insert(AddressType::IPV4); + } + if enable_ipv6_global { + family_global.insert(AddressType::IPV6); + } + if enable_ipv6_local { + family_local.insert(AddressType::IPV6); + } + + // set up the routing table's network config + // if we have static public dialinfo, upgrade our network class + let public_internet_capabilities = { + PUBLIC_INTERNET_CAPABILITIES + .iter() + .copied() + .filter(|cap| !c.capabilities.disable.contains(cap)) + .collect::>() + }; + let local_network_capabilities = { + LOCAL_NETWORK_CAPABILITIES + .iter() + .copied() + .filter(|cap| !c.capabilities.disable.contains(cap)) + .collect::>() + }; + + ProtocolConfig { + outbound, + inbound, + family_global, + family_local, + public_internet_capabilities, + local_network_capabilities, + } + }; + + Ok(NetworkState { + protocol_config, + enable_ipv4, + enable_ipv6_global, + enable_ipv6_local, + stable_interface_addresses, + local_networks, + }) + } +} diff --git a/veilid-core/src/network_manager/native/network_tcp.rs b/veilid-core/src/network_manager/native/network_tcp.rs index 94eee1ae..30a4b41c 100644 --- a/veilid-core/src/network_manager/native/network_tcp.rs +++ b/veilid-core/src/network_manager/native/network_tcp.rs @@ -359,10 +359,9 @@ impl Network { &self, bind_set: NetworkBindSet, is_tls: bool, + protocol_type: ProtocolType, new_protocol_accept_handler: Box, - ) -> EyreResult>> { - let mut out = Vec::::new(); - + ) -> EyreResult { for ip_addr in bind_set.addrs { let mut port = bind_set.port; loop { @@ -407,16 +406,24 @@ impl Network { } // Return interface dial infos we listen on - let idi_addrs = self.translate_unspecified_address(&addr); - for idi_addr in idi_addrs { - out.push(SocketAddress::from_socket_addr(idi_addr)); - } + let mut inner = self.inner.lock(); + let bapp = inner + .bound_address_per_protocol + .entry(protocol_type) + .or_default(); + bapp.push(addr); + + Self::set_preferred_local_address( + &mut inner, + PeerAddress::new(SocketAddress::from_socket_addr(addr), protocol_type), + ); + break; } if !bind_set.search { log_net!(debug "unable to bind to tcp {}", addr); - return Ok(None); + return Ok(false); } if port == 65535u16 { @@ -431,6 +438,6 @@ impl Network { } } - Ok(Some(out)) + Ok(true) } } diff --git a/veilid-core/src/network_manager/native/network_udp.rs b/veilid-core/src/network_manager/native/network_udp.rs index dd4865fa..575f04e1 100644 --- a/veilid-core/src/network_manager/native/network_udp.rs +++ b/veilid-core/src/network_manager/native/network_udp.rs @@ -155,9 +155,7 @@ impl Network { pub(super) async fn create_udp_protocol_handlers( &self, bind_set: NetworkBindSet, - ) -> EyreResult>> { - let mut out = Vec::::new(); - + ) -> EyreResult { for ip_addr in bind_set.addrs { let mut port = bind_set.port; loop { @@ -170,17 +168,27 @@ impl Network { // Return interface dial infos we listen on if bound { - let idi_addrs = self.translate_unspecified_address(&addr); - for idi_addr in idi_addrs { - out.push(DialInfo::udp_from_socketaddr(idi_addr)); - } + let mut inner = self.inner.lock(); + let bapp = inner + .bound_address_per_protocol + .entry(ProtocolType::UDP) + .or_default(); + bapp.push(addr); + + Self::set_preferred_local_address( + &mut inner, + PeerAddress::new( + SocketAddress::from_socket_addr(addr), + ProtocolType::UDP, + ), + ); + break; } } - if !bind_set.search { log_net!(debug "unable to bind to udp {}", addr); - return Ok(None); + return Ok(false); } if port == 65535u16 { @@ -194,7 +202,7 @@ impl Network { } } } - Ok(Some(out)) + Ok(true) } ///////////////////////////////////////////////////////////////// diff --git a/veilid-core/src/network_manager/native/start_protocols.rs b/veilid-core/src/network_manager/native/start_protocols.rs index 082c7daf..c330cd56 100644 --- a/veilid-core/src/network_manager/native/start_protocols.rs +++ b/veilid-core/src/network_manager/native/start_protocols.rs @@ -123,28 +123,23 @@ impl Network { } // Add local dial info to preferred local address table - fn add_preferred_local_address(inner: &mut NetworkInner, pa: PeerAddress) { + pub(super) fn set_preferred_local_address(inner: &mut NetworkInner, pa: PeerAddress) { let key = (pa.protocol_type(), pa.address_type()); let sa = pa.socket_addr(); - let unspec_sa = match sa { - SocketAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, a.port())), - SocketAddr::V6(a) => { - SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, a.port(), 0, 0)) - } - }; - inner.preferred_local_addresses.insert(key, unspec_sa); + // let unspec_sa = match sa { + // SocketAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, a.port())), + // SocketAddr::V6(a) => { + // SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, a.port(), 0, 0)) + // } + // }; + inner.preferred_local_addresses.entry(key).or_insert(sa); } ///////////////////////////////////////////////////// #[instrument(level = "trace", skip_all)] - pub(super) async fn bind_udp_protocol_handlers( - &self, - editor_public_internet: &mut RoutingDomainEditor, - editor_local_network: &mut RoutingDomainEditor, - ) -> EyreResult { + pub(super) async fn bind_udp_protocol_handlers(&self) -> EyreResult { log_net!("UDP: binding protocol handlers"); - let routing_table = self.routing_table(); let (listen_address, public_address, detect_address_changes) = { let c = self.config.get(); ( @@ -172,18 +167,59 @@ impl Network { ); } - let Some(mut local_dial_info_list) = self.create_udp_protocol_handlers(bind_set).await? - else { + if !self.create_udp_protocol_handlers(bind_set).await? { return Ok(StartupDisposition::BindRetry); }; - local_dial_info_list.sort(); - let mut static_public = false; + // Now create tasks for udp listeners + self.create_udp_listener_tasks().await?; - log_net!( - "UDP: protocol handlers bound to {:#?}", - local_dial_info_list - ); + { + let mut inner = self.inner.lock(); + if public_address.is_some() && !detect_address_changes { + inner.static_public_dial_info.insert(ProtocolType::UDP); + } + } + + Ok(StartupDisposition::Success) + } + + #[instrument(level = "trace", skip_all)] + pub(super) async fn register_udp_dial_info( + &self, + editor_public_internet: &mut RoutingDomainEditorPublicInternet, + editor_local_network: &mut RoutingDomainEditorLocalNetwork, + ) -> EyreResult<()> { + log_net!("UDP: registering dial info"); + + let (public_address, detect_address_changes) = { + let c = self.config.get(); + ( + c.network.protocol.udp.public_address.clone(), + c.network.detect_address_changes, + ) + }; + + let local_dial_info_list = { + let mut out = vec![]; + if let Some(bound_addresses) = { + let inner = self.inner.lock(); + inner + .bound_address_per_protocol + .get(&ProtocolType::UDP) + .cloned() + } { + for addr in bound_addresses { + let idi_addrs = self.translate_unspecified_address(addr); + for idi_addr in idi_addrs { + out.push(DialInfo::udp_from_socketaddr(idi_addr)); + } + } + } + out.sort(); + out.dedup(); + out + }; // Add static public dialinfo if it's configured if let Some(public_address) = public_address.as_ref() { @@ -197,24 +233,16 @@ impl Network { let pdi = DialInfo::udp_from_socketaddr(pdi_addr); // Register the public address - editor_public_internet.register_dial_info(pdi.clone(), DialInfoClass::Direct)?; - static_public = true; + editor_public_internet.add_dial_info(pdi.clone(), DialInfoClass::Direct); + editor_public_internet.set_network_class(Some(NetworkClass::InboundCapable)); // See if this public address is also a local interface address we haven't registered yet - let is_interface_address = (|| { - for ip_addr in self.get_stable_interface_addresses() { - if pdi_addr.ip() == ip_addr { - return true; - } - } - false - })(); - - if !local_dial_info_list.contains(&pdi) && is_interface_address { - editor_local_network.register_dial_info( + if self.is_stable_interface_address(pdi_addr.ip()) { + editor_local_network.add_dial_info( DialInfo::udp_from_socketaddr(pdi_addr), DialInfoClass::Direct, - )?; + ); + editor_local_network.set_network_class(Some(NetworkClass::InboundCapable)); } } } @@ -223,48 +251,26 @@ impl Network { for di in &local_dial_info_list { // If the local interface address is global, then register global dial info // if no other public address is specified - if !detect_address_changes - && public_address.is_none() - && routing_table.ensure_dial_info_is_valid(RoutingDomain::PublicInternet, di) - { - editor_public_internet.register_dial_info(di.clone(), DialInfoClass::Direct)?; - static_public = true; + if !detect_address_changes && public_address.is_none() && di.address().is_global() { + editor_public_internet.add_dial_info(di.clone(), DialInfoClass::Direct); + editor_public_internet.set_network_class(Some(NetworkClass::InboundCapable)); } // Register interface dial info as well since the address is on the local interface - editor_local_network.register_dial_info(di.clone(), DialInfoClass::Direct)?; + editor_local_network.add_dial_info(di.clone(), DialInfoClass::Direct); } - { - let mut inner = self.inner.lock(); - if static_public { - inner.static_public_dialinfo.insert(ProtocolType::UDP); - } - for ldi in local_dial_info_list { - Self::add_preferred_local_address(&mut inner, ldi.peer_address()); - } - } - - // Now create tasks for udp listeners - self.create_udp_listener_tasks().await?; - - Ok(StartupDisposition::Success) + Ok(()) } #[instrument(level = "trace", skip_all)] - pub(super) async fn start_ws_listeners( - &self, - editor_public_internet: &mut RoutingDomainEditor, - editor_local_network: &mut RoutingDomainEditor, - ) -> EyreResult { + pub(super) async fn start_ws_listeners(&self) -> EyreResult { log_net!("WS: binding protocol handlers"); - let routing_table = self.routing_table(); - let (listen_address, url, path, detect_address_changes) = { + let (listen_address, url, detect_address_changes) = { let c = self.config.get(); ( c.network.protocol.ws.listen_address.clone(), c.network.protocol.ws.url.clone(), - c.network.protocol.ws.path.clone(), c.network.detect_address_changes, ) }; @@ -285,21 +291,70 @@ impl Network { bind_set.port, bind_set.addrs ); } - let Some(socket_addresses) = self + if !self .start_tcp_listener( bind_set, false, + ProtocolType::WS, Box::new(|c, t| Box::new(WebsocketProtocolHandler::new(c, t))), ) .await? - else { + { return Ok(StartupDisposition::BindRetry); - }; - log_net!("WS: protocol handlers started on {:#?}", socket_addresses); + } + + { + let mut inner = self.inner.lock(); + if url.is_some() && !detect_address_changes { + inner.static_public_dial_info.insert(ProtocolType::WS); + } + } + + Ok(StartupDisposition::Success) + } + + #[instrument(level = "trace", skip_all)] + pub(super) async fn register_ws_dial_info( + &self, + editor_public_internet: &mut RoutingDomainEditorPublicInternet, + editor_local_network: &mut RoutingDomainEditorLocalNetwork, + ) -> EyreResult<()> { + log_net!("WS: registering dial info"); + let (url, path, detect_address_changes) = { + let c = self.config.get(); + ( + c.network.protocol.ws.url.clone(), + c.network.protocol.ws.path.clone(), + c.network.detect_address_changes, + ) + }; - let mut static_public = false; let mut registered_addresses: HashSet = HashSet::new(); + let socket_addresses = { + let mut out = vec![]; + if let Some(bound_addresses) = { + let inner = self.inner.lock(); + inner + .bound_address_per_protocol + .get(&ProtocolType::WS) + .cloned() + } { + for addr in bound_addresses { + for idi_addr in self + .translate_unspecified_address(addr) + .into_iter() + .map(SocketAddress::from_socket_addr) + { + out.push(idi_addr); + } + } + } + out.sort(); + out.dedup(); + out + }; + // Add static public dialinfo if it's configured if let Some(url) = url.as_ref() { let mut split_url = SplitUrl::from_str(url).wrap_err("couldn't split url")?; @@ -318,14 +373,13 @@ impl Network { let pdi = DialInfo::try_ws(SocketAddress::from_socket_addr(gsa), url.clone()) .wrap_err("try_ws failed")?; - editor_public_internet.register_dial_info(pdi.clone(), DialInfoClass::Direct)?; - static_public = true; + editor_public_internet.add_dial_info(pdi.clone(), DialInfoClass::Direct); // See if this public address is also a local interface address if !registered_addresses.contains(&gsa.ip()) && self.is_stable_interface_address(gsa.ip()) { - editor_local_network.register_dial_info(pdi, DialInfoClass::Direct)?; + editor_local_network.add_dial_info(pdi, DialInfoClass::Direct); } registered_addresses.insert(gsa.ip()); @@ -342,40 +396,23 @@ impl Network { let local_di = DialInfo::try_ws(*socket_address, local_url).wrap_err("try_ws failed")?; - if !detect_address_changes - && url.is_none() - && routing_table.ensure_dial_info_is_valid(RoutingDomain::PublicInternet, &local_di) - { + if !detect_address_changes && url.is_none() && local_di.address().is_global() { // Register public dial info - editor_public_internet - .register_dial_info(local_di.clone(), DialInfoClass::Direct)?; - static_public = true; + editor_public_internet.add_dial_info(local_di.clone(), DialInfoClass::Direct); } // Register local dial info - editor_local_network.register_dial_info(local_di, DialInfoClass::Direct)?; + editor_local_network.add_dial_info(local_di, DialInfoClass::Direct); } - let mut inner = self.inner.lock(); - if static_public { - inner.static_public_dialinfo.insert(ProtocolType::WS); - } - for sa in socket_addresses { - Self::add_preferred_local_address(&mut inner, PeerAddress::new(sa, ProtocolType::WS)); - } - - Ok(StartupDisposition::Success) + Ok(()) } #[instrument(level = "trace", skip_all)] - pub(super) async fn start_wss_listeners( - &self, - editor_public_internet: &mut RoutingDomainEditor, - editor_local_network: &mut RoutingDomainEditor, - ) -> EyreResult { + pub(super) async fn start_wss_listeners(&self) -> EyreResult { log_net!("WSS: binding protocol handlers"); - let (listen_address, url, _detect_address_changes) = { + let (listen_address, url, detect_address_changes) = { let c = self.config.get(); ( c.network.protocol.wss.listen_address.clone(), @@ -401,25 +438,49 @@ impl Network { ); } - let Some(socket_addresses) = self + if !self .start_tcp_listener( bind_set, true, + ProtocolType::WSS, Box::new(|c, t| Box::new(WebsocketProtocolHandler::new(c, t))), ) .await? - else { + { return Ok(StartupDisposition::BindRetry); - }; + } - log_net!("WSS: protocol handlers started on {:#?}", socket_addresses); + { + let mut inner = self.inner.lock(); + if url.is_some() && !detect_address_changes { + inner.static_public_dial_info.insert(ProtocolType::WSS); + } + } + + Ok(StartupDisposition::Success) + } + + #[instrument(level = "trace", skip_all)] + pub(super) async fn register_wss_dial_info( + &self, + editor_public_internet: &mut RoutingDomainEditorPublicInternet, + editor_local_network: &mut RoutingDomainEditorLocalNetwork, + ) -> EyreResult<()> { + log_net!("WSS: registering dialinfo"); + + let (url, _detect_address_changes) = { + let c = self.config.get(); + ( + c.network.protocol.wss.url.clone(), + c.network.detect_address_changes, + ) + }; // NOTE: No interface dial info for WSS, as there is no way to connect to a local dialinfo via TLS // If the hostname is specified, it is the public dialinfo via the URL. If no hostname // is specified, then TLS won't validate, so no local dialinfo is possible. // This is not the case with unencrypted websockets, which can be specified solely by an IP address - let mut static_public = false; let mut registered_addresses: HashSet = HashSet::new(); // Add static public dialinfo if it's configured @@ -440,14 +501,13 @@ impl Network { let pdi = DialInfo::try_wss(SocketAddress::from_socket_addr(gsa), url.clone()) .wrap_err("try_wss failed")?; - editor_public_internet.register_dial_info(pdi.clone(), DialInfoClass::Direct)?; - static_public = true; + editor_public_internet.add_dial_info(pdi.clone(), DialInfoClass::Direct); // See if this public address is also a local interface address if !registered_addresses.contains(&gsa.ip()) && self.is_stable_interface_address(gsa.ip()) { - editor_local_network.register_dial_info(pdi, DialInfoClass::Direct)?; + editor_local_network.add_dial_info(pdi, DialInfoClass::Direct); } registered_addresses.insert(gsa.ip()); @@ -456,26 +516,13 @@ impl Network { bail!("WSS URL must be specified due to TLS requirements"); } - let mut inner = self.inner.lock(); - if static_public { - inner.static_public_dialinfo.insert(ProtocolType::WSS); - } - for sa in socket_addresses { - Self::add_preferred_local_address(&mut inner, PeerAddress::new(sa, ProtocolType::WSS)); - } - - Ok(StartupDisposition::Success) + Ok(()) } #[instrument(level = "trace", skip_all)] - pub(super) async fn start_tcp_listeners( - &self, - editor_public_internet: &mut RoutingDomainEditor, - editor_local_network: &mut RoutingDomainEditor, - ) -> EyreResult { + pub(super) async fn start_tcp_listeners(&self) -> EyreResult { log_net!("TCP: binding protocol handlers"); - let routing_table = self.routing_table(); let (listen_address, public_address, detect_address_changes) = { let c = self.config.get(); ( @@ -501,22 +548,70 @@ impl Network { bind_set.port, bind_set.addrs ); } - let Some(socket_addresses) = self + if !self .start_tcp_listener( bind_set, false, + ProtocolType::TCP, Box::new(|c, _| Box::new(RawTcpProtocolHandler::new(c))), ) .await? - else { + { return Ok(StartupDisposition::BindRetry); + } + + { + let mut inner = self.inner.lock(); + if public_address.is_some() && !detect_address_changes { + inner.static_public_dial_info.insert(ProtocolType::TCP); + } + } + + Ok(StartupDisposition::Success) + } + + #[instrument(level = "trace", skip_all)] + pub(super) async fn register_tcp_dial_info( + &self, + editor_public_internet: &mut RoutingDomainEditorPublicInternet, + editor_local_network: &mut RoutingDomainEditorLocalNetwork, + ) -> EyreResult<()> { + log_net!("TCP: registering dialinfo"); + + let (public_address, detect_address_changes) = { + let c = self.config.get(); + ( + c.network.protocol.tcp.public_address.clone(), + c.network.detect_address_changes, + ) }; - log_net!("TCP: protocol handlers started on {:#?}", socket_addresses); - - let mut static_public = false; let mut registered_addresses: HashSet = HashSet::new(); + let socket_addresses = { + let mut out = vec![]; + if let Some(bound_addresses) = { + let inner = self.inner.lock(); + inner + .bound_address_per_protocol + .get(&ProtocolType::TCP) + .cloned() + } { + for addr in bound_addresses { + for idi_addr in self + .translate_unspecified_address(addr) + .into_iter() + .map(SocketAddress::from_socket_addr) + { + out.push(idi_addr); + } + } + } + out.sort(); + out.dedup(); + out + }; + // Add static public dialinfo if it's configured if let Some(public_address) = public_address.as_ref() { // Resolve statically configured public dialinfo @@ -532,12 +627,11 @@ impl Network { } let pdi = DialInfo::tcp_from_socketaddr(pdi_addr); - editor_public_internet.register_dial_info(pdi.clone(), DialInfoClass::Direct)?; - static_public = true; + editor_public_internet.add_dial_info(pdi.clone(), DialInfoClass::Direct); // See if this public address is also a local interface address if self.is_stable_interface_address(pdi_addr.ip()) { - editor_local_network.register_dial_info(pdi, DialInfoClass::Direct)?; + editor_local_network.add_dial_info(pdi, DialInfoClass::Direct); } } } @@ -546,27 +640,14 @@ impl Network { let di = DialInfo::tcp(*socket_address); // Register global dial info if no public address is specified - if !detect_address_changes - && public_address.is_none() - && routing_table.ensure_dial_info_is_valid(RoutingDomain::PublicInternet, &di) - { - editor_public_internet.register_dial_info(di.clone(), DialInfoClass::Direct)?; - static_public = true; + if !detect_address_changes && public_address.is_none() && di.address().is_global() { + editor_public_internet.add_dial_info(di.clone(), DialInfoClass::Direct); } // Register interface dial info - editor_local_network.register_dial_info(di.clone(), DialInfoClass::Direct)?; + editor_local_network.add_dial_info(di.clone(), DialInfoClass::Direct); registered_addresses.insert(socket_address.ip_addr()); } - let mut inner = self.inner.lock(); - - if static_public { - inner.static_public_dialinfo.insert(ProtocolType::TCP); - } - for sa in socket_addresses { - Self::add_preferred_local_address(&mut inner, PeerAddress::new(sa, ProtocolType::TCP)); - } - - Ok(StartupDisposition::Success) + Ok(()) } } diff --git a/veilid-core/src/network_manager/native/tasks/mod.rs b/veilid-core/src/network_manager/native/tasks/mod.rs new file mode 100644 index 00000000..b437ea44 --- /dev/null +++ b/veilid-core/src/network_manager/native/tasks/mod.rs @@ -0,0 +1,110 @@ +mod network_interfaces_task; +mod update_network_class_task; +mod upnp_task; + +use super::*; + +impl Network { + pub(crate) fn setup_tasks(&self) { + // Set update network class tick task + { + let this = self.clone(); + self.unlocked_inner + .update_network_class_task + .set_routine(move |s, l, t| { + Box::pin(this.clone().update_network_class_task_routine( + s, + Timestamp::new(l), + Timestamp::new(t), + )) + }); + } + // Set network interfaces tick task + { + let this = self.clone(); + self.unlocked_inner + .network_interfaces_task + .set_routine(move |s, l, t| { + Box::pin(this.clone().network_interfaces_task_routine( + s, + Timestamp::new(l), + Timestamp::new(t), + )) + }); + } + // Set upnp tick task + { + let this = self.clone(); + self.unlocked_inner.upnp_task.set_routine(move |s, l, t| { + Box::pin( + this.clone() + .upnp_task_routine(s, Timestamp::new(l), Timestamp::new(t)), + ) + }); + } + } + + #[instrument(level = "trace", target = "net", name = "Network::tick", skip_all, err)] + pub(crate) async fn tick(&self) -> EyreResult<()> { + let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else { + log_net!(debug "ignoring due to not started up"); + return Ok(()); + }; + + // Ignore this tick if we need to restart + if self.needs_restart() { + return Ok(()); + } + + let (detect_address_changes, upnp) = { + let config = self.network_manager().config(); + let c = config.get(); + (c.network.detect_address_changes, c.network.upnp) + }; + + // If we need to figure out our network class, tick the task for it + if detect_address_changes { + // Check our network interfaces to see if they have changed + self.unlocked_inner.network_interfaces_task.tick().await?; + + // Check our public dial info to see if it has changed + let public_internet_network_class = self + .routing_table() + .get_network_class(RoutingDomain::PublicInternet) + .unwrap_or(NetworkClass::Invalid); + let needs_public_dial_info_check = self.needs_public_dial_info_check(); + if public_internet_network_class == NetworkClass::Invalid + || needs_public_dial_info_check + { + let routing_table = self.routing_table(); + let rth = routing_table.get_routing_table_health(); + + // We want at least two live entries per crypto kind before we start doing this (bootstrap) + let mut has_at_least_two = true; + for ck in VALID_CRYPTO_KINDS { + if rth + .live_entry_counts + .get(&(RoutingDomain::PublicInternet, ck)) + .copied() + .unwrap_or_default() + < 2 + { + has_at_least_two = false; + break; + } + } + + if has_at_least_two { + self.unlocked_inner.update_network_class_task.tick().await?; + } + } + } + + // If we need to tick upnp, do it + if upnp { + self.unlocked_inner.upnp_task.tick().await?; + } + + Ok(()) + } +} diff --git a/veilid-core/src/network_manager/native/tasks/network_interfaces_task.rs b/veilid-core/src/network_manager/native/tasks/network_interfaces_task.rs new file mode 100644 index 00000000..69199683 --- /dev/null +++ b/veilid-core/src/network_manager/native/tasks/network_interfaces_task.rs @@ -0,0 +1,73 @@ +use super::*; + +impl Network { + #[instrument(level = "trace", target = "net", skip_all, err)] + pub(super) async fn network_interfaces_task_routine( + self, + _stop_token: StopToken, + _l: Timestamp, + _t: Timestamp, + ) -> EyreResult<()> { + let _guard = self.unlocked_inner.network_task_lock.lock().await; + + self.update_network_state().await?; + + Ok(()) + } + + // See if our interface addresses have changed, if so redo public dial info if necessary + async fn update_network_state(&self) -> EyreResult { + let mut local_network_changed = false; + let mut public_internet_changed = false; + + let last_network_state = self.last_network_state(); + let new_network_state = match self.make_network_state().await { + Ok(v) => v, + Err(e) => { + log_net!(debug "Skipping network state update: {}", e); + return Ok(false); + } + }; + + if new_network_state != last_network_state { + // Save new network state + { + let mut inner = self.inner.lock(); + inner.network_state = Some(new_network_state.clone()); + } + + // network state has changed + let mut editor_local_network = self + .unlocked_inner + .routing_table + .edit_local_network_routing_domain(); + editor_local_network.set_local_networks(new_network_state.local_networks); + editor_local_network.clear_dial_info_details(None, None); + + let mut editor_public_internet = self + .unlocked_inner + .routing_table + .edit_public_internet_routing_domain(); + + // Update protocols + self.register_all_dial_info(&mut editor_public_internet, &mut editor_local_network) + .await?; + + local_network_changed = editor_local_network.commit(true).await; + public_internet_changed = editor_public_internet.commit(true).await; + + // Update local network now + if local_network_changed { + editor_local_network.publish(); + } + } + + // If any of the new addresses were PublicInternet addresses, re-run public dial info check + if public_internet_changed { + // inner.network_needs_restart = true; + self.set_needs_public_dial_info_check(None); + } + + Ok(local_network_changed || public_internet_changed) + } +} diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/tasks/update_network_class_task.rs similarity index 91% rename from veilid-core/src/network_manager/native/network_class_discovery.rs rename to veilid-core/src/network_manager/native/tasks/update_network_class_task.rs index bf6a9c16..d1b6dc6f 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/tasks/update_network_class_task.rs @@ -4,6 +4,28 @@ use futures_util::stream::FuturesUnordered; use stop_token::future::FutureExt as StopTokenFutureExt; impl Network { + #[instrument(parent = None, level = "trace", skip(self), err)] + pub async fn update_network_class_task_routine( + self, + stop_token: StopToken, + l: Timestamp, + t: Timestamp, + ) -> EyreResult<()> { + let _guard = self.unlocked_inner.network_task_lock.lock().await; + + // Do the public dial info check + let out = self.do_public_dial_info_check(stop_token, l, t).await; + + // Done with public dial info check + { + let mut inner = self.inner.lock(); + inner.needs_public_dial_info_check = false; + inner.public_dial_info_check_punishment = None; + } + + out + } + #[instrument(level = "trace", skip(self), err)] pub async fn update_with_detected_dial_info(&self, ddi: DetectedDialInfo) -> EyreResult<()> { let existing_network_class = self @@ -16,9 +38,7 @@ impl Network { // If we get any symmetric nat dialinfo, this whole network class is outbound only, // and all dial info should be treated as invalid if !matches!(existing_network_class, NetworkClass::OutboundOnly) { - let mut editor = self - .routing_table() - .edit_routing_domain(RoutingDomain::PublicInternet); + let mut editor = self.routing_table().edit_public_internet_routing_domain(); editor.clear_dial_info_details(None, None); editor.set_network_class(Some(NetworkClass::OutboundOnly)); @@ -67,9 +87,7 @@ impl Network { } if clear || add { - let mut editor = self - .routing_table() - .edit_routing_domain(RoutingDomain::PublicInternet); + let mut editor = self.routing_table().edit_public_internet_routing_domain(); if clear { editor.clear_dial_info_details( @@ -79,11 +97,7 @@ impl Network { } if add { - if let Err(e) = - editor.register_dial_info(did.dial_info.clone(), did.class) - { - log_net!(debug "Failed to register detected dialinfo {:?}: {}", did, e); - } + editor.add_dial_info(did.dial_info.clone(), did.class); } editor.set_network_class(Some(NetworkClass::InboundCapable)); @@ -99,13 +113,19 @@ impl Network { pub async fn do_public_dial_info_check( &self, stop_token: StopToken, - _l: u64, - _t: u64, + _l: Timestamp, + _t: Timestamp, ) -> EyreResult<()> { // Figure out if we can optimize TCP/WS checking since they are often on the same port let (protocol_config, inbound_protocol_map) = { let mut inner = self.inner.lock(); - let protocol_config = inner.protocol_config.clone(); + let Some(protocol_config) = inner + .network_state + .as_ref() + .map(|ns| ns.protocol_config.clone()) + else { + bail!("should not be doing public dial info check before we have an initial network state"); + }; // Allow network to be cleared if external addresses change inner.network_already_cleared = false; @@ -118,7 +138,7 @@ impl Network { // Skip things with static public dialinfo // as they don't need to participate in discovery - if inner.static_public_dialinfo.contains(pt) { + if inner.static_public_dial_info.contains(pt) { continue; } @@ -147,9 +167,7 @@ impl Network { .collect(); // Set most permissive network config - let mut editor = self - .routing_table() - .edit_routing_domain(RoutingDomain::PublicInternet); + let mut editor = self.routing_table().edit_public_internet_routing_domain(); editor.setup_network( protocol_config.outbound, protocol_config.inbound, @@ -171,9 +189,7 @@ impl Network { } inner.network_already_cleared = true; } - let mut editor = this - .routing_table() - .edit_routing_domain(RoutingDomain::PublicInternet); + let mut editor = this.routing_table().edit_public_internet_routing_domain(); editor.clear_dial_info_details(None, None); editor.set_network_class(None); editor.commit(true).await; @@ -261,7 +277,9 @@ impl Network { all_address_types, protocol_config.public_internet_capabilities, ); - editor.commit(true).await; + if editor.commit(true).await { + editor.publish(); + } // See if the dial info changed let new_public_dial_info: HashSet = self @@ -282,25 +300,6 @@ impl Network { Ok(()) } - #[instrument(parent = None, level = "trace", skip(self), err)] - pub async fn update_network_class_task_routine( - self, - stop_token: StopToken, - l: u64, - t: u64, - ) -> EyreResult<()> { - // Do the public dial info check - let out = self.do_public_dial_info_check(stop_token, l, t).await; - - // Done with public dial info check - { - let mut inner = self.inner.lock(); - inner.needs_public_dial_info_check = false; - inner.public_dial_info_check_punishment = None; - } - - out - } /// Make a dialinfo from an address and protocol type pub fn make_dial_info(&self, addr: SocketAddress, protocol_type: ProtocolType) -> DialInfo { diff --git a/veilid-core/src/network_manager/native/tasks/upnp_task.rs b/veilid-core/src/network_manager/native/tasks/upnp_task.rs new file mode 100644 index 00000000..0452705a --- /dev/null +++ b/veilid-core/src/network_manager/native/tasks/upnp_task.rs @@ -0,0 +1,19 @@ +use super::*; + +impl Network { + #[instrument(parent = None, level = "trace", target = "net", skip_all, err)] + pub(super) async fn upnp_task_routine( + self, + _stop_token: StopToken, + _l: Timestamp, + _t: Timestamp, + ) -> EyreResult<()> { + if !self.unlocked_inner.igd_manager.tick().await? { + info!("upnp failed, restarting local network"); + let mut inner = self.inner.lock(); + inner.network_needs_restart = true; + } + + Ok(()) + } +} diff --git a/veilid-core/src/network_manager/network_connection.rs b/veilid-core/src/network_manager/network_connection.rs index a236706e..cf2279f4 100644 --- a/veilid-core/src/network_manager/network_connection.rs +++ b/veilid-core/src/network_manager/network_connection.rs @@ -183,7 +183,7 @@ impl NetworkConnection { self.flow } - #[allow(dead_code)] + #[expect(dead_code)] pub fn unique_flow(&self) -> UniqueFlow { UniqueFlow { flow: self.flow, @@ -207,6 +207,10 @@ impl NetworkConnection { self.protected_nr = Some(protect_nr); } + pub fn unprotect(&mut self) { + self.protected_nr = None; + } + pub fn add_ref(&mut self) { self.ref_count += 1; } @@ -254,13 +258,12 @@ impl NetworkConnection { Ok(NetworkResult::Value(out)) } - #[allow(dead_code)] pub fn stats(&self) -> NetworkConnectionStats { let stats = self.stats.lock(); stats.clone() } - #[allow(dead_code)] + #[expect(dead_code)] pub fn established_time(&self) -> Timestamp { self.established_time } diff --git a/veilid-core/src/network_manager/receipt_manager.rs b/veilid-core/src/network_manager/receipt_manager.rs index 6b961312..149f61f4 100644 --- a/veilid-core/src/network_manager/receipt_manager.rs +++ b/veilid-core/src/network_manager/receipt_manager.rs @@ -7,12 +7,16 @@ use routing_table::*; use stop_token::future::FutureExt; #[derive(Clone, Debug)] -#[allow(dead_code)] pub(crate) enum ReceiptEvent { ReturnedOutOfBand, - ReturnedInBand { inbound_noderef: NodeRef }, + ReturnedInBand { + inbound_noderef: FilteredNodeRef, + }, ReturnedSafety, - ReturnedPrivate { private_route: PublicKey }, + ReturnedPrivate { + #[expect(dead_code)] + private_route: PublicKey, + }, Expired, Cancelled, } @@ -20,7 +24,7 @@ pub(crate) enum ReceiptEvent { #[derive(Clone, Debug)] pub(super) enum ReceiptReturned { OutOfBand, - InBand { inbound_noderef: NodeRef }, + InBand { inbound_noderef: FilteredNodeRef }, Safety, Private { private_route: PublicKey }, } @@ -53,7 +57,6 @@ where type ReceiptCallbackType = Box; type ReceiptSingleShotType = SingleShotEventual; -#[allow(dead_code)] enum ReceiptRecordCallbackType { Normal(ReceiptCallbackType), SingleShot(Option), @@ -92,7 +95,7 @@ impl fmt::Debug for ReceiptRecord { } impl ReceiptRecord { - #[allow(dead_code)] + #[expect(dead_code)] pub fn new( receipt: Receipt, expiration_ts: Timestamp, @@ -352,7 +355,6 @@ impl ReceiptManager { log_net!(debug "finished receipt manager shutdown"); } - #[allow(dead_code)] #[instrument(level = "trace", target = "receipt", skip_all)] pub fn record_receipt( &self, @@ -418,7 +420,7 @@ impl ReceiptManager { inner.next_oldest_ts = new_next_oldest_ts; } - #[allow(dead_code)] + #[expect(dead_code)] pub async fn cancel_receipt(&self, nonce: &Nonce) -> EyreResult<()> { event!(target: "receipt", Level::DEBUG, "== Cancel Receipt {}", nonce.encode()); diff --git a/veilid-core/src/network_manager/send_data.rs b/veilid-core/src/network_manager/send_data.rs index a1a421c4..c06cfa06 100644 --- a/veilid-core/src/network_manager/send_data.rs +++ b/veilid-core/src/network_manager/send_data.rs @@ -15,7 +15,7 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] pub(crate) async fn send_data( &self, - destination_node_ref: NodeRef, + destination_node_ref: FilteredNodeRef, data: Vec, ) -> EyreResult> { // First try to send data to the last flow we've seen this peer on @@ -81,7 +81,7 @@ impl NetworkManager { pub(crate) fn try_possibly_relayed_contact_method( &self, possibly_relayed_contact_method: NodeContactMethod, - destination_node_ref: NodeRef, + destination_node_ref: FilteredNodeRef, data: Vec, ) -> SendPinBoxFuture>> { let this = self.clone(); @@ -179,7 +179,7 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] async fn send_data_ncm_existing( &self, - target_node_ref: NodeRef, + target_node_ref: FilteredNodeRef, data: Vec, ) -> EyreResult> { // First try to send data to the last connection we've seen this peer on @@ -213,7 +213,7 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] async fn send_data_ncm_unreachable( &self, - target_node_ref: NodeRef, + target_node_ref: FilteredNodeRef, data: Vec, ) -> EyreResult> { // Try to send data to the last socket we've seen this peer on @@ -248,8 +248,8 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] async fn send_data_ncm_signal_reverse( &self, - relay_nr: NodeRef, - target_node_ref: NodeRef, + relay_nr: FilteredNodeRef, + target_node_ref: FilteredNodeRef, data: Vec, ) -> EyreResult> { // First try to send data to the last socket we've seen this peer on @@ -291,8 +291,8 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] async fn send_data_ncm_signal_hole_punch( &self, - relay_nr: NodeRef, - target_node_ref: NodeRef, + relay_nr: FilteredNodeRef, + target_node_ref: FilteredNodeRef, data: Vec, ) -> EyreResult> { // First try to send data to the last socket we've seen this peer on @@ -334,7 +334,7 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] async fn send_data_ncm_direct( &self, - node_ref: NodeRef, + node_ref: FilteredNodeRef, dial_info: DialInfo, data: Vec, ) -> EyreResult> { @@ -362,7 +362,7 @@ impl NetworkManager { } SendDataToExistingFlowResult::NotSent(d) => { // Connection couldn't send, kill it - node_ref.clear_last_connection(flow); + node_ref.clear_last_flow(flow); d } } @@ -393,7 +393,7 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] pub(crate) fn get_node_contact_method( &self, - target_node_ref: NodeRef, + target_node_ref: FilteredNodeRef, ) -> EyreResult { let routing_table = self.routing_table(); @@ -415,31 +415,31 @@ impl NetworkManager { } }; - // Get cache key - let ncm_key = NodeContactMethodCacheKey { - node_ids: target_node_ref.node_ids(), - own_node_info_ts: routing_table.get_own_node_info_ts(routing_domain), - target_node_info_ts: target_node_ref.node_info_ts(routing_domain), - target_node_ref_filter: target_node_ref.filter_ref().cloned(), - target_node_ref_sequencing: target_node_ref.sequencing(), - }; - if let Some(ncm) = self.inner.lock().node_contact_method_cache.get(&ncm_key) { - return Ok(ncm.clone()); - } - // Node A is our own node // Use whatever node info we've calculated so far - let peer_a = routing_table.get_own_peer_info(routing_domain); + let peer_a = routing_table.get_current_peer_info(routing_domain); // Node B is the target node let peer_b = match target_node_ref.make_peer_info(routing_domain) { - Some(ni) => ni, + Some(pi) => Arc::new(pi), None => { log_net!("no node info for node {:?}", target_node_ref); return Ok(NodeContactMethod::Unreachable); } }; + // Get cache key + let ncm_key = NodeContactMethodCacheKey { + node_ids: target_node_ref.node_ids(), + own_node_info_ts: peer_a.signed_node_info().timestamp(), + target_node_info_ts: peer_b.signed_node_info().timestamp(), + target_node_ref_filter: target_node_ref.filter(), + target_node_ref_sequencing: target_node_ref.sequencing(), + }; + if let Some(ncm) = self.inner.lock().node_contact_method_cache.get(&ncm_key) { + return Ok(ncm.clone()); + } + // Dial info filter comes from the target node ref but must be filtered by this node's outbound capabilities let dial_info_filter = target_node_ref.dial_info_filter().filtered( &DialInfoFilter::all() @@ -463,7 +463,7 @@ impl NetworkManager { for did in peer_b .signed_node_info() .node_info() - .all_filtered_dial_info_details(DialInfoDetail::NO_SORT, |_| true) + .filtered_dial_info_details(DialInfoDetail::NO_SORT, |_| true) { if let Some(ts) = address_filter.get_dial_info_failed_ts(&did.dial_info) { dial_info_failures_map.insert(did.dial_info, ts); @@ -488,8 +488,8 @@ impl NetworkManager { // Get the best contact method with these parameters from the routing domain let cm = routing_table.get_contact_method( routing_domain, - &peer_a, - &peer_b, + peer_a.clone(), + peer_b.clone(), dial_info_filter, sequencing, dif_sort, @@ -585,8 +585,8 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] async fn do_reverse_connect( &self, - relay_nr: NodeRef, - target_nr: NodeRef, + relay_nr: FilteredNodeRef, + target_nr: FilteredNodeRef, data: Vec, ) -> EyreResult> { // Detect if network is stopping so we can break out of this @@ -611,22 +611,24 @@ impl NetworkManager { )); }; - // Ensure we have a valid network class so our peer info is useful - if !self.routing_table().has_valid_network_class(routing_domain) { + // Get our published peer info + let Some(published_peer_info) = + self.routing_table().get_published_peer_info(routing_domain) + else { return Ok(NetworkResult::no_connection_other( "Network class not yet valid for reverse connect", )); }; - // Get our peer info - let peer_info = self.routing_table().get_own_peer_info(routing_domain); - // Issue the signal let rpc = self.rpc_processor(); network_result_try!(rpc .rpc_call_signal( - Destination::relay(relay_nr.clone(), target_nr.clone()), - SignalInfo::ReverseConnect { receipt, peer_info }, + Destination::relay(relay_nr.clone(), target_nr.unfiltered()), + SignalInfo::ReverseConnect { + receipt, + peer_info: published_peer_info + }, ) .await .wrap_err("failed to send signal")?); @@ -694,8 +696,8 @@ impl NetworkManager { #[instrument(level = "trace", target = "net", skip_all, err)] async fn do_hole_punch( &self, - relay_nr: NodeRef, - target_nr: NodeRef, + relay_nr: FilteredNodeRef, + target_nr: FilteredNodeRef, data: Vec, ) -> EyreResult> { // Detect if network is stopping so we can break out of this @@ -703,12 +705,12 @@ impl NetworkManager { return Ok(NetworkResult::service_unavailable("network is stopping")); }; - // Ensure we are filtered down to UDP (the only hole punch protocol supported today) - assert!(target_nr - .filter_ref() - .map(|nrf| nrf.dial_info_filter.protocol_type_set - == ProtocolTypeSet::only(ProtocolType::UDP)) - .unwrap_or_default()); + // Ensure target is filtered down to UDP (the only hole punch protocol supported today) + // Relay can be any protocol because the signal rpc contains the dialinfo to connect over + assert_eq!( + target_nr.dial_info_filter().protocol_type_set, + ProtocolType::UDP + ); // Build a return receipt for the signal let receipt_timeout = TimestampDuration::new_ms( @@ -727,19 +729,18 @@ impl NetworkManager { )); }; - // Ensure we have a valid network class so our peer info is useful - if !self.routing_table().has_valid_network_class(routing_domain) { + // Get our published peer info + let Some(published_peer_info) = + self.routing_table().get_published_peer_info(routing_domain) + else { return Ok(NetworkResult::no_connection_other( "Network class not yet valid for hole punch", )); }; - // Get our peer info - let peer_info = self.routing_table().get_own_peer_info(routing_domain); - // Get the udp direct dialinfo for the hole punch let hole_punch_did = target_nr - .first_filtered_dial_info_detail() + .first_dial_info_detail() .ok_or_else(|| eyre!("No hole punch capable dialinfo found for node"))?; // Do our half of the hole punch by sending an empty packet @@ -756,8 +757,11 @@ impl NetworkManager { let rpc = self.rpc_processor(); network_result_try!(rpc .rpc_call_signal( - Destination::relay(relay_nr, target_nr.clone()), - SignalInfo::HolePunch { receipt, peer_info }, + Destination::relay(relay_nr, target_nr.unfiltered()), + SignalInfo::HolePunch { + receipt, + peer_info: published_peer_info + }, ) .await .wrap_err("failed to send signal")?); diff --git a/veilid-core/src/network_manager/stats.rs b/veilid-core/src/network_manager/stats.rs index 589add0f..b19dd2a6 100644 --- a/veilid-core/src/network_manager/stats.rs +++ b/veilid-core/src/network_manager/stats.rs @@ -69,7 +69,7 @@ impl NetworkManager { .add_down(bytes); } - #[allow(dead_code)] + #[expect(dead_code)] pub fn get_stats(&self) -> NetworkManagerStats { let inner = self.inner.lock(); inner.stats.clone() diff --git a/veilid-core/src/network_manager/tasks/local_network_address_check.rs b/veilid-core/src/network_manager/tasks/local_network_address_check.rs new file mode 100644 index 00000000..72428f2b --- /dev/null +++ b/veilid-core/src/network_manager/tasks/local_network_address_check.rs @@ -0,0 +1,15 @@ +use super::*; + +impl NetworkManager { + // Determine if a local IP address has changed + // this means we should restart the low level network and and recreate all of our dial info + // Wait until we have received confirmation from N different peers + pub fn report_local_network_socket_address( + &self, + _socket_address: SocketAddress, + _flow: Flow, + _reporting_peer: NodeRef, + ) { + // XXX: Nothing here yet. + } +} diff --git a/veilid-core/src/network_manager/tasks/mod.rs b/veilid-core/src/network_manager/tasks/mod.rs index e663204d..3a68cd4f 100644 --- a/veilid-core/src/network_manager/tasks/mod.rs +++ b/veilid-core/src/network_manager/tasks/mod.rs @@ -1,4 +1,5 @@ -pub mod public_address_check; +pub mod local_network_address_check; +pub mod public_internet_address_check; pub mod rolling_transfers; use super::*; @@ -19,13 +20,13 @@ impl NetworkManager { }); } - // Set public address check task + // Set public internet address check task { let this = self.clone(); self.unlocked_inner - .public_address_check_task + .public_internet_address_check_task .set_routine(move |s, l, t| { - Box::pin(this.clone().public_address_check_task_routine( + Box::pin(this.clone().public_internet_address_check_task_routine( s, Timestamp::new(l), Timestamp::new(t), diff --git a/veilid-core/src/network_manager/tasks/public_address_check.rs b/veilid-core/src/network_manager/tasks/public_internet_address_check.rs similarity index 69% rename from veilid-core/src/network_manager/tasks/public_address_check.rs rename to veilid-core/src/network_manager/tasks/public_internet_address_check.rs index 12f97994..5a7747c9 100644 --- a/veilid-core/src/network_manager/tasks/public_address_check.rs +++ b/veilid-core/src/network_manager/tasks/public_internet_address_check.rs @@ -3,7 +3,7 @@ use super::*; impl NetworkManager { // Clean up the public address check tables, removing entries that have timed out #[instrument(parent = None, level = "trace", skip_all, err)] - pub(crate) async fn public_address_check_task_routine( + pub(crate) async fn public_internet_address_check_task_routine( self, _stop_token: StopToken, _last_ts: Timestamp, @@ -11,32 +11,17 @@ impl NetworkManager { ) -> EyreResult<()> { // go through public_address_inconsistencies_table and time out things that have expired let mut inner = self.inner.lock(); - for pait_v in inner.public_address_inconsistencies_table.values_mut() { - let mut expired = Vec::new(); - for (addr, exp_ts) in pait_v.iter() { - if *exp_ts <= cur_ts { - expired.push(*addr); - } - } - for exp in expired { - pait_v.remove(&exp); - } + for pait_v in inner + .public_internet_address_inconsistencies_table + .values_mut() + { + pait_v.retain(|_addr, exp_ts| { + // Keep it if it's in the future + *exp_ts > cur_ts + }); } Ok(()) } - - // Determine if a local IP address has changed - // this means we should restart the low level network and and recreate all of our dial info - // Wait until we have received confirmation from N different peers - pub fn report_local_network_socket_address( - &self, - _socket_address: SocketAddress, - _flow: Flow, - _reporting_peer: NodeRef, - ) { - // XXX: Nothing here yet. - } - // Determine if a global IP address has changed // this means we should recreate our public dial info if it is not static and rediscover it // Wait until we have received confirmation from N different peers @@ -46,7 +31,7 @@ impl NetworkManager { flow: Flow, // the flow used reporting_peer: NodeRef, // the peer's noderef reporting the socket address ) { - log_network_result!("report_global_socket_address\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer); + log_network_result!("report_public_internet_socket_address:\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer); // Ignore these reports if we are currently detecting public dial info let net = self.net(); @@ -69,17 +54,22 @@ impl NetworkManager { return; } + // Get our current published peer info + let routing_table = self.routing_table(); + let Some(published_peer_info) = + routing_table.get_published_peer_info(RoutingDomain::PublicInternet) + else { + return; + }; + // If we are a webapp we should skip this completely // because we will never get inbound dialinfo directly on our public ip address // If we have an invalid network class, this is not necessary yet - let routing_table = self.routing_table(); - let public_internet_network_class = routing_table - .get_network_class(RoutingDomain::PublicInternet) - .unwrap_or(NetworkClass::Invalid); - if matches!( - public_internet_network_class, - NetworkClass::Invalid | NetworkClass::WebApp - ) { + let public_internet_network_class = published_peer_info + .signed_node_info() + .node_info() + .network_class(); + if matches!(public_internet_network_class, NetworkClass::WebApp) { return; } @@ -119,7 +109,7 @@ impl NetworkManager { let addr_proto_type_key = PublicAddressCheckCacheKey(flow.protocol_type(), flow.address_type()); if inner - .public_address_inconsistencies_table + .public_internet_address_inconsistencies_table .get(&addr_proto_type_key) .map(|pait| pait.contains_key(&reporting_ipblock)) .unwrap_or(false) @@ -130,17 +120,17 @@ impl NetworkManager { // Insert this new public address into the lru cache for the address check // if we've seen this address before, it brings it to the front let pacc = inner - .public_address_check_cache + .public_internet_address_check_cache .entry(addr_proto_type_key) .or_insert_with(|| LruCache::new(PUBLIC_ADDRESS_CHECK_CACHE_SIZE)); pacc.insert(reporting_ipblock, socket_address); // Determine if our external address has likely changed - let mut bad_public_address_detection_punishment: Option< + let mut bad_public_internet_address_detection_punishment: Option< Box, > = None; - let needs_public_address_detection = if matches!( + let needs_public_internet_address_detection = if matches!( public_internet_network_class, NetworkClass::InboundCapable ) { @@ -148,11 +138,12 @@ impl NetworkManager { let dial_info_filter = flow.make_dial_info_filter(); // Get current external ip/port from registered global dialinfo - let current_addresses: BTreeSet = routing_table - .all_filtered_dial_info_details( - RoutingDomain::PublicInternet.into(), - &dial_info_filter, - ) + let current_addresses: BTreeSet = published_peer_info + .signed_node_info() + .node_info() + .filtered_dial_info_details(DialInfoDetail::NO_SORT, |did| { + did.matches_filter(&dial_info_filter) + }) .iter() .map(|did| { // Strip port from direct and mapped addresses @@ -180,7 +171,7 @@ impl NetworkManager { if !current_addresses.contains(a) && !current_addresses.contains(&a.with_port(0)) && !inner - .public_address_inconsistencies_table + .public_internet_address_inconsistencies_table .get(&addr_proto_type_key) .map(|pait| pait.contains_key(reporting_ip_block)) .unwrap_or(false) @@ -195,39 +186,42 @@ impl NetworkManager { // If we have enough inconsistencies to consider changing our public dial info, // add them to our denylist (throttling) and go ahead and check for new // public dialinfo - let inconsistent = if inconsistencies.len() >= PUBLIC_ADDRESS_CHANGE_DETECTION_COUNT { - let exp_ts = Timestamp::now() + PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US; - let pait = inner - .public_address_inconsistencies_table - .entry(addr_proto_type_key) - .or_default(); - for i in &inconsistencies { - pait.insert(*i, exp_ts); - } - - // Run this routine if the inconsistent nodes turn out to be lying - let this = self.clone(); - bad_public_address_detection_punishment = Some(Box::new(move || { - let mut inner = this.inner.lock(); + let inconsistent = + if inconsistencies.len() >= PUBLIC_ADDRESS_CHANGE_INCONSISTENCY_DETECTION_COUNT { + let exp_ts = Timestamp::now() + PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US; let pait = inner - .public_address_inconsistencies_table + .public_internet_address_inconsistencies_table .entry(addr_proto_type_key) .or_default(); - let exp_ts = - Timestamp::now() + PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US; - for i in inconsistencies { - pait.insert(i, exp_ts); + for i in &inconsistencies { + pait.insert(*i, exp_ts); } - })); - true - } else { - false - }; + // Run this routine if the inconsistent nodes turn out to be lying + let this = self.clone(); + bad_public_internet_address_detection_punishment = Some(Box::new(move || { + // xxx does this even work?? + + let mut inner = this.inner.lock(); + let pait = inner + .public_internet_address_inconsistencies_table + .entry(addr_proto_type_key) + .or_default(); + let exp_ts = + Timestamp::now() + PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US; + for i in inconsistencies { + pait.insert(i, exp_ts); + } + })); + + true + } else { + false + }; // // debug code // if inconsistent { - // log_net!("public_address_check_cache: {:#?}\ncurrent_addresses: {:#?}\ninconsistencies: {}", inner + // log_net!("report_public_internet_socket_address: {:#?}\ncurrent_addresses: {:#?}\ninconsistencies: {}", inner // .public_address_check_cache, current_addresses, inconsistencies); // } @@ -246,7 +240,7 @@ impl NetworkManager { if let Some(current_address) = current_address { if current_address == *a { consistencies += 1; - if consistencies >= PUBLIC_ADDRESS_CHANGE_DETECTION_COUNT { + if consistencies >= PUBLIC_ADDRESS_CHANGE_CONSISTENCY_DETECTION_COUNT { consistent = true; break; } @@ -264,26 +258,28 @@ impl NetworkManager { unreachable!(); }; - if needs_public_address_detection { + if needs_public_internet_address_detection { if detect_address_changes { // Reset the address check cache now so we can start detecting fresh - info!("Public address has changed, detecting public dial info"); - log_net!(debug "report_global_socket_address\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer); + info!("PublicInternet address has changed, detecting public dial info"); + log_net!(debug "report_public_internet_socket_address:\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer); log_net!(debug - "public_address_check_cache: {:#?}", - inner.public_address_check_cache + "public_internet_address_check_cache: {:#?}", + inner.public_internet_address_check_cache ); - inner.public_address_check_cache.clear(); + inner.public_internet_address_check_cache.clear(); // Re-detect the public dialinfo - net.set_needs_public_dial_info_check(bad_public_address_detection_punishment); + net.set_needs_public_dial_info_check( + bad_public_internet_address_detection_punishment, + ); } else { - warn!("Public address may have changed. Restarting the server may be required."); - warn!("report_global_socket_address\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer); + warn!("PublicInternet address may have changed. Restarting the server may be required."); + warn!("report_public_internet_socket_address:\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer); warn!( - "public_address_check_cache: {:#?}", - inner.public_address_check_cache + "public_internet_address_check_cache: {:#?}", + inner.public_internet_address_check_cache ); } } diff --git a/veilid-core/src/network_manager/types/address.rs b/veilid-core/src/network_manager/types/address.rs index df660960..63b6acf9 100644 --- a/veilid-core/src/network_manager/types/address.rs +++ b/veilid-core/src/network_manager/types/address.rs @@ -20,7 +20,7 @@ impl Address { SocketAddr::V6(v6) => Address::IPV6(*v6.ip()), } } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub fn from_ip_addr(addr: IpAddr) -> Address { match addr { IpAddr::V4(v4) => Address::IPV4(v4), diff --git a/veilid-core/src/network_manager/types/dial_info/mod.rs b/veilid-core/src/network_manager/types/dial_info/mod.rs index 60032b9f..020bf44c 100644 --- a/veilid-core/src/network_manager/types/dial_info/mod.rs +++ b/veilid-core/src/network_manager/types/dial_info/mod.rs @@ -234,7 +234,7 @@ impl DialInfo { Self::WSS(di) => di.socket_address.address(), } } - #[allow(dead_code)] + #[expect(dead_code)] pub fn set_address(&mut self, address: Address) { match self { Self::UDP(di) => di.socket_address.set_address(address), @@ -259,7 +259,7 @@ impl DialInfo { Self::WSS(di) => di.socket_address.ip_addr(), } } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub fn port(&self) -> u16 { match self { Self::UDP(di) => di.socket_address.port(), @@ -268,7 +268,7 @@ impl DialInfo { Self::WSS(di) => di.socket_address.port(), } } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub fn set_port(&mut self, port: u16) { match self { Self::UDP(di) => di.socket_address.set_port(port), @@ -277,7 +277,6 @@ impl DialInfo { Self::WSS(di) => di.socket_address.set_port(port), } } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] pub fn to_socket_addr(&self) -> SocketAddr { match self { Self::UDP(di) => di.socket_address.socket_addr(), @@ -457,7 +456,7 @@ impl DialInfo { } } } - #[allow(dead_code)] + #[expect(dead_code)] pub async fn to_url(&self) -> String { match self { DialInfo::UDP(di) => intf::ptr_lookup(di.socket_address.ip_addr()) diff --git a/veilid-core/src/network_manager/types/dial_info_filter.rs b/veilid-core/src/network_manager/types/dial_info_filter.rs index ad64fbed..62a2e1e5 100644 --- a/veilid-core/src/network_manager/types/dial_info_filter.rs +++ b/veilid-core/src/network_manager/types/dial_info_filter.rs @@ -46,7 +46,7 @@ impl DialInfoFilter { pub fn is_dead(&self) -> bool { self.protocol_type_set.is_empty() || self.address_type_set.is_empty() } - pub fn with_sequencing(self, sequencing: Sequencing) -> (bool, DialInfoFilter) { + pub fn apply_sequencing(self, sequencing: Sequencing) -> (bool, DialInfoFilter) { // Get first filtered dialinfo match sequencing { Sequencing::NoPreference => (false, self), diff --git a/veilid-core/src/network_manager/types/punishment.rs b/veilid-core/src/network_manager/types/punishment.rs index 06e15204..09b7b33d 100644 --- a/veilid-core/src/network_manager/types/punishment.rs +++ b/veilid-core/src/network_manager/types/punishment.rs @@ -10,7 +10,7 @@ pub enum PunishmentReason { // Node-level punishments FailedToDecodeOperation, WrongSenderPeerInfo, - FailedToVerifySenderPeerInfo, + // FailedToVerifySenderPeerInfo, FailedToRegisterSenderPeerInfo, // Route-level punishments // FailedToDecodeRoutedMessage, diff --git a/veilid-core/src/network_manager/types/signal_info.rs b/veilid-core/src/network_manager/types/signal_info.rs index 0fb49746..45c71a0d 100644 --- a/veilid-core/src/network_manager/types/signal_info.rs +++ b/veilid-core/src/network_manager/types/signal_info.rs @@ -1,21 +1,21 @@ use super::*; /// Parameter for Signal operation -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub enum SignalInfo { /// UDP Hole Punch Request HolePunch { /// /// Receipt to be returned after the hole punch receipt: Vec, /// Sender's peer info - peer_info: PeerInfo, + peer_info: Arc, }, /// Reverse Connection Request ReverseConnect { /// Receipt to be returned by the reverse connection receipt: Vec, /// Sender's peer info - peer_info: PeerInfo, + peer_info: Arc, }, // XXX: WebRTC } diff --git a/veilid-core/src/network_manager/types/socket_address.rs b/veilid-core/src/network_manager/types/socket_address.rs index 097023cc..ad30bbdf 100644 --- a/veilid-core/src/network_manager/types/socket_address.rs +++ b/veilid-core/src/network_manager/types/socket_address.rs @@ -30,7 +30,6 @@ impl SocketAddress { pub fn port(&self) -> u16 { self.port } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] pub fn set_port(&mut self, port: u16) { self.port = port } diff --git a/veilid-core/src/network_manager/wasm/mod.rs b/veilid-core/src/network_manager/wasm/mod.rs index f116baa5..d06458f2 100644 --- a/veilid-core/src/network_manager/wasm/mod.rs +++ b/veilid-core/src/network_manager/wasm/mod.rs @@ -51,6 +51,14 @@ pub const MAX_CAPABILITIES: usize = 64; ///////////////////////////////////////////////////////////////// +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ProtocolConfig { + pub outbound: ProtocolTypeSet, + pub inbound: ProtocolTypeSet, + pub family_global: AddressTypeSet, + pub public_internet_capabilities: Vec, +} + struct NetworkInner { network_needs_restart: bool, protocol_config: ProtocolConfig, @@ -371,7 +379,6 @@ impl Network { }; let family_global = supported_address_types; - let family_local = supported_address_types; let public_internet_capabilities = { PUBLIC_INTERNET_CAPABILITIES @@ -385,8 +392,6 @@ impl Network { outbound, inbound, family_global, - family_local, - local_network_capabilities: vec![], public_internet_capabilities, } }; @@ -396,7 +401,7 @@ impl Network { let mut editor_public_internet = self .unlocked_inner .routing_table - .edit_routing_domain(RoutingDomain::PublicInternet); + .edit_public_internet_routing_domain(); // set up the routing table's network config // if we have static public dialinfo, upgrade our network class @@ -409,8 +414,10 @@ impl Network { ); editor_public_internet.set_network_class(Some(NetworkClass::WebApp)); - // commit routing table edits - editor_public_internet.commit(true).await; + // commit routing domain edits + if editor_public_internet.commit(true).await { + editor_public_internet.publish(); + } Ok(StartupDisposition::Success) } @@ -459,14 +466,9 @@ impl Network { // Reset state let routing_table = self.routing_table(); - - // Drop all dial info routing_table - .edit_routing_domain(RoutingDomain::PublicInternet) - .clear_dial_info_details(None, None) - .set_network_class(None) - .clear_relay_node() - .commit(true) + .edit_public_internet_routing_domain() + .shutdown() .await; // Cancels all async background tasks by dropping join handles diff --git a/veilid-core/src/network_manager/wasm/protocol/mod.rs b/veilid-core/src/network_manager/wasm/protocol/mod.rs index 6b1ae16b..ca423acd 100644 --- a/veilid-core/src/network_manager/wasm/protocol/mod.rs +++ b/veilid-core/src/network_manager/wasm/protocol/mod.rs @@ -6,7 +6,6 @@ use std::io; #[derive(Debug)] pub(in crate::network_manager) enum ProtocolNetworkConnection { - #[allow(dead_code)] //Dummy(DummyNetworkConnection), Ws(ws::WebsocketNetworkConnection), //WebRTC(wrtc::WebRTCNetworkConnection), diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 6360bfb4..017be327 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -243,7 +243,6 @@ impl BucketEntryInner { } // Less is faster - #[allow(dead_code)] pub fn cmp_fastest(e1: &Self, e2: &Self) -> std::cmp::Ordering { // Lower latency to the front if let Some(e1_latency) = &e1.peer_stats.latency { @@ -295,28 +294,18 @@ impl BucketEntryInner { } } - #[allow(dead_code)] + #[expect(dead_code)] pub fn sort_fastest_reliable_fn( cur_ts: Timestamp, ) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering { move |e1, e2| Self::cmp_fastest_reliable(cur_ts, e1, e2) } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - pub fn clear_signed_node_info(&mut self, routing_domain: RoutingDomain) { - // Get the correct signed_node_info for the chosen routing domain - let opt_current_sni = match routing_domain { - RoutingDomain::LocalNetwork => &mut self.local_network.signed_node_info, - RoutingDomain::PublicInternet => &mut self.public_internet.signed_node_info, - }; - *opt_current_sni = None; - } - pub fn update_signed_node_info( &mut self, routing_domain: RoutingDomain, signed_node_info: SignedNodeInfo, - ) { + ) -> bool { // Get the correct signed_node_info for the chosen routing domain let opt_current_sni = match routing_domain { RoutingDomain::LocalNetwork => &mut self.local_network.signed_node_info, @@ -341,11 +330,11 @@ impl BucketEntryInner { self.updated_since_last_network_change = true; self.make_not_dead(Timestamp::now()); } - return; + return false; } // See if anything has changed in this update beside the timestamp - if signed_node_info.node_info() != current_sni.node_info() { + if !signed_node_info.equivalent(current_sni) { node_info_changed = true; } } @@ -369,6 +358,8 @@ impl BucketEntryInner { if node_info_changed { self.clear_last_flows_except_latest(); } + + node_info_changed } pub fn has_node_info(&self, routing_domain_set: RoutingDomainSet) -> bool { @@ -425,7 +416,7 @@ impl BucketEntryInner { let node_ids = self.node_ids(); opt_current_sni .as_ref() - .map(|s| PeerInfo::new(node_ids, *s.clone())) + .map(|s| PeerInfo::new(routing_domain, node_ids, *s.clone())) } pub fn best_routing_domain( @@ -588,7 +579,7 @@ impl BucketEntryInner { self.envelope_support = envelope_support; } - #[allow(dead_code)] + #[expect(dead_code)] pub fn envelope_support(&self) -> Vec { self.envelope_support.clone() } diff --git a/veilid-core/src/routing_table/debug.rs b/veilid-core/src/routing_table/debug.rs index ddeb7b15..e1ed0213 100644 --- a/veilid-core/src/routing_table/debug.rs +++ b/veilid-core/src/routing_table/debug.rs @@ -55,6 +55,14 @@ impl RoutingTable { out } + pub(crate) fn debug_info_nodeid(&self) -> String { + let mut out = String::new(); + for nid in self.unlocked_inner.node_ids().iter() { + out += &format!("{}\n", nid); + } + out + } + pub(crate) fn debug_info_nodeinfo(&self) -> String { let mut out = String::new(); let inner = self.inner.read(); @@ -94,13 +102,25 @@ impl RoutingTable { out } - pub(crate) fn debug_info_peerinfo(&self, routing_domain: RoutingDomain) -> String { + pub(crate) fn debug_info_peerinfo( + &self, + routing_domain: RoutingDomain, + published: bool, + ) -> String { let mut out = String::new(); - out += &format!( - "{:?} PeerInfo:\n {:#?}\n", - routing_domain, - self.get_own_peer_info(routing_domain) - ); + if published { + out += &format!( + "{:?} Published PeerInfo:\n {:#?}\n", + routing_domain, + self.get_published_peer_info(routing_domain) + ); + } else { + out += &format!( + "{:?} Current PeerInfo:\n {:#?}\n", + routing_domain, + self.get_current_peer_info(routing_domain) + ); + } out } @@ -113,7 +133,7 @@ impl RoutingTable { PunishmentReason::InvalidFraming => "PFRAME", PunishmentReason::FailedToDecodeOperation => "PDECOP", PunishmentReason::WrongSenderPeerInfo => "PSPBAD", - PunishmentReason::FailedToVerifySenderPeerInfo => "PSPVER", + // PunishmentReason::FailedToVerifySenderPeerInfo => "PSPVER", PunishmentReason::FailedToRegisterSenderPeerInfo => "PSPREG", // }, diff --git a/veilid-core/src/routing_table/find_peers.rs b/veilid-core/src/routing_table/find_peers.rs index 6a7f6377..bfe8a887 100644 --- a/veilid-core/src/routing_table/find_peers.rs +++ b/veilid-core/src/routing_table/find_peers.rs @@ -7,37 +7,33 @@ impl RoutingTable { #[instrument(level = "trace", target = "rtab", skip_all)] pub fn find_preferred_closest_peers( &self, + routing_domain: RoutingDomain, key: TypedKey, capabilities: &[Capability], - ) -> NetworkResult> { - if !self.has_valid_network_class(RoutingDomain::PublicInternet) { - // Our own node info is not yet available, drop this request. - return NetworkResult::service_unavailable( - "Not finding closest peers because our network class is still invalid", - ); - } + ) -> NetworkResult>> { if Crypto::validate_crypto_kind(key.kind).is_err() { return NetworkResult::invalid_message("invalid crypto kind"); } + let Some(published_peer_info) = self.get_published_peer_info(routing_domain) else { + return NetworkResult::service_unavailable( + "Not finding closest peers because our network class is still invalid", + ); + }; + // find N nodes closest to the target node in our routing table - let own_peer_info = self.get_own_peer_info(RoutingDomain::PublicInternet); let filter = Box::new( |rti: &RoutingTableInner, opt_entry: Option>| { - // Ensure only things that are valid/signed in the PublicInternet domain are returned - if !rti.filter_has_valid_signed_node_info( - RoutingDomain::PublicInternet, - true, - opt_entry.clone(), - ) { + // Ensure only things that are valid/signed in the chosen routing domain are returned + if !rti.filter_has_valid_signed_node_info(routing_domain, true, opt_entry.clone()) { return false; } // Ensure capabilities are met match opt_entry { Some(entry) => entry.with(rti, |_rti, e| { - e.has_all_capabilities(RoutingDomain::PublicInternet, capabilities) + e.has_all_capabilities(routing_domain, capabilities) }), - None => own_peer_info + None => published_peer_info .signed_node_info() .node_info() .has_all_capabilities(capabilities), @@ -57,7 +53,7 @@ impl RoutingTable { filters, // transform |rti, entry| { - rti.transform_to_peer_info(RoutingDomain::PublicInternet, &own_peer_info, entry) + rti.transform_to_peer_info(routing_domain, published_peer_info.clone(), entry) }, ) { Ok(v) => v, @@ -76,9 +72,10 @@ impl RoutingTable { #[instrument(level = "trace", target = "rtab", skip_all)] pub fn find_preferred_peers_closer_to_key( &self, + routing_domain: RoutingDomain, key: TypedKey, required_capabilities: Vec, - ) -> NetworkResult> { + ) -> NetworkResult>> { // add node information for the requesting node to our routing table let crypto_kind = key.kind; let own_node_id = self.node_id(crypto_kind); @@ -99,14 +96,12 @@ impl RoutingTable { }; // Ensure only things that have a minimum set of capabilities are returned entry.with(rti, |rti, e| { - if !e - .has_all_capabilities(RoutingDomain::PublicInternet, &required_capabilities) - { + if !e.has_all_capabilities(routing_domain, &required_capabilities) { return false; } // Ensure only things that are valid/signed in the PublicInternet domain are returned if !rti.filter_has_valid_signed_node_info( - RoutingDomain::PublicInternet, + routing_domain, true, Some(entry.clone()), ) { @@ -139,7 +134,7 @@ impl RoutingTable { // transform |rti, entry| { entry.unwrap().with(rti, |_rti, e| { - e.make_peer_info(RoutingDomain::PublicInternet).unwrap() + Arc::new(e.make_peer_info(routing_domain).unwrap()) }) }, ) { @@ -174,7 +169,7 @@ impl RoutingTable { vcrypto: CryptoSystemVersion, key_far: TypedKey, key_near: TypedKey, - peers: &[PeerInfo], + peers: &[Arc], ) -> EyreResult { let kind = vcrypto.kind(); diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index c08b9472..52fbd00c 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -3,11 +3,8 @@ mod bucket_entry; mod debug; mod find_peers; mod node_ref; -mod node_ref_filter; mod privacy; mod route_spec_store; -mod routing_domain_editor; -mod routing_domains; mod routing_table_inner; mod stats_accounting; mod tasks; @@ -26,11 +23,8 @@ use hashlink::LruCache; pub(crate) use bucket_entry::*; pub(crate) use node_ref::*; -pub(crate) use node_ref_filter::*; pub(crate) use privacy::*; pub(crate) use route_spec_store::*; -pub(crate) use routing_domain_editor::*; -pub(crate) use routing_domains::*; pub(crate) use routing_table_inner::*; pub(crate) use stats_accounting::*; @@ -57,9 +51,6 @@ const ROUTING_TABLE: &str = "routing_table"; const SERIALIZED_BUCKET_MAP: &[u8] = b"serialized_bucket_map"; const CACHE_VALIDITY_KEY: &[u8] = b"cache_validity_key"; -// Critical sections -const LOCK_TAG_TICK: &str = "TICK"; - type LowLevelProtocolPorts = BTreeSet<(LowLevelProtocolType, AddressType, u16)>; type ProtocolToPortMapping = BTreeMap<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>; #[derive(Clone, Debug)] @@ -499,15 +490,6 @@ impl RoutingTable { Ok(()) } - /// Set up the local network routing domain with our local routing table configuration - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - pub fn configure_local_network_routing_domain(&self, local_networks: Vec<(IpAddr, IpAddr)>) { - log_net!(debug "configure_local_network_routing_domain: {:#?}", local_networks); - self.inner - .write() - .configure_local_network_routing_domain(local_networks); - } - ///////////////////////////////////// /// Locked operations @@ -519,7 +501,7 @@ impl RoutingTable { self.inner.read().route_spec_store.as_ref().unwrap().clone() } - pub fn relay_node(&self, domain: RoutingDomain) -> Option { + pub fn relay_node(&self, domain: RoutingDomain) -> Option { self.inner.read().relay_node(domain) } @@ -541,13 +523,6 @@ impl RoutingTable { .all_filtered_dial_info_details(routing_domain_set, filter) } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - pub fn ensure_dial_info_is_valid(&self, domain: RoutingDomain, dial_info: &DialInfo) -> bool { - self.inner - .read() - .ensure_dial_info_is_valid(domain, dial_info) - } - pub fn signed_node_info_is_valid_in_routing_domain( &self, routing_domain: RoutingDomain, @@ -562,8 +537,8 @@ impl RoutingTable { pub fn get_contact_method( &self, routing_domain: RoutingDomain, - peer_a: &PeerInfo, - peer_b: &PeerInfo, + peer_a: Arc, + peer_b: Arc, dial_info_filter: DialInfoFilter, sequencing: Sequencing, dif_sort: Option>, @@ -578,14 +553,24 @@ impl RoutingTable { ) } - #[instrument(level = "debug", skip(self))] - pub fn edit_routing_domain(&self, domain: RoutingDomain) -> RoutingDomainEditor { - RoutingDomainEditor::new(self.clone(), domain) + /// Edit the PublicInternet RoutingDomain + pub fn edit_public_internet_routing_domain(&self) -> RoutingDomainEditorPublicInternet { + RoutingDomainEditorPublicInternet::new(self.clone()) } - /// Return a copy of our node's peerinfo - pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo { - self.inner.read().get_own_peer_info(routing_domain) + /// Edit the LocalNetwork RoutingDomain + pub fn edit_local_network_routing_domain(&self) -> RoutingDomainEditorLocalNetwork { + RoutingDomainEditorLocalNetwork::new(self.clone()) + } + + /// Return a copy of our node's peerinfo (may not yet be published) + pub fn get_published_peer_info(&self, routing_domain: RoutingDomain) -> Option> { + self.inner.read().get_published_peer_info(routing_domain) + } + + /// Return a copy of our node's peerinfo (may not yet be published) + pub fn get_current_peer_info(&self, routing_domain: RoutingDomain) -> Arc { + self.inner.read().get_current_peer_info(routing_domain) } /// If we have a valid network class in this routing domain, then our 'NodeInfo' is valid @@ -594,12 +579,8 @@ impl RoutingTable { self.inner.read().has_valid_network_class(routing_domain) } - /// Return our current node info timestamp - pub fn get_own_node_info_ts(&self, routing_domain: RoutingDomain) -> Timestamp { - self.inner.read().get_own_node_info_ts(routing_domain) - } - /// Return the domain's currently registered network class + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option { self.inner.read().get_network_class(routing_domain) } @@ -633,7 +614,7 @@ impl RoutingTable { &self, routing_domain: RoutingDomain, cur_ts: Timestamp, - ) -> Vec { + ) -> Vec { self.inner .read() .get_nodes_needing_ping(self.clone(), routing_domain, cur_ts) @@ -671,7 +652,7 @@ impl RoutingTable { node_id: TypedKey, routing_domain_set: RoutingDomainSet, dial_info_filter: DialInfoFilter, - ) -> EyreResult> { + ) -> EyreResult> { self.inner.read().lookup_and_filter_noderef( self.clone(), node_id, @@ -686,16 +667,12 @@ impl RoutingTable { #[instrument(level = "trace", skip_all, err)] pub fn register_node_with_peer_info( &self, - routing_domain: RoutingDomain, - peer_info: PeerInfo, + peer_info: Arc, allow_invalid: bool, - ) -> EyreResult { - self.inner.write().register_node_with_peer_info( - self.clone(), - routing_domain, - peer_info, - allow_invalid, - ) + ) -> EyreResult { + self.inner + .write() + .register_node_with_peer_info(self.clone(), peer_info, allow_invalid) } /// Shortcut function to add a node to our routing table if it doesn't exist @@ -703,12 +680,14 @@ impl RoutingTable { #[instrument(level = "trace", skip_all, err)] pub fn register_node_with_existing_connection( &self, + routing_domain: RoutingDomain, node_id: TypedKey, flow: Flow, timestamp: Timestamp, - ) -> EyreResult { + ) -> EyreResult { self.inner.write().register_node_with_existing_connection( self.clone(), + routing_domain, node_id, flow, timestamp, @@ -810,7 +789,7 @@ impl RoutingTable { } /// Makes a filter that finds nodes with a matching inbound dialinfo - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub fn make_inbound_dial_info_entry_filter<'a>( routing_domain: RoutingDomain, dial_info_filter: DialInfoFilter, @@ -864,14 +843,18 @@ impl RoutingTable { }) } - pub fn find_fast_public_nodes_filtered( + pub fn find_fast_non_local_nodes_filtered( &self, + routing_domain: RoutingDomain, node_count: usize, filters: VecDeque, ) -> Vec { - self.inner - .read() - .find_fast_public_nodes_filtered(self.clone(), node_count, filters) + self.inner.read().find_fast_non_local_nodes_filtered( + self.clone(), + routing_domain, + node_count, + filters, + ) } /// Retrieve up to N of each type of protocol capable nodes for a single crypto kind @@ -953,7 +936,7 @@ impl RoutingTable { protocol_types_len * 2 * max_per_type, filters, |_rti, entry: Option>| { - NodeRef::new(self.clone(), entry.unwrap().clone(), None) + NodeRef::new(self.clone(), entry.unwrap().clone()) }, ) } @@ -1018,28 +1001,22 @@ impl RoutingTable { .sort_and_clean_closest_noderefs(node_id, closest_nodes) } - #[instrument(level = "trace", skip(self, peers))] - pub fn register_find_node_answer( + #[instrument(level = "trace", skip(self, peer_info_list))] + pub fn register_nodes_with_peer_info_list( &self, - crypto_kind: CryptoKind, - peers: Vec, + peer_info_list: Vec>, ) -> Vec { // Register nodes we'd found - let mut out = Vec::::with_capacity(peers.len()); - for p in peers { - // Ensure we're getting back nodes we asked for - if !p.node_ids().kinds().contains(&crypto_kind) { - continue; - } - + let mut out = Vec::::with_capacity(peer_info_list.len()); + for p in peer_info_list { // Don't register our own node if self.matches_own_node_id(p.node_ids()) { continue; } // Register the node if it's new - match self.register_node_with_peer_info(RoutingDomain::PublicInternet, p, false) { - Ok(nr) => out.push(nr), + match self.register_node_with_peer_info(p, false) { + Ok(nr) => out.push(nr.unfiltered()), Err(e) => { log_rtab!(debug "failed to register node with peer info from find node answer: {}", e); } @@ -1051,7 +1028,7 @@ impl RoutingTable { /// Finds nodes near a particular node id /// Ensures all returned nodes have a set of capabilities enabled #[instrument(level = "trace", skip(self), err)] - pub async fn find_node( + pub async fn find_nodes_close_to_node_id( &self, node_ref: NodeRef, node_id: TypedKey, @@ -1062,33 +1039,38 @@ impl RoutingTable { let res = network_result_try!( rpc_processor .clone() - .rpc_call_find_node(Destination::direct(node_ref), node_id, capabilities) + .rpc_call_find_node( + Destination::direct(node_ref.default_filtered()), + node_id, + capabilities + ) .await? ); // register nodes we'd found Ok(NetworkResult::value( - self.register_find_node_answer(node_id.kind, res.answer), + self.register_nodes_with_peer_info_list(res.answer), )) } /// Ask a remote node to list the nodes it has around the current node /// Ensures all returned nodes have a set of capabilities enabled #[instrument(level = "trace", skip(self), err)] - pub async fn find_self( + pub async fn find_nodes_close_to_self( &self, crypto_kind: CryptoKind, node_ref: NodeRef, capabilities: Vec, ) -> EyreResult>> { let self_node_id = self.node_id(crypto_kind); - self.find_node(node_ref, self_node_id, capabilities).await + self.find_nodes_close_to_node_id(node_ref, self_node_id, capabilities) + .await } /// Ask a remote node to list the nodes it has around itself /// Ensures all returned nodes have a set of capabilities enabled #[instrument(level = "trace", skip(self), err)] - pub async fn find_target( + pub async fn find_nodes_close_to_node_ref( &self, crypto_kind: CryptoKind, node_ref: NodeRef, @@ -1097,7 +1079,8 @@ impl RoutingTable { let Some(target_node_id) = node_ref.node_ids().get(crypto_kind) else { bail!("no target node ids for this crypto kind"); }; - self.find_node(node_ref, target_node_id, capabilities).await + self.find_nodes_close_to_node_id(node_ref, target_node_id, capabilities) + .await } /// Ask node to 'find node' on own node so we can get some more nodes near ourselves @@ -1111,7 +1094,7 @@ impl RoutingTable { capabilities: Vec, ) { // Ask node for nodes closest to our own node - let closest_nodes = network_result_value_or_log!(match self.find_self(crypto_kind, node_ref.clone(), capabilities.clone()).await { + let closest_nodes = network_result_value_or_log!(match self.find_nodes_close_to_self(crypto_kind, node_ref.clone(), capabilities.clone()).await { Err(e) => { log_rtab!(error "find_self failed for {:?}: {:?}", @@ -1127,7 +1110,7 @@ impl RoutingTable { // Ask each node near us to find us as well if wide { for closest_nr in closest_nodes { - network_result_value_or_log!(match self.find_self(crypto_kind, closest_nr.clone(), capabilities.clone()).await { + network_result_value_or_log!(match self.find_nodes_close_to_self(crypto_kind, closest_nr.clone(), capabilities.clone()).await { Err(e) => { log_rtab!(error "find_self failed for {:?}: {:?}", diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs deleted file mode 100644 index 64fa362a..00000000 --- a/veilid-core/src/routing_table/node_ref.rs +++ /dev/null @@ -1,623 +0,0 @@ -use super::*; -use crate::crypto::*; -use alloc::fmt; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub(crate) struct NodeRefBaseCommon { - routing_table: RoutingTable, - entry: Arc, - filter: Option, - sequencing: Sequencing, - #[cfg(feature = "tracking")] - track_id: usize, -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub(crate) trait NodeRefBase: Sized { - // Common field access - fn common(&self) -> &NodeRefBaseCommon; - fn common_mut(&mut self) -> &mut NodeRefBaseCommon; - - // Comparators - fn same_entry(&self, other: &T) -> bool { - Arc::ptr_eq(&self.common().entry, &other.common().entry) - } - fn same_bucket_entry(&self, entry: &Arc) -> bool { - Arc::ptr_eq(&self.common().entry, entry) - } - - // Implementation-specific operators - fn operate(&self, f: F) -> T - where - F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T; - fn operate_mut(&self, f: F) -> T - where - F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T; - - // Filtering - fn filter_ref(&self) -> Option<&NodeRefFilter> { - self.common().filter.as_ref() - } - - fn take_filter(&mut self) -> Option { - self.common_mut().filter.take() - } - - fn set_filter(&mut self, filter: Option) { - self.common_mut().filter = filter - } - - fn set_sequencing(&mut self, sequencing: Sequencing) { - self.common_mut().sequencing = sequencing; - } - fn sequencing(&self) -> Sequencing { - self.common().sequencing - } - - fn merge_filter(&mut self, filter: NodeRefFilter) { - let common_mut = self.common_mut(); - if let Some(self_filter) = common_mut.filter.take() { - common_mut.filter = Some(self_filter.filtered(&filter)); - } else { - common_mut.filter = Some(filter); - } - } - - // fn is_filter_dead(&self) -> bool { - // if let Some(filter) = &self.common().filter { - // filter.is_dead() - // } else { - // false - // } - // } - - fn routing_domain_set(&self) -> RoutingDomainSet { - self.common() - .filter - .as_ref() - .map(|f| f.routing_domain_set) - .unwrap_or(RoutingDomainSet::all()) - } - - fn dial_info_filter(&self) -> DialInfoFilter { - self.common() - .filter - .as_ref() - .map(|f| f.dial_info_filter) - .unwrap_or(DialInfoFilter::all()) - } - - fn best_routing_domain(&self) -> Option { - self.operate(|rti, e| { - e.best_routing_domain( - rti, - self.common() - .filter - .as_ref() - .map(|f| f.routing_domain_set) - .unwrap_or(RoutingDomainSet::all()), - ) - }) - } - - // Accessors - fn routing_table(&self) -> RoutingTable { - self.common().routing_table.clone() - } - fn node_ids(&self) -> TypedKeyGroup { - self.operate(|_rti, e| e.node_ids()) - } - fn best_node_id(&self) -> TypedKey { - self.operate(|_rti, e| e.best_node_id()) - } - fn update_node_status(&self, routing_domain: RoutingDomain, node_status: NodeStatus) { - self.operate_mut(|_rti, e| { - e.update_node_status(routing_domain, node_status); - }); - } - // fn envelope_support(&self) -> Vec { - // self.operate(|_rti, e| e.envelope_support()) - // } - fn add_envelope_version(&self, envelope_version: u8) { - self.operate_mut(|_rti, e| e.add_envelope_version(envelope_version)) - } - // fn set_envelope_support(&self, envelope_support: Vec) { - // self.operate_mut(|_rti, e| e.set_envelope_support(envelope_support)) - // } - fn best_envelope_version(&self) -> Option { - self.operate(|_rti, e| e.best_envelope_version()) - } - fn state_reason(&self, cur_ts: Timestamp) -> BucketEntryStateReason { - self.operate(|_rti, e| e.state_reason(cur_ts)) - } - fn state(&self, cur_ts: Timestamp) -> BucketEntryState { - self.operate(|_rti, e| e.state(cur_ts)) - } - fn peer_stats(&self) -> PeerStats { - self.operate(|_rti, e| e.peer_stats().clone()) - } - - // Per-RoutingDomain accessors - fn make_peer_info(&self, routing_domain: RoutingDomain) -> Option { - self.operate(|_rti, e| e.make_peer_info(routing_domain)) - } - fn node_info(&self, routing_domain: RoutingDomain) -> Option { - self.operate(|_rti, e| e.node_info(routing_domain).cloned()) - } - fn signed_node_info_has_valid_signature(&self, routing_domain: RoutingDomain) -> bool { - self.operate(|_rti, e| { - e.signed_node_info(routing_domain) - .map(|sni| sni.has_any_signature()) - .unwrap_or(false) - }) - } - fn node_info_ts(&self, routing_domain: RoutingDomain) -> Timestamp { - self.operate(|_rti, e| { - e.signed_node_info(routing_domain) - .map(|sni| sni.timestamp()) - .unwrap_or(0u64.into()) - }) - } - fn has_seen_our_node_info_ts( - &self, - routing_domain: RoutingDomain, - our_node_info_ts: Timestamp, - ) -> bool { - self.operate(|_rti, e| e.has_seen_our_node_info_ts(routing_domain, our_node_info_ts)) - } - fn set_seen_our_node_info_ts(&self, routing_domain: RoutingDomain, seen_ts: Timestamp) { - self.operate_mut(|_rti, e| e.set_seen_our_node_info_ts(routing_domain, seen_ts)); - } - // fn network_class(&self, routing_domain: RoutingDomain) -> Option { - // self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.network_class())) - // } - // fn outbound_protocols(&self, routing_domain: RoutingDomain) -> Option { - // self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.outbound_protocols())) - // } - // fn address_types(&self, routing_domain: RoutingDomain) -> Option { - // self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.address_types())) - // } - // fn node_info_outbound_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { - // let mut dif = DialInfoFilter::all(); - // if let Some(outbound_protocols) = self.outbound_protocols(routing_domain) { - // dif = dif.with_protocol_type_set(outbound_protocols); - // } - // if let Some(address_types) = self.address_types(routing_domain) { - // dif = dif.with_address_type_set(address_types); - // } - // dif - // } - fn relay(&self, routing_domain: RoutingDomain) -> EyreResult> { - self.operate_mut(|rti, e| { - let Some(sni) = e.signed_node_info(routing_domain) else { - return Ok(None); - }; - let Some(rpi) = sni.relay_peer_info() else { - return Ok(None); - }; - // If relay is ourselves, then return None, because we can't relay through ourselves - // and to contact this node we should have had an existing inbound connection - if rti.unlocked_inner.matches_own_node_id(rpi.node_ids()) { - bail!("Can't relay though ourselves"); - } - - // Register relay node and return noderef - let nr = - rti.register_node_with_peer_info(self.routing_table(), routing_domain, rpi, false)?; - Ok(Some(nr)) - }) - } - - // Filtered accessors - fn first_filtered_dial_info_detail(&self) -> Option { - let routing_domain_set = self.routing_domain_set(); - let dial_info_filter = self.dial_info_filter(); - let sequencing = self.common().sequencing; - let (ordered, dial_info_filter) = dial_info_filter.with_sequencing(sequencing); - - let sort = if ordered { - Some(DialInfoDetail::ordered_sequencing_sort) - } else { - None - }; - - if dial_info_filter.is_dead() { - return None; - } - - let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); - - self.operate(|_rt, e| { - for routing_domain in routing_domain_set { - if let Some(ni) = e.node_info(routing_domain) { - if let Some(did) = ni.first_filtered_dial_info_detail(sort, filter) { - return Some(did); - } - } - } - None - }) - } - - fn all_filtered_dial_info_details(&self) -> Vec { - let routing_domain_set = self.routing_domain_set(); - let dial_info_filter = self.dial_info_filter(); - - let (sort, dial_info_filter) = match self.common().sequencing { - Sequencing::NoPreference => (None, dial_info_filter), - Sequencing::PreferOrdered => ( - Some(DialInfoDetail::ordered_sequencing_sort), - dial_info_filter, - ), - Sequencing::EnsureOrdered => ( - Some(DialInfoDetail::ordered_sequencing_sort), - dial_info_filter.filtered( - &DialInfoFilter::all().with_protocol_type_set(ProtocolType::all_ordered_set()), - ), - ), - }; - - let mut out = Vec::new(); - self.operate(|_rt, e| { - for routing_domain in routing_domain_set { - if let Some(ni) = e.node_info(routing_domain) { - let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); - if let Some(did) = ni.first_filtered_dial_info_detail(sort, filter) { - out.push(did); - } - } - } - }); - out.remove_duplicates(); - out - } - - /// Get the most recent 'last connection' to this node - /// Filtered first and then sorted by ordering preference and then by most recent - fn last_flow(&self) -> Option { - self.operate(|rti, e| { - // apply sequencing to filter and get sort - let sequencing = self.common().sequencing; - let filter = self.common().filter.unwrap_or_default(); - let (ordered, filter) = filter.with_sequencing(sequencing); - let mut last_connections = e.last_flows(rti, true, filter); - - if ordered { - last_connections.sort_by(|a, b| { - ProtocolType::ordered_sequencing_sort(a.0.protocol_type(), b.0.protocol_type()) - }); - } - - last_connections.first().map(|x| x.0) - }) - } - - fn clear_last_connections(&self) { - self.operate_mut(|_rti, e| e.clear_last_flows()) - } - - fn set_last_flow(&self, flow: Flow, ts: Timestamp) { - self.operate_mut(|rti, e| { - e.set_last_flow(flow, ts); - rti.touch_recent_peer(e.best_node_id(), flow); - }) - } - - fn clear_last_connection(&self, flow: Flow) { - self.operate_mut(|_rti, e| { - e.remove_last_flow(flow); - }) - } - - fn has_any_dial_info(&self) -> bool { - self.operate(|_rti, e| { - for rtd in RoutingDomain::all() { - if let Some(sni) = e.signed_node_info(rtd) { - if sni.has_any_dial_info() { - return true; - } - } - } - false - }) - } - - fn report_protected_connection_dropped(&self) { - self.stats_failed_to_send(Timestamp::now(), false); - } - - fn report_failed_route_test(&self) { - self.stats_failed_to_send(Timestamp::now(), false); - } - - fn stats_question_sent(&self, ts: Timestamp, bytes: ByteCount, expects_answer: bool) { - self.operate_mut(|rti, e| { - rti.transfer_stats_accounting().add_up(bytes); - e.question_sent(ts, bytes, expects_answer); - }) - } - fn stats_question_rcvd(&self, ts: Timestamp, bytes: ByteCount) { - self.operate_mut(|rti, e| { - rti.transfer_stats_accounting().add_down(bytes); - e.question_rcvd(ts, bytes); - }) - } - fn stats_answer_sent(&self, bytes: ByteCount) { - self.operate_mut(|rti, e| { - rti.transfer_stats_accounting().add_up(bytes); - e.answer_sent(bytes); - }) - } - fn stats_answer_rcvd(&self, send_ts: Timestamp, recv_ts: Timestamp, bytes: ByteCount) { - self.operate_mut(|rti, e| { - rti.transfer_stats_accounting().add_down(bytes); - rti.latency_stats_accounting() - .record_latency(recv_ts.saturating_sub(send_ts)); - e.answer_rcvd(send_ts, recv_ts, bytes); - }) - } - fn stats_question_lost(&self) { - self.operate_mut(|_rti, e| { - e.question_lost(); - }) - } - fn stats_failed_to_send(&self, ts: Timestamp, expects_answer: bool) { - self.operate_mut(|_rti, e| { - e.failed_to_send(ts, expects_answer); - }) - } -} - -//////////////////////////////////////////////////////////////////////////////////// - -/// Reference to a routing table entry -/// Keeps entry in the routing table until all references are gone -pub(crate) struct NodeRef { - common: NodeRefBaseCommon, -} - -impl NodeRef { - pub fn new( - routing_table: RoutingTable, - entry: Arc, - filter: Option, - ) -> Self { - entry.ref_count.fetch_add(1u32, Ordering::AcqRel); - - Self { - common: NodeRefBaseCommon { - routing_table, - entry, - filter, - sequencing: Sequencing::NoPreference, - #[cfg(feature = "tracking")] - track_id: entry.track(), - }, - } - } - - pub fn filtered_clone(&self, filter: NodeRefFilter) -> Self { - let mut out = self.clone(); - out.merge_filter(filter); - out - } - - pub fn locked<'a>(&self, rti: &'a RoutingTableInner) -> NodeRefLocked<'a> { - NodeRefLocked::new(rti, self.clone()) - } - pub fn locked_mut<'a>(&self, rti: &'a mut RoutingTableInner) -> NodeRefLockedMut<'a> { - NodeRefLockedMut::new(rti, self.clone()) - } -} - -impl NodeRefBase for NodeRef { - fn common(&self) -> &NodeRefBaseCommon { - &self.common - } - - fn common_mut(&mut self) -> &mut NodeRefBaseCommon { - &mut self.common - } - - fn operate(&self, f: F) -> T - where - F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, - { - let inner = &*self.common.routing_table.inner.read(); - self.common.entry.with(inner, f) - } - - fn operate_mut(&self, f: F) -> T - where - F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, - { - let inner = &mut *self.common.routing_table.inner.write(); - self.common.entry.with_mut(inner, f) - } -} - -impl Clone for NodeRef { - fn clone(&self) -> Self { - self.common - .entry - .ref_count - .fetch_add(1u32, Ordering::AcqRel); - - Self { - common: NodeRefBaseCommon { - routing_table: self.common.routing_table.clone(), - entry: self.common.entry.clone(), - filter: self.common.filter, - sequencing: self.common.sequencing, - #[cfg(feature = "tracking")] - track_id: self.common.entry.write().track(), - }, - } - } -} - -impl fmt::Display for NodeRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.common.entry.with_inner(|e| e.best_node_id())) - } -} - -impl fmt::Debug for NodeRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("NodeRef") - .field("node_ids", &self.common.entry.with_inner(|e| e.node_ids())) - .field("filter", &self.common.filter) - .field("sequencing", &self.common.sequencing) - .finish() - } -} - -impl Drop for NodeRef { - fn drop(&mut self) { - #[cfg(feature = "tracking")] - self.common.entry.write().untrack(self.track_id); - - // drop the noderef and queue a bucket kick if it was the last one - let new_ref_count = self - .common - .entry - .ref_count - .fetch_sub(1u32, Ordering::AcqRel) - - 1; - if new_ref_count == 0 { - // get node ids with inner unlocked because nothing could be referencing this entry now - // and we don't know when it will get dropped, possibly inside a lock - let node_ids = self.common().entry.with_inner(|e| e.node_ids()); - self.common.routing_table.queue_bucket_kicks(node_ids); - } - } -} - -//////////////////////////////////////////////////////////////////////////////////// - -/// Locked reference to a routing table entry -/// For internal use inside the RoutingTable module where you have -/// already locked a RoutingTableInner -/// Keeps entry in the routing table until all references are gone -pub(crate) struct NodeRefLocked<'a> { - inner: Mutex<&'a RoutingTableInner>, - nr: NodeRef, -} - -impl<'a> NodeRefLocked<'a> { - pub fn new(inner: &'a RoutingTableInner, nr: NodeRef) -> Self { - Self { - inner: Mutex::new(inner), - nr, - } - } - - pub fn unlocked(&self) -> NodeRef { - self.nr.clone() - } -} - -impl<'a> NodeRefBase for NodeRefLocked<'a> { - fn common(&self) -> &NodeRefBaseCommon { - &self.nr.common - } - - fn common_mut(&mut self) -> &mut NodeRefBaseCommon { - &mut self.nr.common - } - - fn operate(&self, f: F) -> T - where - F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, - { - let inner = &*self.inner.lock(); - self.nr.common.entry.with(inner, f) - } - - fn operate_mut(&self, _f: F) -> T - where - F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, - { - panic!("need to locked_mut() for this operation") - } -} - -impl<'a> fmt::Display for NodeRefLocked<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.nr) - } -} - -impl<'a> fmt::Debug for NodeRefLocked<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("NodeRefLocked") - .field("nr", &self.nr) - .finish() - } -} - -//////////////////////////////////////////////////////////////////////////////////// - -/// Mutable locked reference to a routing table entry -/// For internal use inside the RoutingTable module where you have -/// already locked a RoutingTableInner -/// Keeps entry in the routing table until all references are gone -pub(crate) struct NodeRefLockedMut<'a> { - inner: Mutex<&'a mut RoutingTableInner>, - nr: NodeRef, -} - -impl<'a> NodeRefLockedMut<'a> { - pub fn new(inner: &'a mut RoutingTableInner, nr: NodeRef) -> Self { - Self { - inner: Mutex::new(inner), - nr, - } - } - - // pub fn unlocked(&self) -> NodeRef { - // self.nr.clone() - // } -} - -impl<'a> NodeRefBase for NodeRefLockedMut<'a> { - fn common(&self) -> &NodeRefBaseCommon { - &self.nr.common - } - - fn common_mut(&mut self) -> &mut NodeRefBaseCommon { - &mut self.nr.common - } - - fn operate(&self, f: F) -> T - where - F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, - { - let inner = &*self.inner.lock(); - self.nr.common.entry.with(inner, f) - } - - fn operate_mut(&self, f: F) -> T - where - F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, - { - let inner = &mut *self.inner.lock(); - self.nr.common.entry.with_mut(inner, f) - } -} - -impl<'a> fmt::Display for NodeRefLockedMut<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.nr) - } -} - -impl<'a> fmt::Debug for NodeRefLockedMut<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("NodeRefLockedMut") - .field("nr", &self.nr) - .finish() - } -} diff --git a/veilid-core/src/routing_table/node_ref/filtered_node_ref.rs b/veilid-core/src/routing_table/node_ref/filtered_node_ref.rs new file mode 100644 index 00000000..2484abb3 --- /dev/null +++ b/veilid-core/src/routing_table/node_ref/filtered_node_ref.rs @@ -0,0 +1,168 @@ +use super::*; + +pub struct FilteredNodeRef { + routing_table: RoutingTable, + entry: Arc, + filter: NodeRefFilter, + sequencing: Sequencing, + #[cfg(feature = "tracking")] + track_id: usize, +} + +impl FilteredNodeRef { + pub fn new( + routing_table: RoutingTable, + entry: Arc, + filter: NodeRefFilter, + sequencing: Sequencing, + ) -> Self { + entry.ref_count.fetch_add(1u32, Ordering::AcqRel); + + Self { + routing_table, + entry, + filter, + sequencing, + #[cfg(feature = "tracking")] + track_id: entry.track(), + } + } + + pub fn unfiltered(&self) -> NodeRef { + NodeRef::new(self.routing_table.clone(), self.entry.clone()) + } + + pub fn filtered_clone(&self, filter: NodeRefFilter) -> FilteredNodeRef { + let mut out = self.clone(); + out.merge_filter(filter); + out + } + + pub fn sequencing_clone(&self, sequencing: Sequencing) -> FilteredNodeRef { + FilteredNodeRef::new( + self.routing_table.clone(), + self.entry.clone(), + self.filter(), + sequencing, + ) + } + + pub fn locked<'a>(&self, rti: &'a RoutingTableInner) -> LockedFilteredNodeRef<'a> { + LockedFilteredNodeRef::new(rti, self.clone()) + } + + #[expect(dead_code)] + pub fn locked_mut<'a>(&self, rti: &'a mut RoutingTableInner) -> LockedMutFilteredNodeRef<'a> { + LockedMutFilteredNodeRef::new(rti, self.clone()) + } + + pub fn set_filter(&mut self, filter: NodeRefFilter) { + self.filter = filter + } + + pub fn merge_filter(&mut self, filter: NodeRefFilter) { + self.filter = self.filter.filtered(&filter); + } + + pub fn set_sequencing(&mut self, sequencing: Sequencing) { + self.sequencing = sequencing; + } +} + +impl NodeRefAccessorsTrait for FilteredNodeRef { + fn routing_table(&self) -> RoutingTable { + self.routing_table.clone() + } + fn entry(&self) -> Arc { + self.entry.clone() + } + + fn sequencing(&self) -> Sequencing { + self.sequencing + } + + fn routing_domain_set(&self) -> RoutingDomainSet { + self.filter.routing_domain_set + } + + fn filter(&self) -> NodeRefFilter { + self.filter + } + + fn take_filter(&mut self) -> NodeRefFilter { + let f = self.filter; + self.filter = NodeRefFilter::new(); + f + } + + fn dial_info_filter(&self) -> DialInfoFilter { + self.filter.dial_info_filter + } +} + +impl NodeRefOperateTrait for FilteredNodeRef { + fn operate(&self, f: F) -> T + where + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, + { + let inner = &*self.routing_table.inner.read(); + self.entry.with(inner, f) + } + + fn operate_mut(&self, f: F) -> T + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, + { + let inner = &mut *self.routing_table.inner.write(); + self.entry.with_mut(inner, f) + } +} + +impl NodeRefCommonTrait for FilteredNodeRef {} + +impl Clone for FilteredNodeRef { + fn clone(&self) -> Self { + self.entry.ref_count.fetch_add(1u32, Ordering::AcqRel); + + Self { + routing_table: self.routing_table.clone(), + entry: self.entry.clone(), + filter: self.filter, + sequencing: self.sequencing, + #[cfg(feature = "tracking")] + track_id: self.entry.write().track(), + } + } +} + +impl fmt::Display for FilteredNodeRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.entry.with_inner(|e| e.best_node_id())) + } +} + +impl fmt::Debug for FilteredNodeRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FilteredNodeRef") + .field("node_ids", &self.entry.with_inner(|e| e.node_ids())) + .field("filter", &self.filter) + .field("sequencing", &self.sequencing) + .finish() + } +} + +impl Drop for FilteredNodeRef { + fn drop(&mut self) { + #[cfg(feature = "tracking")] + self.entry.write().untrack(self.track_id); + + // drop the noderef and queue a bucket kick if it was the last one + let new_ref_count = self.entry.ref_count.fetch_sub(1u32, Ordering::AcqRel) - 1; + if new_ref_count == 0 { + // get node ids with inner unlocked because nothing could be referencing this entry now + // and we don't know when it will get dropped, possibly inside a lock + let node_ids = self.entry.with_inner(|e| e.node_ids()); + self.routing_table.queue_bucket_kicks(node_ids); + } + } +} diff --git a/veilid-core/src/routing_table/node_ref/mod.rs b/veilid-core/src/routing_table/node_ref/mod.rs new file mode 100644 index 00000000..5162ebd4 --- /dev/null +++ b/veilid-core/src/routing_table/node_ref/mod.rs @@ -0,0 +1,184 @@ +mod filtered_node_ref; +mod node_ref_filter; +mod node_ref_lock; +mod node_ref_lock_mut; +mod traits; + +use super::*; + +pub use filtered_node_ref::*; +pub use node_ref_filter::*; +pub use node_ref_lock::*; +pub use node_ref_lock_mut::*; +pub use traits::*; + +/////////////////////////////////////////////////////////////////////////// +// Default NodeRef + +pub struct NodeRef { + routing_table: RoutingTable, + entry: Arc, + #[cfg(feature = "tracking")] + track_id: usize, +} + +impl NodeRef { + pub fn new(routing_table: RoutingTable, entry: Arc) -> Self { + entry.ref_count.fetch_add(1u32, Ordering::AcqRel); + + Self { + routing_table, + entry, + #[cfg(feature = "tracking")] + track_id: entry.track(), + } + } + + pub fn default_filtered(&self) -> FilteredNodeRef { + FilteredNodeRef::new( + self.routing_table.clone(), + self.entry.clone(), + NodeRefFilter::new(), + Sequencing::default(), + ) + } + + pub fn sequencing_filtered(&self, sequencing: Sequencing) -> FilteredNodeRef { + FilteredNodeRef::new( + self.routing_table.clone(), + self.entry.clone(), + NodeRefFilter::new(), + sequencing, + ) + } + + pub fn routing_domain_filtered>( + &self, + routing_domain_set: R, + ) -> FilteredNodeRef { + FilteredNodeRef::new( + self.routing_table.clone(), + self.entry.clone(), + NodeRefFilter::new().with_routing_domain_set(routing_domain_set.into()), + Sequencing::default(), + ) + } + + pub fn custom_filtered(&self, filter: NodeRefFilter) -> FilteredNodeRef { + FilteredNodeRef::new( + self.routing_table.clone(), + self.entry.clone(), + filter, + Sequencing::default(), + ) + } + + #[expect(dead_code)] + pub fn dial_info_filtered(&self, filter: DialInfoFilter) -> FilteredNodeRef { + FilteredNodeRef::new( + self.routing_table.clone(), + self.entry.clone(), + NodeRefFilter::new().with_dial_info_filter(filter), + Sequencing::default(), + ) + } + + pub fn locked<'a>(&self, rti: &'a RoutingTableInner) -> LockedNodeRef<'a> { + LockedNodeRef::new(rti, self.clone()) + } + pub fn locked_mut<'a>(&self, rti: &'a mut RoutingTableInner) -> LockedMutNodeRef<'a> { + LockedMutNodeRef::new(rti, self.clone()) + } +} + +impl NodeRefAccessorsTrait for NodeRef { + fn routing_table(&self) -> RoutingTable { + self.routing_table.clone() + } + fn entry(&self) -> Arc { + self.entry.clone() + } + + fn sequencing(&self) -> Sequencing { + Sequencing::NoPreference + } + + fn routing_domain_set(&self) -> RoutingDomainSet { + RoutingDomainSet::all() + } + + fn filter(&self) -> NodeRefFilter { + NodeRefFilter::new() + } + + fn take_filter(&mut self) -> NodeRefFilter { + NodeRefFilter::new() + } + + fn dial_info_filter(&self) -> DialInfoFilter { + DialInfoFilter::all() + } +} + +impl NodeRefOperateTrait for NodeRef { + fn operate(&self, f: F) -> T + where + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, + { + let inner = &*self.routing_table.inner.read(); + self.entry.with(inner, f) + } + + fn operate_mut(&self, f: F) -> T + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, + { + let inner = &mut *self.routing_table.inner.write(); + self.entry.with_mut(inner, f) + } +} + +impl NodeRefCommonTrait for NodeRef {} + +impl Clone for NodeRef { + fn clone(&self) -> Self { + self.entry.ref_count.fetch_add(1u32, Ordering::AcqRel); + + Self { + routing_table: self.routing_table.clone(), + entry: self.entry.clone(), + #[cfg(feature = "tracking")] + track_id: self.entry.write().track(), + } + } +} + +impl fmt::Display for NodeRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.entry.with_inner(|e| e.best_node_id())) + } +} + +impl fmt::Debug for NodeRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NodeRef") + .field("node_ids", &self.entry.with_inner(|e| e.node_ids())) + .finish() + } +} + +impl Drop for NodeRef { + fn drop(&mut self) { + #[cfg(feature = "tracking")] + self.entry.write().untrack(self.track_id); + + // drop the noderef and queue a bucket kick if it was the last one + let new_ref_count = self.entry.ref_count.fetch_sub(1u32, Ordering::AcqRel) - 1; + if new_ref_count == 0 { + // get node ids with inner unlocked because nothing could be referencing this entry now + // and we don't know when it will get dropped, possibly inside a lock + let node_ids = self.entry.with_inner(|e| e.node_ids()); + self.routing_table.queue_bucket_kicks(node_ids); + } + } +} diff --git a/veilid-core/src/routing_table/node_ref_filter.rs b/veilid-core/src/routing_table/node_ref/node_ref_filter.rs similarity index 93% rename from veilid-core/src/routing_table/node_ref_filter.rs rename to veilid-core/src/routing_table/node_ref/node_ref_filter.rs index 68509832..94ced1ae 100644 --- a/veilid-core/src/routing_table/node_ref_filter.rs +++ b/veilid-core/src/routing_table/node_ref/node_ref_filter.rs @@ -35,7 +35,7 @@ impl NodeRefFilter { self.dial_info_filter = self.dial_info_filter.with_protocol_type(protocol_type); self } - #[allow(dead_code)] + #[expect(dead_code)] pub fn with_protocol_type_set(mut self, protocol_set: ProtocolTypeSet) -> Self { self.dial_info_filter = self.dial_info_filter.with_protocol_type_set(protocol_set); self @@ -44,7 +44,7 @@ impl NodeRefFilter { self.dial_info_filter = self.dial_info_filter.with_address_type(address_type); self } - #[allow(dead_code)] + #[expect(dead_code)] pub fn with_address_type_set(mut self, address_set: AddressTypeSet) -> Self { self.dial_info_filter = self.dial_info_filter.with_address_type_set(address_set); self @@ -56,12 +56,12 @@ impl NodeRefFilter { .filtered(&other_filter.dial_info_filter); self } - #[allow(dead_code)] + #[expect(dead_code)] pub fn is_dead(&self) -> bool { self.dial_info_filter.is_dead() || self.routing_domain_set.is_empty() } - pub fn with_sequencing(mut self, sequencing: Sequencing) -> (bool, Self) { - let (ordered, dif) = self.dial_info_filter.with_sequencing(sequencing); + pub fn apply_sequencing(mut self, sequencing: Sequencing) -> (bool, Self) { + let (ordered, dif) = self.dial_info_filter.apply_sequencing(sequencing); self.dial_info_filter = dif; (ordered, self) } diff --git a/veilid-core/src/routing_table/node_ref/node_ref_lock.rs b/veilid-core/src/routing_table/node_ref/node_ref_lock.rs new file mode 100644 index 00000000..ec16d185 --- /dev/null +++ b/veilid-core/src/routing_table/node_ref/node_ref_lock.rs @@ -0,0 +1,102 @@ +use super::*; + +pub type LockedNodeRef<'a> = NodeRefLock<'a, NodeRef>; +pub type LockedFilteredNodeRef<'a> = NodeRefLock<'a, FilteredNodeRef>; + +/// Locked reference to a routing table entry +/// For internal use inside the RoutingTable module where you have +/// already locked a RoutingTableInner +/// Keeps entry in the routing table until all references are gone +pub struct NodeRefLock< + 'a, + N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone, +> { + inner: Mutex<&'a RoutingTableInner>, + nr: N, +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + NodeRefLock<'a, N> +{ + pub fn new(inner: &'a RoutingTableInner, nr: N) -> Self { + Self { + inner: Mutex::new(inner), + nr, + } + } + + pub fn unlocked(&self) -> N { + self.nr.clone() + } +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + NodeRefAccessorsTrait for NodeRefLock<'a, N> +{ + fn routing_table(&self) -> RoutingTable { + self.nr.routing_table() + } + fn entry(&self) -> Arc { + self.nr.entry() + } + + fn sequencing(&self) -> Sequencing { + self.nr.sequencing() + } + + fn routing_domain_set(&self) -> RoutingDomainSet { + self.nr.routing_domain_set() + } + + fn filter(&self) -> NodeRefFilter { + self.nr.filter() + } + + fn take_filter(&mut self) -> NodeRefFilter { + self.nr.take_filter() + } + + fn dial_info_filter(&self) -> DialInfoFilter { + self.nr.dial_info_filter() + } +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + NodeRefOperateTrait for NodeRefLock<'a, N> +{ + fn operate(&self, f: F) -> T + where + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, + { + let inner = &*self.inner.lock(); + self.nr.entry().with(inner, f) + } + + fn operate_mut(&self, _f: F) -> T + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, + { + panic!("need to locked_mut() for this operation") + } +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + NodeRefCommonTrait for NodeRefLock<'a, N> +{ +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + fmt::Display for NodeRefLock<'a, N> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.nr) + } +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + fmt::Debug for NodeRefLock<'a, N> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NodeRefLock").field("nr", &self.nr).finish() + } +} diff --git a/veilid-core/src/routing_table/node_ref/node_ref_lock_mut.rs b/veilid-core/src/routing_table/node_ref/node_ref_lock_mut.rs new file mode 100644 index 00000000..568af273 --- /dev/null +++ b/veilid-core/src/routing_table/node_ref/node_ref_lock_mut.rs @@ -0,0 +1,106 @@ +use super::*; + +pub type LockedMutNodeRef<'a> = NodeRefLockMut<'a, NodeRef>; +pub type LockedMutFilteredNodeRef<'a> = NodeRefLockMut<'a, FilteredNodeRef>; + +/// Mutable locked reference to a routing table entry +/// For internal use inside the RoutingTable module where you have +/// already locked a RoutingTableInner +/// Keeps entry in the routing table until all references are gone +pub struct NodeRefLockMut< + 'a, + N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone, +> { + inner: Mutex<&'a mut RoutingTableInner>, + nr: N, +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + NodeRefLockMut<'a, N> +{ + pub fn new(inner: &'a mut RoutingTableInner, nr: N) -> Self { + Self { + inner: Mutex::new(inner), + nr, + } + } + + #[expect(dead_code)] + pub fn unlocked(&self) -> N { + self.nr.clone() + } +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + NodeRefAccessorsTrait for NodeRefLockMut<'a, N> +{ + fn routing_table(&self) -> RoutingTable { + self.nr.routing_table() + } + fn entry(&self) -> Arc { + self.nr.entry() + } + + fn sequencing(&self) -> Sequencing { + self.nr.sequencing() + } + + fn routing_domain_set(&self) -> RoutingDomainSet { + self.nr.routing_domain_set() + } + + fn filter(&self) -> NodeRefFilter { + self.nr.filter() + } + + fn take_filter(&mut self) -> NodeRefFilter { + self.nr.take_filter() + } + + fn dial_info_filter(&self) -> DialInfoFilter { + self.nr.dial_info_filter() + } +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + NodeRefOperateTrait for NodeRefLockMut<'a, N> +{ + fn operate(&self, f: F) -> T + where + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, + { + let inner = &*self.inner.lock(); + self.nr.entry().with(inner, f) + } + + fn operate_mut(&self, f: F) -> T + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, + { + let inner = &mut *self.inner.lock(); + self.nr.entry().with_mut(inner, f) + } +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + NodeRefCommonTrait for NodeRefLockMut<'a, N> +{ +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + fmt::Display for NodeRefLockMut<'a, N> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.nr) + } +} + +impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone> + fmt::Debug for NodeRefLockMut<'a, N> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NodeRefLockMut") + .field("nr", &self.nr) + .finish() + } +} diff --git a/veilid-core/src/routing_table/node_ref/traits.rs b/veilid-core/src/routing_table/node_ref/traits.rs new file mode 100644 index 00000000..2530b3bc --- /dev/null +++ b/veilid-core/src/routing_table/node_ref/traits.rs @@ -0,0 +1,300 @@ +use super::*; + +// Field accessors +pub trait NodeRefAccessorsTrait { + fn routing_table(&self) -> RoutingTable; + fn entry(&self) -> Arc; + fn sequencing(&self) -> Sequencing; + fn routing_domain_set(&self) -> RoutingDomainSet; + fn filter(&self) -> NodeRefFilter; + fn take_filter(&mut self) -> NodeRefFilter; + fn dial_info_filter(&self) -> DialInfoFilter; + // fn node_info_outbound_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter; + // fn is_filter_dead(&self) -> bool; +} + +// Operate on entry +pub trait NodeRefOperateTrait { + fn operate(&self, f: F) -> T + where + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T; + fn operate_mut(&self, f: F) -> T + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T; +} + +// Common Operations +pub trait NodeRefCommonTrait: NodeRefAccessorsTrait + NodeRefOperateTrait { + fn same_entry(&self, other: &T) -> bool { + Arc::ptr_eq(&self.entry(), &other.entry()) + } + + fn same_bucket_entry(&self, entry: &Arc) -> bool { + Arc::ptr_eq(&self.entry(), entry) + } + + fn node_ids(&self) -> TypedKeyGroup { + self.operate(|_rti, e| e.node_ids()) + } + fn best_node_id(&self) -> TypedKey { + self.operate(|_rti, e| e.best_node_id()) + } + + fn update_node_status(&self, routing_domain: RoutingDomain, node_status: NodeStatus) { + self.operate_mut(|_rti, e| { + e.update_node_status(routing_domain, node_status); + }); + } + fn best_routing_domain(&self) -> Option { + self.operate(|rti, e| e.best_routing_domain(rti, self.routing_domain_set())) + } + + // fn envelope_support(&self) -> Vec { + // self.operate(|_rti, e| e.envelope_support()) + // } + fn add_envelope_version(&self, envelope_version: u8) { + self.operate_mut(|_rti, e| e.add_envelope_version(envelope_version)) + } + // fn set_envelope_support(&self, envelope_support: Vec) { + // self.operate_mut(|_rti, e| e.set_envelope_support(envelope_support)) + // } + fn best_envelope_version(&self) -> Option { + self.operate(|_rti, e| e.best_envelope_version()) + } + fn state_reason(&self, cur_ts: Timestamp) -> BucketEntryStateReason { + self.operate(|_rti, e| e.state_reason(cur_ts)) + } + fn state(&self, cur_ts: Timestamp) -> BucketEntryState { + self.operate(|_rti, e| e.state(cur_ts)) + } + fn peer_stats(&self) -> PeerStats { + self.operate(|_rti, e| e.peer_stats().clone()) + } + + fn make_peer_info(&self, routing_domain: RoutingDomain) -> Option { + self.operate(|_rti, e| e.make_peer_info(routing_domain)) + } + fn node_info(&self, routing_domain: RoutingDomain) -> Option { + self.operate(|_rti, e| e.node_info(routing_domain).cloned()) + } + fn signed_node_info_has_valid_signature(&self, routing_domain: RoutingDomain) -> bool { + self.operate(|_rti, e| { + e.signed_node_info(routing_domain) + .map(|sni| sni.has_any_signature()) + .unwrap_or(false) + }) + } + fn node_info_ts(&self, routing_domain: RoutingDomain) -> Timestamp { + self.operate(|_rti, e| { + e.signed_node_info(routing_domain) + .map(|sni| sni.timestamp()) + .unwrap_or(0u64.into()) + }) + } + fn has_seen_our_node_info_ts( + &self, + routing_domain: RoutingDomain, + our_node_info_ts: Timestamp, + ) -> bool { + self.operate(|_rti, e| e.has_seen_our_node_info_ts(routing_domain, our_node_info_ts)) + } + fn set_seen_our_node_info_ts(&self, routing_domain: RoutingDomain, seen_ts: Timestamp) { + self.operate_mut(|_rti, e| e.set_seen_our_node_info_ts(routing_domain, seen_ts)); + } + // fn network_class(&self, routing_domain: RoutingDomain) -> Option { + // self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.network_class())) + // } + // fn outbound_protocols(&self, routing_domain: RoutingDomain) -> Option { + // self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.outbound_protocols())) + // } + // fn address_types(&self, routing_domain: RoutingDomain) -> Option { + // self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.address_types())) + // } + + fn relay(&self, routing_domain: RoutingDomain) -> EyreResult> { + self.operate_mut(|rti, e| { + let Some(sni) = e.signed_node_info(routing_domain) else { + return Ok(None); + }; + let Some(rpi) = sni.relay_peer_info(routing_domain) else { + return Ok(None); + }; + // If relay is ourselves, then return None, because we can't relay through ourselves + // and to contact this node we should have had an existing inbound connection + if rti.unlocked_inner.matches_own_node_id(rpi.node_ids()) { + bail!("Can't relay though ourselves"); + } + + // Register relay node and return noderef + let nr = rti.register_node_with_peer_info(self.routing_table(), rpi, false)?; + Ok(Some(nr)) + }) + } + // DialInfo + fn first_dial_info_detail(&self) -> Option { + let routing_domain_set = self.routing_domain_set(); + let dial_info_filter = self.dial_info_filter(); + let sequencing = self.sequencing(); + let (ordered, dial_info_filter) = dial_info_filter.apply_sequencing(sequencing); + let sort = ordered.then_some(DialInfoDetail::ordered_sequencing_sort); + + if dial_info_filter.is_dead() { + return None; + } + + let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); + + self.operate(|_rt, e| { + for routing_domain in routing_domain_set { + if let Some(ni) = e.node_info(routing_domain) { + if let Some(did) = ni.first_filtered_dial_info_detail(sort, filter) { + return Some(did); + } + } + } + None + }) + } + + fn dial_info_details(&self) -> Vec { + let routing_domain_set = self.routing_domain_set(); + let dial_info_filter = self.dial_info_filter(); + let sequencing = self.sequencing(); + let (ordered, dial_info_filter) = dial_info_filter.apply_sequencing(sequencing); + let sort = ordered.then_some(DialInfoDetail::ordered_sequencing_sort); + + let mut out = Vec::new(); + + if dial_info_filter.is_dead() { + return out; + } + + let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); + + self.operate(|_rt, e| { + for routing_domain in routing_domain_set { + if let Some(ni) = e.node_info(routing_domain) { + let mut dids = ni.filtered_dial_info_details(sort, filter); + out.append(&mut dids); + } + } + }); + out.remove_duplicates(); + out + } + + /// Get the most recent 'last connection' to this node + /// Filtered first and then sorted by ordering preference and then by most recent + fn last_flow(&self) -> Option { + self.operate(|rti, e| { + // apply sequencing to filter and get sort + let sequencing = self.sequencing(); + let filter = self.filter(); + let (ordered, filter) = filter.apply_sequencing(sequencing); + let mut last_flows = e.last_flows(rti, true, filter); + + if ordered { + last_flows.sort_by(|a, b| { + ProtocolType::ordered_sequencing_sort(a.0.protocol_type(), b.0.protocol_type()) + }); + } + + last_flows.first().map(|x| x.0) + }) + } + + /// Get all the 'last connection' flows for this node + #[expect(dead_code)] + fn last_flows(&self) -> Vec { + self.operate(|rti, e| { + // apply sequencing to filter and get sort + let sequencing = self.sequencing(); + let filter = self.filter(); + let (ordered, filter) = filter.apply_sequencing(sequencing); + let mut last_flows = e.last_flows(rti, true, filter); + + if ordered { + last_flows.sort_by(|a, b| { + ProtocolType::ordered_sequencing_sort(a.0.protocol_type(), b.0.protocol_type()) + }); + } + + last_flows.into_iter().map(|x| x.0).collect() + }) + } + + fn clear_last_flows(&self) { + self.operate_mut(|_rti, e| e.clear_last_flows()) + } + + fn set_last_flow(&self, flow: Flow, ts: Timestamp) { + self.operate_mut(|rti, e| { + e.set_last_flow(flow, ts); + rti.touch_recent_peer(e.best_node_id(), flow); + }) + } + + fn clear_last_flow(&self, flow: Flow) { + self.operate_mut(|_rti, e| { + e.remove_last_flow(flow); + }) + } + + fn has_any_dial_info(&self) -> bool { + self.operate(|_rti, e| { + for rtd in RoutingDomain::all() { + if let Some(sni) = e.signed_node_info(rtd) { + if sni.has_any_dial_info() { + return true; + } + } + } + false + }) + } + + fn report_protected_connection_dropped(&self) { + self.stats_failed_to_send(Timestamp::now(), false); + } + + fn report_failed_route_test(&self) { + self.stats_failed_to_send(Timestamp::now(), false); + } + + fn stats_question_sent(&self, ts: Timestamp, bytes: ByteCount, expects_answer: bool) { + self.operate_mut(|rti, e| { + rti.transfer_stats_accounting().add_up(bytes); + e.question_sent(ts, bytes, expects_answer); + }) + } + fn stats_question_rcvd(&self, ts: Timestamp, bytes: ByteCount) { + self.operate_mut(|rti, e| { + rti.transfer_stats_accounting().add_down(bytes); + e.question_rcvd(ts, bytes); + }) + } + fn stats_answer_sent(&self, bytes: ByteCount) { + self.operate_mut(|rti, e| { + rti.transfer_stats_accounting().add_up(bytes); + e.answer_sent(bytes); + }) + } + fn stats_answer_rcvd(&self, send_ts: Timestamp, recv_ts: Timestamp, bytes: ByteCount) { + self.operate_mut(|rti, e| { + rti.transfer_stats_accounting().add_down(bytes); + rti.latency_stats_accounting() + .record_latency(recv_ts.saturating_sub(send_ts)); + e.answer_rcvd(send_ts, recv_ts, bytes); + }) + } + fn stats_question_lost(&self) { + self.operate_mut(|_rti, e| { + e.question_lost(); + }) + } + fn stats_failed_to_send(&self, ts: Timestamp, expects_answer: bool) { + self.operate_mut(|_rti, e| { + e.failed_to_send(ts, expects_answer); + }) + } +} diff --git a/veilid-core/src/routing_table/privacy.rs b/veilid-core/src/routing_table/privacy.rs index e2926823..b15f9b8f 100644 --- a/veilid-core/src/routing_table/privacy.rs +++ b/veilid-core/src/routing_table/privacy.rs @@ -4,7 +4,7 @@ use super::*; // Compiled Privacy Objects /// An encrypted private/safety route hop -#[derive(Clone, Debug)] +#[derive(Clone)] pub(crate) struct RouteHopData { /// The nonce used in the encryption ENC(Xn,DH(PKn,SKapr)) pub nonce: Nonce, @@ -12,13 +12,22 @@ pub(crate) struct RouteHopData { pub blob: Vec, } +impl fmt::Debug for RouteHopData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RouteHopData") + .field("nonce", &self.nonce) + .field("blob", &format!("len={}", self.blob.len())) + .finish() + } +} + /// How to find a route node #[derive(Clone, Debug)] pub(crate) enum RouteNode { /// Route node is optimized, no contact method information as this node id has been seen before NodeId(PublicKey), /// Route node with full contact method information to ensure the peer is reachable - PeerInfo(Box), + PeerInfo(Arc), } impl RouteNode { @@ -47,12 +56,8 @@ impl RouteNode { } RouteNode::PeerInfo(pi) => { // - match routing_table.register_node_with_peer_info( - RoutingDomain::PublicInternet, - *pi.clone(), - false, - ) { - Ok(nr) => Some(nr), + match routing_table.register_node_with_peer_info(pi.clone(), false) { + Ok(nr) => Some(nr.unfiltered()), Err(e) => { log_rtab!(debug "failed to register route node: {}", e); None diff --git a/veilid-core/src/routing_table/route_spec_store/mod.rs b/veilid-core/src/routing_table/route_spec_store/mod.rs index 4fc4d510..4a1bbadb 100644 --- a/veilid-core/src/routing_table/route_spec_store/mod.rs +++ b/veilid-core/src/routing_table/route_spec_store/mod.rs @@ -238,16 +238,14 @@ impl RouteSpecStore { ); } - // Ensure we have a valid network class so our peer info is useful - if !rti.has_valid_network_class(RoutingDomain::PublicInternet) { + // Get our peer info + let Some(published_peer_info) = rti.get_published_peer_info(RoutingDomain::PublicInternet) + else { apibail_try_again!( "unable to allocate route until we have a valid PublicInternet network class" ); }; - // Get our peer info - let our_peer_info = rti.get_own_peer_info(RoutingDomain::PublicInternet); - // Get relay node if we have one let opt_own_relay_nr = rti .relay_node(RoutingDomain::PublicInternet) @@ -297,7 +295,7 @@ impl RouteSpecStore { // or our relay is on their ipblock, or their relay is on our relays same ipblock // our node vs their node - if our_peer_info + if published_peer_info .signed_node_info() .node_info() .node_is_on_same_ipblock(sni.node_info(), ip6_prefix_size) @@ -306,20 +304,22 @@ impl RouteSpecStore { } if let Some(rni) = sni.relay_info() { // our node vs their relay - if our_peer_info + if published_peer_info .signed_node_info() .node_info() .node_is_on_same_ipblock(rni, ip6_prefix_size) { return false; } - if let Some(our_rni) = our_peer_info.signed_node_info().relay_info() { + if let Some(our_rni) = published_peer_info.signed_node_info().relay_info() { // our relay vs their relay if our_rni.node_is_on_same_ipblock(rni, ip6_prefix_size) { return false; } } - } else if let Some(our_rni) = our_peer_info.signed_node_info().relay_info() { + } else if let Some(our_rni) = + published_peer_info.signed_node_info().relay_info() + { // our relay vs their node if our_rni.node_is_on_same_ipblock(sni.node_info(), ip6_prefix_size) { return false; @@ -419,7 +419,7 @@ impl RouteSpecStore { let routing_table = self.unlocked_inner.routing_table.clone(); let transform = |_rti: &RoutingTableInner, entry: Option>| -> NodeRef { - NodeRef::new(routing_table.clone(), entry.unwrap(), None) + NodeRef::new(routing_table.clone(), entry.unwrap()) }; // Pull the whole routing table in sorted order @@ -432,12 +432,14 @@ impl RouteSpecStore { } // Get peer info for everything - let nodes_pi: Vec = nodes + let nodes_pi: Vec> = nodes .iter() .map(|nr| { - nr.locked(rti) - .make_peer_info(RoutingDomain::PublicInternet) - .unwrap() + Arc::new( + nr.locked(rti) + .make_peer_info(RoutingDomain::PublicInternet) + .unwrap(), + ) }) .collect(); @@ -487,14 +489,14 @@ impl RouteSpecStore { // Ensure this route is viable by checking that each node can contact the next one let mut can_do_sequenced = true; if directions.contains(Direction::Outbound) { - let mut previous_node = &our_peer_info; + let mut previous_node = published_peer_info.clone(); let mut reachable = true; for n in permutation { - let current_node = nodes_pi.get(*n).unwrap(); + let current_node = nodes_pi.get(*n).cloned().unwrap(); let cm = rti.get_contact_method( RoutingDomain::PublicInternet, - previous_node, - current_node, + previous_node.clone(), + current_node.clone(), DialInfoFilter::all(), sequencing, None, @@ -508,8 +510,8 @@ impl RouteSpecStore { if can_do_sequenced { let cm = rti.get_contact_method( RoutingDomain::PublicInternet, - previous_node, - current_node, + previous_node.clone(), + current_node.clone(), DialInfoFilter::all(), Sequencing::EnsureOrdered, None, @@ -526,14 +528,14 @@ impl RouteSpecStore { } } if directions.contains(Direction::Inbound) { - let mut next_node = &our_peer_info; + let mut next_node = published_peer_info.clone(); let mut reachable = true; for n in permutation.iter().rev() { - let current_node = nodes_pi.get(*n).unwrap(); + let current_node = nodes_pi.get(*n).cloned().unwrap(); let cm = rti.get_contact_method( RoutingDomain::PublicInternet, - next_node, - current_node, + next_node.clone(), + current_node.clone(), DialInfoFilter::all(), sequencing, None, @@ -547,8 +549,8 @@ impl RouteSpecStore { if can_do_sequenced { let cm = rti.get_contact_method( RoutingDomain::PublicInternet, - next_node, - current_node, + next_node.clone(), + current_node.clone(), DialInfoFilter::all(), Sequencing::EnsureOrdered, None, @@ -710,18 +712,18 @@ impl RouteSpecStore { } #[instrument(level = "trace", target = "route", skip(self), ret, err)] - async fn test_allocated_route(&self, private_route_id: RouteId) -> VeilidAPIResult { + async fn test_allocated_route( + &self, + private_route_id: RouteId, + ) -> VeilidAPIResult> { // Make loopback route to test with let (dest, hops) = { // Get the best allocated route for this id let (key, hops) = { let inner = &mut *self.inner.lock(); let Some(rssd) = inner.content.get_detail(&private_route_id) else { - apibail_invalid_argument!( - "route id not allocated", - "private_route_id", - private_route_id - ); + // Route id is already dead + return Ok(Some(false)); }; let Some(key) = rssd.get_best_route_set_key() else { apibail_internal!("no best key to test allocated route"); @@ -733,7 +735,21 @@ impl RouteSpecStore { }; // Get the private route to send to - let private_route = self.assemble_private_route(&key, None)?; + let private_route = match self.assemble_private_route(&key, None) { + Ok(v) => v, + Err(VeilidAPIError::InvalidTarget { message: _ }) => { + // Route missing means its dead + return Ok(Some(false)); + } + Err(VeilidAPIError::TryAgain { message: _ }) => { + // Try again means we didn't test because we couldnt assemble + return Ok(None); + } + Err(e) => { + return Err(e); + } + }; + // Always test routes with safety routes that are more likely to succeed let stability = Stability::Reliable; // Routes should test with the most likely to succeed sequencing they are capable of @@ -769,15 +785,15 @@ impl RouteSpecStore { for hop in hops { hop.report_failed_route_test(); } - return Ok(false); + return Ok(Some(false)); } }; - Ok(true) + Ok(Some(true)) } #[instrument(level = "trace", target = "route", skip(self), ret, err)] - async fn test_remote_route(&self, private_route_id: RouteId) -> VeilidAPIResult { + async fn test_remote_route(&self, private_route_id: RouteId) -> VeilidAPIResult> { // Make private route test let dest = { // Get the route to test @@ -812,11 +828,11 @@ impl RouteSpecStore { NetworkResult::Value(v) => v, _ => { // Did not error, but did not come back, just return false - return Ok(false); + return Ok(Some(false)); } }; - Ok(true) + Ok(Some(true)) } /// Release an allocated route that is no longer in use @@ -848,7 +864,7 @@ impl RouteSpecStore { /// Test an allocated route for continuity #[instrument(level = "trace", target = "route", skip(self), ret, err)] - pub async fn test_route(&self, id: RouteId) -> VeilidAPIResult { + pub async fn test_route(&self, id: RouteId) -> VeilidAPIResult> { let is_remote = self.is_route_id_remote(&id); if is_remote { self.test_remote_route(id).await @@ -1040,23 +1056,18 @@ impl RouteSpecStore { .lookup_node_ref(routing_table.clone(), TypedKey::new(crypto_kind, id)) .map_err(VeilidAPIError::internal)?, RouteNode::PeerInfo(pi) => Some( - rti.register_node_with_peer_info( - routing_table.clone(), - RoutingDomain::PublicInternet, - *pi, - false, - ) - .map_err(VeilidAPIError::internal)?, + rti.register_node_with_peer_info(routing_table.clone(), pi, false) + .map_err(VeilidAPIError::internal)? + .unfiltered(), ), }; - if opt_first_hop.is_none() { + let Some(first_hop) = opt_first_hop else { // Can't reach this private route any more apibail_generic!("can't reach private route any more"); - } - let mut first_hop = opt_first_hop.unwrap(); + }; // Set sequencing requirement - first_hop.set_sequencing(sequencing); + let mut first_hop = first_hop.sequencing_filtered(sequencing); // Enforce the routing domain first_hop.merge_filter( @@ -1115,10 +1126,10 @@ impl RouteSpecStore { || safety_rssd.get_stats().last_received_ts.is_some(); // Get the first hop noderef of the safety route - let mut first_hop = safety_rssd.hop_node_ref(0).unwrap(); + let first_hop = safety_rssd.hop_node_ref(0).unwrap(); // Ensure sequencing requirement is set on first hop - first_hop.set_sequencing(safety_spec.sequencing); + let mut first_hop = first_hop.sequencing_filtered(safety_spec.sequencing); // Enforce the routing domain first_hop @@ -1201,7 +1212,7 @@ impl RouteSpecStore { if pi.is_none() { apibail_internal!("peer info should exist for route but doesn't"); } - RouteNode::PeerInfo(Box::new(pi.unwrap())) + RouteNode::PeerInfo(Arc::new(pi.unwrap())) }, next_hop: Some(route_hop_data), }; @@ -1386,11 +1397,10 @@ impl RouteSpecStore { }; // Ensure our network class is valid before attempting to assemble any routes - if !rti.has_valid_network_class(RoutingDomain::PublicInternet) { - apibail_try_again!( - "unable to assemble route until we have a valid PublicInternet network class" - ); - } + let Some(published_peer_info) = rti.get_published_peer_info(RoutingDomain::PublicInternet) + else { + apibail_try_again!("unable to assemble route until we have published peerinfo"); + }; // Make innermost route hop to our own node let mut route_hop = RouteHop { @@ -1404,8 +1414,7 @@ impl RouteSpecStore { }; RouteNode::NodeId(node_id.value) } else { - let pi = rti.get_own_peer_info(RoutingDomain::PublicInternet); - RouteNode::PeerInfo(Box::new(pi)) + RouteNode::PeerInfo(published_peer_info) }, next_hop: None, }; @@ -1449,7 +1458,7 @@ impl RouteSpecStore { if pi.is_none() { apibail_internal!("peer info should exist for route but doesn't"); } - RouteNode::PeerInfo(Box::new(pi.unwrap())) + RouteNode::PeerInfo(Arc::new(pi.unwrap())) }, next_hop: Some(route_hop_data), } @@ -1474,7 +1483,8 @@ impl RouteSpecStore { ) -> VeilidAPIResult { let inner = &*self.inner.lock(); let Some(rsid) = inner.content.get_id_by_key(key) else { - apibail_invalid_argument!("route key does not exist", "key", key); + // Route doesn't exist + apibail_invalid_target!("route id does not exist"); }; let Some(rssd) = inner.content.get_detail(&rsid) else { apibail_internal!("route id does not exist"); @@ -1504,7 +1514,7 @@ impl RouteSpecStore { ) -> VeilidAPIResult> { let inner = &*self.inner.lock(); let Some(rssd) = inner.content.get_detail(id) else { - apibail_invalid_argument!("route id does not exist", "id", id); + apibail_invalid_target!("route id does not exist"); }; // See if we can optimize this compilation yet @@ -1529,10 +1539,7 @@ impl RouteSpecStore { let cur_ts = Timestamp::now(); // decode the pr blob - let private_routes = RouteSpecStore::blob_to_private_routes( - self.unlocked_inner.routing_table.crypto(), - blob, - )?; + let private_routes = self.blob_to_private_routes(blob)?; // make the route id let id = self.generate_remote_route_id(&private_routes)?; @@ -1620,7 +1627,11 @@ impl RouteSpecStore { /// Check to see if this remote (not ours) private route has seen our current node info yet /// This happens when you communicate with a private route without a safety route - pub fn has_remote_private_route_seen_our_node_info(&self, key: &PublicKey) -> bool { + pub fn has_remote_private_route_seen_our_node_info( + &self, + key: &PublicKey, + published_peer_info: &PeerInfo, + ) -> bool { let inner = &*self.inner.lock(); // Check for local route. If this is not a remote private route, @@ -1633,18 +1644,15 @@ impl RouteSpecStore { if let Some(rrid) = inner.cache.get_remote_private_route_id_by_key(key) { let cur_ts = Timestamp::now(); if let Some(rpri) = inner.cache.peek_remote_private_route(cur_ts, &rrid) { - let our_node_info_ts = self - .unlocked_inner - .routing_table - .get_own_node_info_ts(RoutingDomain::PublicInternet); - return rpri.has_seen_our_node_info_ts(our_node_info_ts); + return rpri + .has_seen_our_node_info_ts(published_peer_info.signed_node_info().timestamp()); } } false } - /// Mark a remote private route as having seen our current node info + /// Mark a remote private route as having seen our current published node info /// PRIVACY: /// We do not accept node info timestamps from remote private routes because this would /// enable a deanonymization attack, whereby a node could be 'pinged' with a doctored node_info with a @@ -1655,10 +1663,14 @@ impl RouteSpecStore { key: &PublicKey, cur_ts: Timestamp, ) -> VeilidAPIResult<()> { - let our_node_info_ts = self + let Some(our_node_info_ts) = self .unlocked_inner .routing_table - .get_own_node_info_ts(RoutingDomain::PublicInternet); + .get_published_peer_info(RoutingDomain::PublicInternet) + .map(|pi| pi.signed_node_info().timestamp()) + else { + apibail_internal!("peer info is not yet published"); + }; let inner = &mut *self.inner.lock(); @@ -1676,7 +1688,7 @@ impl RouteSpecStore { } } - apibail_invalid_argument!("private route is missing from store", "key", key); + apibail_invalid_target!("private route is missing from store"); } /// Get the route statistics for any route we know about, local or remote @@ -1732,7 +1744,7 @@ impl RouteSpecStore { pub fn mark_route_published(&self, id: &RouteId, published: bool) -> VeilidAPIResult<()> { let inner = &mut *self.inner.lock(); let Some(rssd) = inner.content.get_detail_mut(id) else { - apibail_invalid_argument!("route does not exist", "id", id); + apibail_invalid_target!("route does not exist"); }; rssd.set_published(published); Ok(()) @@ -1776,10 +1788,10 @@ impl RouteSpecStore { } /// Convert binary blob to private route vector - pub fn blob_to_private_routes( - crypto: Crypto, - blob: Vec, - ) -> VeilidAPIResult> { + pub fn blob_to_private_routes(&self, blob: Vec) -> VeilidAPIResult> { + // Get crypto + let crypto = self.unlocked_inner.routing_table.crypto(); + // Deserialize count if blob.is_empty() { apibail_invalid_argument!( @@ -1795,6 +1807,9 @@ impl RouteSpecStore { } // Deserialize stream of private routes + let decode_context = RPCDecodeContext { + routing_domain: RoutingDomain::PublicInternet, + }; let mut pr_slice = &blob[1..]; let mut out = Vec::with_capacity(pr_count); for _ in 0..pr_count { @@ -1807,7 +1822,7 @@ impl RouteSpecStore { let pr_reader = reader .get_root::() .map_err(VeilidAPIError::internal)?; - let private_route = decode_private_route(&pr_reader).map_err(|e| { + let private_route = decode_private_route(&decode_context, &pr_reader).map_err(|e| { VeilidAPIError::invalid_argument("failed to decode private route", "e", e) })?; private_route.validate(crypto.clone()).map_err(|e| { diff --git a/veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs b/veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs index 35db90a7..75bf4e70 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs @@ -15,7 +15,8 @@ pub(crate) struct CompiledRoute { /// The secret used to encrypt the message payload pub secret: SecretKey, /// The node ref to the first hop in the compiled route - pub first_hop: NodeRef, + /// filtered to the safetyselection it was compiled with + pub first_hop: FilteredNodeRef, } /// Ephemeral data used to help the RouteSpecStore operate efficiently diff --git a/veilid-core/src/routing_table/route_spec_store/route_stats.rs b/veilid-core/src/routing_table/route_spec_store/route_stats.rs index c8d390c7..c224ced0 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_stats.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_stats.rs @@ -94,7 +94,7 @@ impl RouteStats { } /// Get the transfer stats - #[allow(dead_code)] + #[expect(dead_code)] pub fn transfer_stats(&self) -> &TransferStatsDownUp { &self.transfer_stats_down_up } diff --git a/veilid-core/src/routing_table/routing_domain_editor.rs b/veilid-core/src/routing_table/routing_domain_editor.rs deleted file mode 100644 index 30fe97df..00000000 --- a/veilid-core/src/routing_table/routing_domain_editor.rs +++ /dev/null @@ -1,268 +0,0 @@ -use super::*; - -#[derive(Debug)] -enum RoutingDomainChange { - ClearDialInfoDetails { - address_type: Option, - protocol_type: Option, - }, - ClearRelayNode, - SetRelayNode { - relay_node: NodeRef, - }, - SetRelayNodeKeepalive { - ts: Option, - }, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - AddDialInfoDetail { - dial_info_detail: DialInfoDetail, - }, - SetupNetwork { - outbound_protocols: ProtocolTypeSet, - inbound_protocols: ProtocolTypeSet, - address_types: AddressTypeSet, - capabilities: Vec, - }, - SetNetworkClass { - network_class: Option, - }, -} - -pub(crate) struct RoutingDomainEditor { - routing_table: RoutingTable, - routing_domain: RoutingDomain, - changes: Vec, -} - -impl RoutingDomainEditor { - pub(super) fn new(routing_table: RoutingTable, routing_domain: RoutingDomain) -> Self { - Self { - routing_table, - routing_domain, - changes: Vec::new(), - } - } - - #[instrument(level = "debug", skip(self))] - pub fn clear_dial_info_details( - &mut self, - address_type: Option, - protocol_type: Option, - ) -> &mut Self { - self.changes - .push(RoutingDomainChange::ClearDialInfoDetails { - address_type, - protocol_type, - }); - self - } - #[instrument(level = "debug", skip(self))] - pub fn clear_relay_node(&mut self) -> &mut Self { - self.changes.push(RoutingDomainChange::ClearRelayNode); - self - } - #[instrument(level = "debug", skip(self))] - pub fn set_relay_node(&mut self, relay_node: NodeRef) -> &mut Self { - self.changes - .push(RoutingDomainChange::SetRelayNode { relay_node }); - self - } - #[instrument(level = "debug", skip(self))] - pub fn set_relay_node_keepalive(&mut self, ts: Option) -> &mut Self { - self.changes - .push(RoutingDomainChange::SetRelayNodeKeepalive { ts }); - self - } - #[instrument(level = "debug", skip(self))] - pub fn register_dial_info( - &mut self, - dial_info: DialInfo, - class: DialInfoClass, - ) -> EyreResult<&mut Self> { - if !self - .routing_table - .ensure_dial_info_is_valid(self.routing_domain, &dial_info) - { - return Err(eyre!( - "dial info '{}' is not valid in routing domain '{:?}'", - dial_info, - self.routing_domain - )); - } - - self.changes.push(RoutingDomainChange::AddDialInfoDetail { - dial_info_detail: DialInfoDetail { - dial_info: dial_info.clone(), - class, - }, - }); - - Ok(self) - } - #[instrument(level = "debug", skip(self))] - pub fn setup_network( - &mut self, - outbound_protocols: ProtocolTypeSet, - inbound_protocols: ProtocolTypeSet, - address_types: AddressTypeSet, - capabilities: Vec, - ) -> &mut Self { - self.changes.push(RoutingDomainChange::SetupNetwork { - outbound_protocols, - inbound_protocols, - address_types, - capabilities, - }); - self - } - - #[instrument(level = "debug", skip(self))] - pub fn set_network_class(&mut self, network_class: Option) -> &mut Self { - self.changes - .push(RoutingDomainChange::SetNetworkClass { network_class }); - self - } - - #[instrument(level = "debug", skip(self))] - pub async fn commit(&mut self, pause_tasks: bool) { - // No locking if we have nothing to do - if self.changes.is_empty() { - return; - } - - // Briefly pause routing table ticker while changes are made - let _tick_guard = if pause_tasks { - Some(self.routing_table.pause_tasks().await) - } else { - None - }; - - // Apply changes - log_rtab!("[{:?}] COMMIT: {:?}", self.routing_domain, self.changes); - let mut peer_info_changed = false; - { - let mut inner = self.routing_table.inner.write(); - inner.with_routing_domain_mut(self.routing_domain, |detail| { - for change in self.changes.drain(..) { - match change { - RoutingDomainChange::ClearDialInfoDetails { - address_type, - protocol_type, - } => { - if !detail.common_mut().dial_info_details().is_empty() { - if address_type.is_some() || protocol_type.is_some() { - info!( - "[{:?}] cleared dial info: {}:{}", - self.routing_domain, - address_type - .map(|at| format!("{:?}", at)) - .unwrap_or("---".to_string()), - protocol_type - .map(|at| format!("{:?}", at)) - .unwrap_or("---".to_string()), - ); - } else { - info!("[{:?}] cleared all dial info", self.routing_domain); - } - } - detail - .common_mut() - .clear_dial_info_details(address_type, protocol_type); - peer_info_changed = true; - } - RoutingDomainChange::ClearRelayNode => { - if detail.common_mut().relay_node().is_some() { - info!("[{:?}] cleared relay node", self.routing_domain); - } - detail.common_mut().set_relay_node(None); - peer_info_changed = true; - } - RoutingDomainChange::SetRelayNode { relay_node } => { - info!("[{:?}] set relay node: {}", self.routing_domain, relay_node); - detail.common_mut().set_relay_node(Some(relay_node.clone())); - peer_info_changed = true; - } - RoutingDomainChange::SetRelayNodeKeepalive { ts } => { - trace!("[{:?}] relay node keepalive: {:?}", self.routing_domain, ts); - detail.common_mut().set_relay_node_last_keepalive(ts); - } - RoutingDomainChange::AddDialInfoDetail { dial_info_detail } => { - info!( - "[{:?}] dial info: {:?}:{}", - self.routing_domain, - dial_info_detail.class, - dial_info_detail.dial_info - ); - detail - .common_mut() - .add_dial_info_detail(dial_info_detail.clone()); - - peer_info_changed = true; - } - RoutingDomainChange::SetupNetwork { - outbound_protocols, - inbound_protocols, - address_types, - capabilities, - } => { - let old_outbound_protocols = detail.common().outbound_protocols(); - let old_inbound_protocols = detail.common().inbound_protocols(); - let old_address_types = detail.common().address_types(); - let old_capabilities = detail.common().capabilities(); - - let this_changed = old_outbound_protocols != outbound_protocols - || old_inbound_protocols != inbound_protocols - || old_address_types != address_types - || old_capabilities != *capabilities; - - if this_changed { - info!( - "[{:?}] setup network: outbound {:?} inbound {:?} address types {:?} capabilities {:?}", - self.routing_domain, - outbound_protocols, - inbound_protocols, - address_types, - capabilities - ); - - detail.common_mut().setup_network( - outbound_protocols, - inbound_protocols, - address_types, - capabilities.clone(), - ); - peer_info_changed = true; - } - } - RoutingDomainChange::SetNetworkClass { network_class } => { - let old_network_class = detail.common().network_class(); - - let this_changed = old_network_class != network_class; - if this_changed { - if let Some(network_class) = network_class { - info!( - "[{:?}] set network class: {:?}", - self.routing_domain, network_class, - ); - } else { - info!("[{:?}] cleared network class", self.routing_domain,); - } - detail.common_mut().set_network_class(network_class); - peer_info_changed = true; - } - } - } - } - }); - if peer_info_changed { - // Allow signed node info updates at same timestamp for otherwise dead nodes if our network has changed - inner.reset_all_updated_since_last_network_change(); - } - } - // Clear the routespecstore cache if our PublicInternet dial info has changed - if peer_info_changed && self.routing_domain == RoutingDomain::PublicInternet { - let rss = self.routing_table.route_spec_store(); - rss.reset(); - } - } -} diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs deleted file mode 100644 index f03dc669..00000000 --- a/veilid-core/src/routing_table/routing_domains.rs +++ /dev/null @@ -1,622 +0,0 @@ -use super::*; - -/// Mechanism required to contact another node -#[derive(Clone, Debug)] -pub(crate) enum ContactMethod { - /// Node is not reachable by any means - Unreachable, - /// Connection should have already existed - Existing, - /// Contact the node directly - Direct(DialInfo), - /// Request via signal the node connect back directly (relay, target) - SignalReverse(TypedKey, TypedKey), - /// Request via signal the node negotiate a hole punch (relay, target) - SignalHolePunch(TypedKey, TypedKey), - /// Must use an inbound relay to reach the node - InboundRelay(TypedKey), - /// Must use outbound relay to reach the node - OutboundRelay(TypedKey), -} - -#[derive(Debug)] -pub(crate) struct RoutingDomainDetailCommon { - routing_domain: RoutingDomain, - network_class: Option, - outbound_protocols: ProtocolTypeSet, - inbound_protocols: ProtocolTypeSet, - address_types: AddressTypeSet, - relay_node: Option, - relay_node_last_keepalive: Option, - capabilities: Vec, - dial_info_details: Vec, - // caches - cached_peer_info: Mutex>, -} - -impl RoutingDomainDetailCommon { - pub fn new(routing_domain: RoutingDomain) -> Self { - Self { - routing_domain, - network_class: Default::default(), - outbound_protocols: Default::default(), - inbound_protocols: Default::default(), - address_types: Default::default(), - relay_node: Default::default(), - relay_node_last_keepalive: Default::default(), - capabilities: Default::default(), - dial_info_details: Default::default(), - cached_peer_info: Mutex::new(Default::default()), - } - } - - // Set from network manager - pub(super) fn setup_network( - &mut self, - outbound_protocols: ProtocolTypeSet, - inbound_protocols: ProtocolTypeSet, - address_types: AddressTypeSet, - capabilities: Vec, - ) { - self.outbound_protocols = outbound_protocols; - self.inbound_protocols = inbound_protocols; - self.address_types = address_types; - self.capabilities = capabilities; - self.clear_cache(); - } - - pub(super) fn set_network_class(&mut self, network_class: Option) { - self.network_class = network_class; - self.clear_cache(); - } - pub fn network_class(&self) -> Option { - self.network_class - } - pub fn outbound_protocols(&self) -> ProtocolTypeSet { - self.outbound_protocols - } - pub fn inbound_protocols(&self) -> ProtocolTypeSet { - self.inbound_protocols - } - pub fn address_types(&self) -> AddressTypeSet { - self.address_types - } - pub fn capabilities(&self) -> Vec { - self.capabilities.clone() - } - pub fn relay_node(&self) -> Option { - self.relay_node.clone() - } - pub fn relay_node_last_keepalive(&self) -> Option { - self.relay_node_last_keepalive - } - pub(super) fn set_relay_node(&mut self, opt_relay_node: Option) { - self.relay_node = opt_relay_node.map(|nr| { - nr.filtered_clone(NodeRefFilter::new().with_routing_domain(self.routing_domain)) - }); - self.relay_node_last_keepalive = None; - self.clear_cache(); - } - pub(super) fn set_relay_node_last_keepalive(&mut self, ts: Option) { - self.relay_node_last_keepalive = ts; - } - pub fn dial_info_details(&self) -> &Vec { - &self.dial_info_details - } - pub(super) fn clear_dial_info_details( - &mut self, - address_type: Option, - protocol_type: Option, - ) { - self.dial_info_details.retain_mut(|e| { - let mut remove = true; - if let Some(pt) = protocol_type { - if pt != e.dial_info.protocol_type() { - remove = false; - } - } - if let Some(at) = address_type { - if at != e.dial_info.address_type() { - remove = false; - } - } - !remove - }); - self.clear_cache(); - } - pub(super) fn add_dial_info_detail(&mut self, did: DialInfoDetail) { - self.dial_info_details.push(did); - self.dial_info_details.sort(); - self.clear_cache(); - } - - pub fn has_valid_network_class(&self) -> bool { - self.network_class.unwrap_or(NetworkClass::Invalid) != NetworkClass::Invalid - } - - fn make_peer_info(&self, rti: &RoutingTableInner) -> PeerInfo { - let node_info = NodeInfo::new( - self.network_class.unwrap_or(NetworkClass::Invalid), - self.outbound_protocols, - self.address_types, - VALID_ENVELOPE_VERSIONS.to_vec(), - VALID_CRYPTO_KINDS.to_vec(), - self.capabilities.clone(), - self.dial_info_details.clone(), - ); - - let relay_info = if let Some(rn) = &self.relay_node { - let opt_relay_pi = rn.locked(rti).make_peer_info(self.routing_domain); - if let Some(relay_pi) = opt_relay_pi { - let (relay_ids, relay_sni) = relay_pi.destructure(); - match relay_sni { - SignedNodeInfo::Direct(d) => Some((relay_ids, d)), - SignedNodeInfo::Relayed(_) => { - warn!("relay node should not have a relay itself! if this happens, a relay updated its signed node info and became a relay, which should cause the relay to be dropped"); - None - } - } - } else { - None - } - } else { - None - }; - - let signed_node_info = match relay_info { - Some((relay_ids, relay_sdni)) => SignedNodeInfo::Relayed( - SignedRelayedNodeInfo::make_signatures( - rti.unlocked_inner.crypto(), - rti.unlocked_inner.node_id_typed_key_pairs(), - node_info, - relay_ids, - relay_sdni, - ) - .unwrap(), - ), - None => SignedNodeInfo::Direct( - SignedDirectNodeInfo::make_signatures( - rti.unlocked_inner.crypto(), - rti.unlocked_inner.node_id_typed_key_pairs(), - node_info, - ) - .unwrap(), - ), - }; - - PeerInfo::new(rti.unlocked_inner.node_ids(), signed_node_info) - } - - pub fn with_peer_info(&self, rti: &RoutingTableInner, f: F) -> R - where - F: FnOnce(&PeerInfo) -> R, - { - let mut cpi = self.cached_peer_info.lock(); - if cpi.is_none() { - // Regenerate peer info - let pi = self.make_peer_info(rti); - - // Cache the peer info - *cpi = Some(pi); - } - f(cpi.as_ref().unwrap()) - } - - #[allow(dead_code)] - pub fn inbound_dial_info_filter(&self) -> DialInfoFilter { - DialInfoFilter::all() - .with_protocol_type_set(self.inbound_protocols) - .with_address_type_set(self.address_types) - } - pub fn outbound_dial_info_filter(&self) -> DialInfoFilter { - DialInfoFilter::all() - .with_protocol_type_set(self.outbound_protocols) - .with_address_type_set(self.address_types) - } - - pub(super) fn clear_cache(&self) { - *self.cached_peer_info.lock() = None; - } -} - -/// General trait for all routing domains -pub(crate) trait RoutingDomainDetail { - // Common accessors - fn common(&self) -> &RoutingDomainDetailCommon; - fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon; - - /// Can this routing domain contain a particular address - fn can_contain_address(&self, address: Address) -> bool; - - /// Get the contact method required for node A to reach node B in this routing domain - /// Routing table must be locked for reading to use this function - fn get_contact_method( - &self, - rti: &RoutingTableInner, - peer_a: &PeerInfo, - peer_b: &PeerInfo, - dial_info_filter: DialInfoFilter, - sequencing: Sequencing, - dif_sort: Option>, - ) -> ContactMethod; -} - -///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Public Internet routing domain internals -#[derive(Debug)] -pub struct PublicInternetRoutingDomainDetail { - /// Common implementation for all routing domains - common: RoutingDomainDetailCommon, -} - -impl Default for PublicInternetRoutingDomainDetail { - fn default() -> Self { - Self { - common: RoutingDomainDetailCommon::new(RoutingDomain::PublicInternet), - } - } -} - -fn first_filtered_dial_info_detail_between_nodes( - from_node: &NodeInfo, - to_node: &NodeInfo, - dial_info_filter: &DialInfoFilter, - sequencing: Sequencing, - dif_sort: Option>, -) -> Option { - // Consider outbound capabilities - let dial_info_filter = (*dial_info_filter).filtered( - &DialInfoFilter::all() - .with_address_type_set(from_node.address_types()) - .with_protocol_type_set(from_node.outbound_protocols()), - ); - - // Apply sequencing and get sort - // Include sorting by external dial info sort for rotating through dialinfo - // based on an external preference table, for example the one kept by - // AddressFilter to deprioritize dialinfo that have recently failed to connect - let (ordered, dial_info_filter) = dial_info_filter.with_sequencing(sequencing); - let sort: Option> = if ordered { - if let Some(dif_sort) = dif_sort { - Some(Box::new(move |a, b| { - let mut ord = dif_sort(a, b); - if ord == core::cmp::Ordering::Equal { - ord = DialInfoDetail::ordered_sequencing_sort(a, b); - } - ord - })) - } else { - Some(Box::new(move |a, b| { - DialInfoDetail::ordered_sequencing_sort(a, b) - })) - } - } else if let Some(dif_sort) = dif_sort { - Some(Box::new(move |a, b| dif_sort(a, b))) - } else { - None - }; - - // If the filter is dead then we won't be able to connect - if dial_info_filter.is_dead() { - return None; - } - - // Get the best match dial info for node B if we have it - let direct_filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); - to_node.first_filtered_dial_info_detail(sort, direct_filter) -} - -impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { - fn common(&self) -> &RoutingDomainDetailCommon { - &self.common - } - fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon { - &mut self.common - } - fn can_contain_address(&self, address: Address) -> bool { - address.is_global() - } - fn get_contact_method( - &self, - rti: &RoutingTableInner, - peer_a: &PeerInfo, - peer_b: &PeerInfo, - dial_info_filter: DialInfoFilter, - sequencing: Sequencing, - dif_sort: Option>, - ) -> ContactMethod { - let ip6_prefix_size = rti - .unlocked_inner - .config - .get() - .network - .max_connections_per_ip6_prefix_size as usize; - - // Get the nodeinfos for convenience - let node_a = peer_a.signed_node_info().node_info(); - let node_b = peer_b.signed_node_info().node_info(); - - // Check to see if these nodes are on the same network - let same_ipblock = node_a.node_is_on_same_ipblock(node_b, ip6_prefix_size); - - // Get the node ids that would be used between these peers - let cck = common_crypto_kinds(&peer_a.node_ids().kinds(), &peer_b.node_ids().kinds()); - let Some(best_ck) = cck.first().copied() else { - // No common crypto kinds between these nodes, can't contact - return ContactMethod::Unreachable; - }; - - //let node_a_id = peer_a.node_ids().get(best_ck).unwrap(); - let node_b_id = peer_b.node_ids().get(best_ck).unwrap(); - - // Get the best match dial info for node B if we have it - // Don't try direct inbound at all if the two nodes are on the same ipblock to avoid hairpin NAT issues - // as well avoiding direct traffic between same-network nodes. This would be done in the LocalNetwork RoutingDomain. - if let Some(target_did) = (!same_ipblock) - .then(|| { - first_filtered_dial_info_detail_between_nodes( - node_a, - node_b, - &dial_info_filter, - sequencing, - dif_sort.clone(), - ) - }) - .flatten() - { - // Do we need to signal before going inbound? - if !target_did.class.requires_signal() { - // Go direct without signaling - return ContactMethod::Direct(target_did.dial_info); - } - - // Get the target's inbound relay, it must have one or it is not reachable - if let Some(node_b_relay) = peer_b.signed_node_info().relay_info() { - // Note that relay_peer_info could be node_a, in which case a connection already exists - // and we only get here if the connection had dropped, in which case node_a is unreachable until - // it gets a new relay connection up - if peer_b - .signed_node_info() - .relay_ids() - .contains_any(peer_a.node_ids()) - { - return ContactMethod::Existing; - } - - // Get best node id to contact relay with - let Some(node_b_relay_id) = peer_b.signed_node_info().relay_ids().get(best_ck) - else { - // No best relay id - return ContactMethod::Unreachable; - }; - - // Can node A reach the inbound relay directly? - if first_filtered_dial_info_detail_between_nodes( - node_a, - node_b_relay, - &dial_info_filter, - sequencing, - dif_sort.clone(), - ) - .is_some() - { - // Can node A receive anything inbound ever? - if matches!(node_a.network_class(), NetworkClass::InboundCapable) { - ///////// Reverse connection - - // Get the best match dial info for an reverse inbound connection from node B to node A - if let Some(reverse_did) = first_filtered_dial_info_detail_between_nodes( - node_b, - node_a, - &dial_info_filter, - sequencing, - dif_sort.clone(), - ) { - // Ensure we aren't on the same public IP address (no hairpin nat) - if reverse_did.dial_info.ip_addr() != target_did.dial_info.ip_addr() { - // Can we receive a direct reverse connection? - if !reverse_did.class.requires_signal() { - return ContactMethod::SignalReverse( - node_b_relay_id, - node_b_id, - ); - } - } - } - - ///////// UDP hole-punch - - // Does node B have a direct udp dialinfo node A can reach? - let udp_dial_info_filter = dial_info_filter - .filtered(&DialInfoFilter::all().with_protocol_type(ProtocolType::UDP)); - if let Some(target_udp_did) = first_filtered_dial_info_detail_between_nodes( - node_a, - node_b, - &udp_dial_info_filter, - sequencing, - dif_sort.clone(), - ) { - // Does node A have a direct udp dialinfo that node B can reach? - if let Some(reverse_udp_did) = - first_filtered_dial_info_detail_between_nodes( - node_b, - node_a, - &udp_dial_info_filter, - sequencing, - dif_sort.clone(), - ) - { - // Ensure we aren't on the same public IP address (no hairpin nat) - if reverse_udp_did.dial_info.ip_addr() - != target_udp_did.dial_info.ip_addr() - { - // The target and ourselves have a udp dialinfo that they can reach - return ContactMethod::SignalHolePunch( - node_b_relay_id, - node_b_id, - ); - } - } - } - // Otherwise we have to inbound relay - } - - return ContactMethod::InboundRelay(node_b_relay_id); - } - } - } - // If the node B has no direct dial info or is on the same ipblock, it needs to have an inbound relay - else if let Some(node_b_relay) = peer_b.signed_node_info().relay_info() { - // Note that relay_peer_info could be node_a, in which case a connection already exists - // and we only get here if the connection had dropped, in which case node_b is unreachable until - // it gets a new relay connection up - if peer_b - .signed_node_info() - .relay_ids() - .contains_any(peer_a.node_ids()) - { - return ContactMethod::Existing; - } - - // Get best node id to contact relay with - let Some(node_b_relay_id) = peer_b.signed_node_info().relay_ids().get(best_ck) else { - // No best relay id - return ContactMethod::Unreachable; - }; - - // Can we reach the inbound relay? - if first_filtered_dial_info_detail_between_nodes( - node_a, - node_b_relay, - &dial_info_filter, - sequencing, - dif_sort.clone(), - ) - .is_some() - { - ///////// Reverse connection - - // Get the best match dial info for an reverse inbound connection from node B to node A - // unless both nodes are on the same ipblock - if let Some(reverse_did) = (!same_ipblock) - .then(|| { - first_filtered_dial_info_detail_between_nodes( - node_b, - node_a, - &dial_info_filter, - sequencing, - dif_sort.clone(), - ) - }) - .flatten() - { - // Can we receive a direct reverse connection? - if !reverse_did.class.requires_signal() { - return ContactMethod::SignalReverse(node_b_relay_id, node_b_id); - } - } - - return ContactMethod::InboundRelay(node_b_relay_id); - } - } - - // If node A can't reach the node by other means, it may need to use its outbound relay - if peer_a - .signed_node_info() - .node_info() - .network_class() - .outbound_wants_relay() - { - if let Some(node_a_relay_id) = peer_a.signed_node_info().relay_ids().get(best_ck) { - // Ensure it's not our relay we're trying to reach - if node_a_relay_id != node_b_id { - return ContactMethod::OutboundRelay(node_a_relay_id); - } - } - } - - ContactMethod::Unreachable - } -} - -/// Local Network routing domain internals -#[derive(Debug)] -pub struct LocalNetworkRoutingDomainDetail { - /// The local networks this domain will communicate with - local_networks: Vec<(IpAddr, IpAddr)>, - /// Common implementation for all routing domains - common: RoutingDomainDetailCommon, -} - -impl Default for LocalNetworkRoutingDomainDetail { - fn default() -> Self { - Self { - local_networks: Default::default(), - common: RoutingDomainDetailCommon::new(RoutingDomain::LocalNetwork), - } - } -} - -impl LocalNetworkRoutingDomainDetail { - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - pub fn set_local_networks(&mut self, mut local_networks: Vec<(IpAddr, IpAddr)>) -> bool { - local_networks.sort(); - if local_networks == self.local_networks { - return false; - } - self.local_networks = local_networks; - true - } -} - -impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { - fn common(&self) -> &RoutingDomainDetailCommon { - &self.common - } - fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon { - &mut self.common - } - fn can_contain_address(&self, address: Address) -> bool { - let ip = address.ip_addr(); - for localnet in &self.local_networks { - if ipaddr_in_network(ip, localnet.0, localnet.1) { - return true; - } - } - false - } - - fn get_contact_method( - &self, - _rti: &RoutingTableInner, - peer_a: &PeerInfo, - peer_b: &PeerInfo, - dial_info_filter: DialInfoFilter, - sequencing: Sequencing, - dif_sort: Option>, - ) -> ContactMethod { - // Get the nodeinfos for convenience - let node_a = peer_a.signed_node_info().node_info(); - let node_b = peer_b.signed_node_info().node_info(); - - // Get the node ids that would be used between these peers - let cck = common_crypto_kinds(&peer_a.node_ids().kinds(), &peer_b.node_ids().kinds()); - let Some(_best_ck) = cck.first().copied() else { - // No common crypto kinds between these nodes, can't contact - return ContactMethod::Unreachable; - }; - - if let Some(target_did) = first_filtered_dial_info_detail_between_nodes( - node_a, - node_b, - &dial_info_filter, - sequencing, - dif_sort, - ) { - return ContactMethod::Direct(target_did.dial_info); - } - - ContactMethod::Unreachable - } -} diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner/mod.rs similarity index 87% rename from veilid-core/src/routing_table/routing_table_inner.rs rename to veilid-core/src/routing_table/routing_table_inner/mod.rs index 185faa88..e39cc115 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner/mod.rs @@ -1,8 +1,15 @@ +mod routing_domains; + use super::*; +pub use routing_domains::*; + use weak_table::PtrWeakHashSet; pub const RECENT_PEERS_TABLE_SIZE: usize = 64; +// Critical sections +pub const LOCK_TAG_TICK: &str = "TICK"; + pub type EntryCounts = BTreeMap<(RoutingDomain, CryptoKind), usize>; ////////////////////////////////////////////////////////////////////////// @@ -87,34 +94,36 @@ impl RoutingTableInner { } } - pub fn with_routing_domain_mut(&mut self, domain: RoutingDomain, f: F) -> R + fn with_public_internet_routing_domain_mut(&mut self, f: F) -> R where - F: FnOnce(&mut dyn RoutingDomainDetail) -> R, + F: FnOnce(&mut PublicInternetRoutingDomainDetail) -> R, { - match domain { - RoutingDomain::PublicInternet => f(&mut self.public_internet_routing_domain), - RoutingDomain::LocalNetwork => f(&mut self.local_network_routing_domain), - } + f(&mut self.public_internet_routing_domain) + } + fn with_local_network_routing_domain_mut(&mut self, f: F) -> R + where + F: FnOnce(&mut LocalNetworkRoutingDomainDetail) -> R, + { + f(&mut self.local_network_routing_domain) } - pub fn relay_node(&self, domain: RoutingDomain) -> Option { - self.with_routing_domain(domain, |rd| rd.common().relay_node()) + pub fn relay_node(&self, domain: RoutingDomain) -> Option { + self.with_routing_domain(domain, |rdd| rdd.relay_node()) } pub fn relay_node_last_keepalive(&self, domain: RoutingDomain) -> Option { - self.with_routing_domain(domain, |rd| rd.common().relay_node_last_keepalive()) + self.with_routing_domain(domain, |rdd| rdd.relay_node_last_keepalive()) } - #[allow(dead_code)] + #[expect(dead_code)] pub fn has_dial_info(&self, domain: RoutingDomain) -> bool { - self.with_routing_domain(domain, |rd| !rd.common().dial_info_details().is_empty()) + self.with_routing_domain(domain, |rdd| !rdd.dial_info_details().is_empty()) } pub fn dial_info_details(&self, domain: RoutingDomain) -> Vec { - self.with_routing_domain(domain, |rd| rd.common().dial_info_details().clone()) + self.with_routing_domain(domain, |rdd| rdd.dial_info_details().clone()) } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] pub fn first_filtered_dial_info_detail( &self, routing_domain_set: RoutingDomainSet, @@ -124,8 +133,8 @@ impl RoutingTableInner { return None; } for routing_domain in routing_domain_set { - let did = self.with_routing_domain(routing_domain, |rd| { - for did in rd.common().dial_info_details() { + let did = self.with_routing_domain(routing_domain, |rdd| { + for did in rdd.dial_info_details() { if did.matches_filter(filter) { return Some(did.clone()); } @@ -149,8 +158,8 @@ impl RoutingTableInner { return ret; } for routing_domain in routing_domain_set { - self.with_routing_domain(routing_domain, |rd| { - for did in rd.common().dial_info_details() { + self.with_routing_domain(routing_domain, |rdd| { + for did in rdd.dial_info_details() { if did.matches_filter(filter) { ret.push(did.clone()); } @@ -161,37 +170,20 @@ impl RoutingTableInner { ret } - pub fn ensure_dial_info_is_valid(&self, domain: RoutingDomain, dial_info: &DialInfo) -> bool { - let address = dial_info.socket_address().address(); - let can_contain_address = - self.with_routing_domain(domain, |rd| rd.can_contain_address(address)); - - if !can_contain_address { - log_rtab!(debug "can not add dial info to this routing domain"); - return false; - } - if !dial_info.is_valid() { - log_rtab!(debug - "shouldn't be registering invalid addresses: {:?}", - dial_info - ); - return false; - } - true - } - pub fn node_info_is_valid_in_routing_domain( &self, routing_domain: RoutingDomain, node_info: &NodeInfo, ) -> bool { // Ensure all of the dial info works in this routing domain - for did in node_info.dial_info_detail_list() { - if !self.ensure_dial_info_is_valid(routing_domain, &did.dial_info) { - return false; + self.with_routing_domain(routing_domain, |rdd| { + for did in node_info.dial_info_detail_list() { + if !rdd.ensure_dial_info_is_valid(&did.dial_info) { + return false; + } } - } - true + true + }) } pub fn signed_node_info_is_valid_in_routing_domain( @@ -223,8 +215,8 @@ impl RoutingTableInner { pub fn get_contact_method( &self, routing_domain: RoutingDomain, - peer_a: &PeerInfo, - peer_b: &PeerInfo, + peer_a: Arc, + peer_b: Arc, dial_info_filter: DialInfoFilter, sequencing: Sequencing, dif_sort: Option>, @@ -244,41 +236,42 @@ impl RoutingTableInner { }); } + /// Publish the node's current peer info to the world if it is valid + pub fn publish_peer_info(&mut self, routing_domain: RoutingDomain) -> bool { + self.with_routing_domain(routing_domain, |rdd| rdd.publish_peer_info(self)) + } + /// Unpublish the node's current peer info + pub fn unpublish_peer_info(&mut self, routing_domain: RoutingDomain) { + self.with_routing_domain(routing_domain, |rdd| rdd.unpublish_peer_info()) + } + + /// Get the current published peer info + pub fn get_published_peer_info(&self, routing_domain: RoutingDomain) -> Option> { + self.with_routing_domain(routing_domain, |rdd| rdd.get_published_peer_info()) + } + /// Return if this routing domain has a valid network class pub fn has_valid_network_class(&self, routing_domain: RoutingDomain) -> bool { - self.with_routing_domain(routing_domain, |rdd| rdd.common().has_valid_network_class()) + self.with_routing_domain(routing_domain, |rdd| rdd.has_valid_network_class()) } - /// Return a copy of our node's peerinfo - pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo { - self.with_routing_domain(routing_domain, |rdd| { - rdd.common().with_peer_info(self, |pi| pi.clone()) - }) - } - - /// Return our current node info timestamp - pub fn get_own_node_info_ts(&self, routing_domain: RoutingDomain) -> Timestamp { - self.with_routing_domain(routing_domain, |rdd| { - rdd.common() - .with_peer_info(self, |pi| pi.signed_node_info().timestamp()) - }) + /// Return a copy of our node's current peerinfo (may not yet be published) + pub fn get_current_peer_info(&self, routing_domain: RoutingDomain) -> Arc { + self.with_routing_domain(routing_domain, |rdd| rdd.get_peer_info(self)) } /// Return the domain's currently registered network class pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option { - self.with_routing_domain(routing_domain, |rdd| rdd.common().network_class()) + self.with_routing_domain(routing_domain, |rdd| rdd.network_class()) } /// Return the domain's filter for what we can receivein the form of a dial info filter - #[allow(dead_code)] pub fn get_inbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { - self.with_routing_domain(routing_domain, |rdd| { - rdd.common().inbound_dial_info_filter() - }) + self.with_routing_domain(routing_domain, |rdd| rdd.inbound_dial_info_filter()) } /// Return the domain's filter for what we can receive in the form of a node ref filter - #[allow(dead_code)] + #[expect(dead_code)] pub fn get_inbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { let dif = self.get_inbound_dial_info_filter(routing_domain); NodeRefFilter::new() @@ -288,9 +281,7 @@ impl RoutingTableInner { /// Return the domain's filter for what we can send out in the form of a dial info filter pub fn get_outbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { - self.with_routing_domain(routing_domain, |rdd| { - rdd.common().outbound_dial_info_filter() - }) + self.with_routing_domain(routing_domain, |rdd| rdd.outbound_dial_info_filter()) } /// Return the domain's filter for what we can receive in the form of a node ref filter pub fn get_outbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { @@ -327,30 +318,6 @@ impl RoutingTableInner { } } - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - pub fn configure_local_network_routing_domain( - &mut self, - local_networks: Vec<(IpAddr, IpAddr)>, - ) { - log_net!(debug "configure_local_network_routing_domain: {:#?}", local_networks); - - let changed = self - .local_network_routing_domain - .set_local_networks(local_networks); - - // If the local network topology has changed, nuke the existing local node info and let new local discovery happen - if changed { - let cur_ts = Timestamp::now(); - self.with_entries_mut(cur_ts, BucketEntryState::Dead, |rti, e| { - e.with_mut(rti, |_rti, e| { - e.clear_signed_node_info(RoutingDomain::LocalNetwork); - e.reset_updated_since_last_network_change(); - }); - Option::<()>::None - }); - } - } - /// Attempt to empty the routing table /// should only be performed when there are no node_refs (detached) pub fn purge_buckets(&mut self) { @@ -401,11 +368,11 @@ impl RoutingTableInner { let bucket = self.get_bucket_mut(bucket_index); let bucket_depth = Self::bucket_depth(bucket_index); - if let Some(_dead_node_ids) = bucket.kick(bucket_depth, exempt_peers) { + if let Some(dead_node_ids) = bucket.kick(bucket_depth, exempt_peers) { // Remove expired entries self.all_entries.remove_expired(); - log_rtab!(debug "Bucket {}:{} kicked Routing table now has {} nodes", bucket_index.0, bucket_index.1, self.bucket_entry_count()); + log_rtab!(debug "Bucket {}:{} kicked Routing table now has {} nodes\nKicked nodes:{:#?}", bucket_index.0, bucket_index.1, self.bucket_entry_count(), dead_node_ids); // Now purge the routing table inner vectors //let filter = |k: &DHTKey| dead_node_ids.contains(k); @@ -515,11 +482,13 @@ impl RoutingTableInner { outer_self: RoutingTable, routing_domain: RoutingDomain, cur_ts: Timestamp, - ) -> Vec { - let own_node_info_ts = self.get_own_node_info_ts(routing_domain); + ) -> Vec { + let opt_own_node_info_ts = self + .get_published_peer_info(routing_domain) + .map(|pi| pi.signed_node_info().timestamp()); // Collect all entries that are 'needs_ping' and have some node info making them reachable somehow - let mut node_refs = Vec::::with_capacity(self.bucket_entry_count()); + let mut node_refs = Vec::::with_capacity(self.bucket_entry_count()); self.with_entries(cur_ts, BucketEntryState::Unreliable, |rti, entry| { let entry_needs_ping = |e: &BucketEntryInner| { // If this entry isn't in the routing domain we are checking, don't include it @@ -534,7 +503,9 @@ impl RoutingTableInner { } // If this entry needs a ping because this node hasn't seen our latest node info, then do it - if !e.has_seen_our_node_info_ts(routing_domain, own_node_info_ts) { + if opt_own_node_info_ts.is_some() + && !e.has_seen_our_node_info_ts(routing_domain, opt_own_node_info_ts.unwrap()) + { return true; } @@ -547,10 +518,11 @@ impl RoutingTableInner { }; if entry.with_inner(entry_needs_ping) { - node_refs.push(NodeRef::new( + node_refs.push(FilteredNodeRef::new( outer_self.clone(), entry, - Some(NodeRefFilter::new().with_routing_domain(routing_domain)), + NodeRefFilter::new().with_routing_domain(routing_domain), + Sequencing::default(), )); } Option::<()>::None @@ -558,11 +530,11 @@ impl RoutingTableInner { node_refs } - #[allow(dead_code)] + #[expect(dead_code)] pub fn get_all_alive_nodes(&self, outer_self: RoutingTable, cur_ts: Timestamp) -> Vec { let mut node_refs = Vec::::with_capacity(self.bucket_entry_count()); self.with_entries(cur_ts, BucketEntryState::Unreliable, |_rti, entry| { - node_refs.push(NodeRef::new(outer_self.clone(), entry, None)); + node_refs.push(NodeRef::new(outer_self.clone(), entry)); Option::<()>::None }); node_refs @@ -673,7 +645,7 @@ impl RoutingTableInner { } // Make a noderef to return - let nr = NodeRef::new(outer_self.clone(), best_entry.clone(), None); + let nr = NodeRef::new(outer_self.clone(), best_entry.clone()); // Update the entry with the update func best_entry.with_mut_inner(|e| update_func(self, e)); @@ -696,7 +668,7 @@ impl RoutingTableInner { } // Make node ref to return - let nr = NodeRef::new(outer_self.clone(), new_entry.clone(), None); + let nr = NodeRef::new(outer_self.clone(), new_entry.clone()); // Update the entry with the update func new_entry.with_mut_inner(|e| update_func(self, e)); @@ -742,7 +714,7 @@ impl RoutingTableInner { let bucket = self.get_bucket(bucket_index); Ok(bucket .entry(&node_id.value) - .map(|e| NodeRef::new(outer_self, e, None))) + .map(|e| NodeRef::new(outer_self, e))) } /// Resolve an existing routing table entry and return a filtered reference to it @@ -753,10 +725,10 @@ impl RoutingTableInner { node_id: TypedKey, routing_domain_set: RoutingDomainSet, dial_info_filter: DialInfoFilter, - ) -> EyreResult> { + ) -> EyreResult> { let nr = self.lookup_node_ref(outer_self, node_id)?; Ok(nr.map(|nr| { - nr.filtered_clone( + nr.custom_filtered( NodeRefFilter::new() .with_dial_info_filter(dial_info_filter) .with_routing_domain_set(routing_domain_set), @@ -789,10 +761,11 @@ impl RoutingTableInner { pub fn register_node_with_peer_info( &mut self, outer_self: RoutingTable, - routing_domain: RoutingDomain, - peer_info: PeerInfo, + peer_info: Arc, allow_invalid: bool, - ) -> EyreResult { + ) -> EyreResult { + let routing_domain = peer_info.routing_domain(); + // if our own node is in the list, then ignore it as we don't add ourselves to our own routing table if self .unlocked_inner @@ -830,30 +803,41 @@ impl RoutingTableInner { } // Register relay info first if we have that and the relay isn't us - if let Some(relay_peer_info) = peer_info.signed_node_info().relay_peer_info() { + if let Some(relay_peer_info) = peer_info.signed_node_info().relay_peer_info(routing_domain) + { if !self .unlocked_inner .matches_own_node_id(relay_peer_info.node_ids()) { - self.register_node_with_peer_info( - outer_self.clone(), - routing_domain, - relay_peer_info, - false, - )?; + self.register_node_with_peer_info(outer_self.clone(), relay_peer_info, false)?; } } - let (node_ids, signed_node_info) = peer_info.destructure(); - let mut nr = self.create_node_ref(outer_self, &node_ids, |_rti, e| { - e.update_signed_node_info(routing_domain, signed_node_info); + let (_routing_domain, node_ids, signed_node_info) = + Arc::unwrap_or_clone(peer_info).destructure(); + let mut updated = false; + let nr = self.create_node_ref(outer_self, &node_ids, |_rti, e| { + updated = e.update_signed_node_info(routing_domain, signed_node_info); })?; - nr.set_filter(Some( - NodeRefFilter::new().with_routing_domain(routing_domain), - )); + if updated { + // If this is our relay, then redo our own peerinfo because + // if we have relayed peerinfo, then changing the relay's peerinfo + // changes our own peer info + self.with_routing_domain(routing_domain, |rd| { + let opt_our_relay_node_ids = rd + .relay_node() + .map(|relay_nr| relay_nr.locked(self).node_ids()); + if let Some(our_relay_node_ids) = opt_our_relay_node_ids { + if our_relay_node_ids.contains_any(&node_ids) { + rd.refresh(); + rd.publish_peer_info(self); + } + } + }); + } - Ok(nr) + Ok(nr.custom_filtered(NodeRefFilter::new().with_routing_domain(routing_domain))) } /// Shortcut function to add a node to our routing table if it doesn't exist @@ -862,16 +846,20 @@ impl RoutingTableInner { pub fn register_node_with_existing_connection( &mut self, outer_self: RoutingTable, + routing_domain: RoutingDomain, node_id: TypedKey, flow: Flow, timestamp: Timestamp, - ) -> EyreResult { + ) -> EyreResult { let nr = self.create_node_ref(outer_self, &TypedKeyGroup::from(node_id), |_rti, e| { //e.make_not_dead(timestamp); e.touch_last_seen(timestamp); })?; // set the most recent node address for connection finding and udp replies nr.locked_mut(self).set_last_flow(flow, timestamp); + + // Enforce routing domain + let nr = nr.custom_filtered(NodeRefFilter::new().with_routing_domain(routing_domain)); Ok(nr) } @@ -940,34 +928,41 @@ impl RoutingTableInner { // Retrieve the fastest nodes in the routing table matching an entry filter #[instrument(level = "trace", skip_all)] - pub fn find_fast_public_nodes_filtered( + pub fn find_fast_non_local_nodes_filtered( &self, outer_self: RoutingTable, + routing_domain: RoutingDomain, node_count: usize, mut filters: VecDeque, ) -> Vec { - let public_node_filter = - Box::new(|_rti: &RoutingTableInner, v: Option>| { + assert_ne!( + routing_domain, + RoutingDomain::LocalNetwork, + "LocalNetwork is not a valid non-local RoutingDomain" + ); + let public_node_filter = Box::new( + move |_rti: &RoutingTableInner, v: Option>| { let entry = v.unwrap(); entry.with_inner(|e| { // skip nodes on local network if e.node_info(RoutingDomain::LocalNetwork).is_some() { return false; } - // skip nodes not on public internet - if e.node_info(RoutingDomain::PublicInternet).is_none() { + // skip nodes not on desired routing domain + if e.node_info(routing_domain).is_none() { return false; } true }) - }) as RoutingTableEntryFilter; + }, + ) as RoutingTableEntryFilter; filters.push_front(public_node_filter); self.find_preferred_fastest_nodes( node_count, filters, |_rti: &RoutingTableInner, v: Option>| { - NodeRef::new(outer_self.clone(), v.unwrap().clone(), None) + NodeRef::new(outer_self.clone(), v.unwrap().clone()) }, ) } @@ -996,12 +991,14 @@ impl RoutingTableInner { pub fn transform_to_peer_info( &self, routing_domain: RoutingDomain, - own_peer_info: &PeerInfo, + own_peer_info: Arc, entry: Option>, - ) -> PeerInfo { + ) -> Arc { match entry { None => own_peer_info.clone(), - Some(entry) => entry.with_inner(|e| e.make_peer_info(routing_domain).unwrap()), + Some(entry) => { + entry.with_inner(|e| Arc::new(e.make_peer_info(routing_domain).unwrap())) + } } } @@ -1242,7 +1239,7 @@ impl RoutingTableInner { ) -> Vec { // Lock all noderefs let kind = node_id.kind; - let mut closest_nodes_locked: Vec = closest_nodes + let mut closest_nodes_locked: Vec = closest_nodes .iter() .filter_map(|nr| { let nr_locked = nr.locked(self); @@ -1267,12 +1264,12 @@ impl RoutingTableInner { pub(crate) fn make_closest_noderef_sort( crypto: Crypto, node_id: TypedKey, -) -> impl Fn(&NodeRefLocked, &NodeRefLocked) -> core::cmp::Ordering { +) -> impl Fn(&LockedNodeRef, &LockedNodeRef) -> core::cmp::Ordering { let kind = node_id.kind; // Get cryptoversion to check distance with let vcrypto = crypto.get(kind).unwrap(); - move |a: &NodeRefLocked, b: &NodeRefLocked| -> core::cmp::Ordering { + move |a: &LockedNodeRef, b: &LockedNodeRef| -> core::cmp::Ordering { // same nodes are always the same if a.same_entry(b) { return core::cmp::Ordering::Equal; diff --git a/veilid-core/src/routing_table/routing_table_inner/routing_domains/editor.rs b/veilid-core/src/routing_table/routing_table_inner/routing_domains/editor.rs new file mode 100644 index 00000000..060ff1af --- /dev/null +++ b/veilid-core/src/routing_table/routing_table_inner/routing_domains/editor.rs @@ -0,0 +1,116 @@ +use super::*; + +pub trait RoutingDomainEditorCommonTrait { + fn clear_dial_info_details( + &mut self, + address_type: Option, + protocol_type: Option, + ) -> &mut Self; + fn clear_relay_node(&mut self) -> &mut Self; + fn set_relay_node(&mut self, relay_node: NodeRef) -> &mut Self; + fn set_relay_node_keepalive(&mut self, ts: Option) -> &mut Self; + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] + fn add_dial_info(&mut self, dial_info: DialInfo, class: DialInfoClass) -> &mut Self; + fn setup_network( + &mut self, + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + capabilities: Vec, + ) -> &mut Self; + fn set_network_class(&mut self, network_class: Option) -> &mut Self; + fn commit(&mut self, pause_tasks: bool) -> SendPinBoxFutureLifetime<'_, bool>; + fn shutdown(&mut self) -> SendPinBoxFutureLifetime<'_, ()>; + fn publish(&mut self); +} + +pub(super) trait RoutingDomainDetailApplyCommonChange { + /// Make a change from the routing domain editor + fn apply_common_change(&mut self, change: RoutingDomainChangeCommon); +} + +impl RoutingDomainDetailApplyCommonChange for T { + /// Make a change from the routing domain editor + fn apply_common_change(&mut self, change: RoutingDomainChangeCommon) { + match change { + RoutingDomainChangeCommon::ClearDialInfoDetails { + address_type, + protocol_type, + } => { + self.common_mut() + .clear_dial_info_details(address_type, protocol_type); + } + + RoutingDomainChangeCommon::ClearRelayNode => { + self.common_mut().set_relay_node(None); + } + + RoutingDomainChangeCommon::SetRelayNode { relay_node } => { + self.common_mut().set_relay_node(Some(relay_node.clone())) + } + + RoutingDomainChangeCommon::SetRelayNodeKeepalive { ts } => { + self.common_mut().set_relay_node_last_keepalive(ts); + } + RoutingDomainChangeCommon::AddDialInfo { dial_info_detail } => { + if !self.ensure_dial_info_is_valid(&dial_info_detail.dial_info) { + return; + } + + self.common_mut() + .add_dial_info_detail(dial_info_detail.clone()); + } + // RoutingDomainChange::RemoveDialInfoDetail { dial_info_detail } => { + // self.common + // .remove_dial_info_detail(dial_info_detail.clone()); + // } + RoutingDomainChangeCommon::SetupNetwork { + outbound_protocols, + inbound_protocols, + address_types, + capabilities, + } => { + self.common_mut().setup_network( + outbound_protocols, + inbound_protocols, + address_types, + capabilities.clone(), + ); + } + RoutingDomainChangeCommon::SetNetworkClass { network_class } => { + self.common_mut().set_network_class(network_class); + } + } + } +} + +#[derive(Debug)] +pub(super) enum RoutingDomainChangeCommon { + ClearDialInfoDetails { + address_type: Option, + protocol_type: Option, + }, + ClearRelayNode, + SetRelayNode { + relay_node: NodeRef, + }, + SetRelayNodeKeepalive { + ts: Option, + }, + AddDialInfo { + dial_info_detail: DialInfoDetail, + }, + // #[cfg_attr(target_arch = "wasm32", expect(dead_code))] + // RemoveDialInfoDetail { + // dial_info_detail: DialInfoDetail, + // }, + SetupNetwork { + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + capabilities: Vec, + }, + SetNetworkClass { + network_class: Option, + }, +} diff --git a/veilid-core/src/routing_table/routing_table_inner/routing_domains/local_network/editor.rs b/veilid-core/src/routing_table/routing_table_inner/routing_domains/local_network/editor.rs new file mode 100644 index 00000000..4b8de70f --- /dev/null +++ b/veilid-core/src/routing_table/routing_table_inner/routing_domains/local_network/editor.rs @@ -0,0 +1,270 @@ +#![cfg_attr(target_arch = "wasm32", expect(dead_code))] + +use super::*; + +#[derive(Debug)] +enum RoutingDomainChangeLocalNetwork { + SetLocalNetworks { + local_networks: Vec<(IpAddr, IpAddr)>, + }, + Common(RoutingDomainChangeCommon), +} + +pub struct RoutingDomainEditorLocalNetwork { + routing_table: RoutingTable, + changes: Vec, +} + +impl RoutingDomainEditorLocalNetwork { + pub(in crate::routing_table) fn new(routing_table: RoutingTable) -> Self { + Self { + routing_table: routing_table.clone(), + changes: Vec::new(), + } + } + + pub fn set_local_networks(&mut self, local_networks: Vec<(IpAddr, IpAddr)>) -> &mut Self { + self.changes + .push(RoutingDomainChangeLocalNetwork::SetLocalNetworks { local_networks }); + self + } +} + +impl RoutingDomainEditorCommonTrait for RoutingDomainEditorLocalNetwork { + #[instrument(level = "debug", skip(self))] + fn clear_dial_info_details( + &mut self, + address_type: Option, + protocol_type: Option, + ) -> &mut Self { + self.changes.push(RoutingDomainChangeLocalNetwork::Common( + RoutingDomainChangeCommon::ClearDialInfoDetails { + address_type, + protocol_type, + }, + )); + + self + } + #[instrument(level = "debug", skip(self))] + fn clear_relay_node(&mut self) -> &mut Self { + self.changes.push(RoutingDomainChangeLocalNetwork::Common( + RoutingDomainChangeCommon::ClearRelayNode, + )); + self + } + #[instrument(level = "debug", skip(self))] + fn set_relay_node(&mut self, relay_node: NodeRef) -> &mut Self { + self.changes.push(RoutingDomainChangeLocalNetwork::Common( + RoutingDomainChangeCommon::SetRelayNode { relay_node }, + )); + self + } + #[instrument(level = "debug", skip(self))] + fn set_relay_node_keepalive(&mut self, ts: Option) -> &mut Self { + self.changes.push(RoutingDomainChangeLocalNetwork::Common( + RoutingDomainChangeCommon::SetRelayNodeKeepalive { ts }, + )); + self + } + #[instrument(level = "debug", skip(self))] + fn add_dial_info(&mut self, dial_info: DialInfo, class: DialInfoClass) -> &mut Self { + self.changes.push(RoutingDomainChangeLocalNetwork::Common( + RoutingDomainChangeCommon::AddDialInfo { + dial_info_detail: DialInfoDetail { + dial_info: dial_info.clone(), + class, + }, + }, + )); + self + } + // #[instrument(level = "debug", skip_all)] + // fn retain_dial_info bool>( + // &mut self, + // closure: F, + // ) -> EyreResult<&mut Self> { + // let dids = self.routing_table.dial_info_details(self.routing_domain); + // for did in dids { + // if !closure(&did.dial_info, did.class) { + // self.changes + // .push(RoutingDomainChangePublicInternet::Common(RoutingDomainChange::RemoveDialInfoDetail { + // dial_info_detail: did, + // })); + // } + // } + + // Ok(self) + // } + + #[instrument(level = "debug", skip(self))] + fn setup_network( + &mut self, + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + capabilities: Vec, + ) -> &mut Self { + self.changes.push(RoutingDomainChangeLocalNetwork::Common( + RoutingDomainChangeCommon::SetupNetwork { + outbound_protocols, + inbound_protocols, + address_types, + capabilities, + }, + )); + self + } + + #[instrument(level = "debug", skip(self))] + fn set_network_class(&mut self, network_class: Option) -> &mut Self { + self.changes.push(RoutingDomainChangeLocalNetwork::Common( + RoutingDomainChangeCommon::SetNetworkClass { network_class }, + )); + self + } + + #[instrument(level = "debug", skip(self))] + fn commit(&mut self, pause_tasks: bool) -> SendPinBoxFutureLifetime<'_, bool> { + Box::pin(async move { + // No locking if we have nothing to do + if self.changes.is_empty() { + return false; + } + // Briefly pause routing table ticker while changes are made + let _tick_guard = if pause_tasks { + Some(self.routing_table.pause_tasks().await) + } else { + None + }; + + // Apply changes + let mut peer_info_changed = false; + + let mut rti_lock = self.routing_table.inner.write(); + let rti = &mut rti_lock; + rti.with_local_network_routing_domain_mut(|detail| { + let old_dial_info_details = detail.dial_info_details().clone(); + let old_relay_node = detail.relay_node(); + let old_outbound_protocols = detail.outbound_protocols(); + let old_inbound_protocols = detail.inbound_protocols(); + let old_address_types = detail.address_types(); + let old_capabilities = detail.capabilities(); + let old_network_class = detail.network_class(); + + for change in self.changes.drain(..) { + match change { + RoutingDomainChangeLocalNetwork::Common(common_change) => { + detail.apply_common_change(common_change); + } + RoutingDomainChangeLocalNetwork::SetLocalNetworks { local_networks } => { + detail.set_local_networks(local_networks); + } + } + } + + let new_dial_info_details = detail.dial_info_details().clone(); + let new_relay_node = detail.relay_node(); + let new_outbound_protocols = detail.outbound_protocols(); + let new_inbound_protocols = detail.inbound_protocols(); + let new_address_types = detail.address_types(); + let new_capabilities = detail.capabilities(); + let new_network_class = detail.network_class(); + + // Compare and see if peerinfo needs republication + let removed_dial_info = old_dial_info_details + .iter() + .filter(|di| !new_dial_info_details.contains(di)) + .collect::>(); + if !removed_dial_info.is_empty() { + info!("[LocalNetwork] removed dial info: {:#?}", removed_dial_info); + peer_info_changed = true; + } + let added_dial_info = new_dial_info_details + .iter() + .filter(|di| !old_dial_info_details.contains(di)) + .collect::>(); + if !added_dial_info.is_empty() { + info!("[LocalNetwork] added dial info: {:#?}", added_dial_info); + peer_info_changed = true; + } + if let Some(nrn) = new_relay_node { + if let Some(orn) = old_relay_node { + if !nrn.same_entry(&orn) { + info!("[LocalNetwork] change relay: {} -> {}", orn, nrn); + peer_info_changed = true; + } + } else { + info!("[LocalNetwork] set relay: {}", nrn); + peer_info_changed = true; + } + } + if old_outbound_protocols != new_outbound_protocols { + info!( + "[LocalNetwork] changed network: outbound {:?}->{:?}\n", + old_outbound_protocols, new_outbound_protocols + ); + peer_info_changed = true; + } + if old_inbound_protocols != new_inbound_protocols { + info!( + "[LocalNetwork] changed network: inbound {:?}->{:?}\n", + old_inbound_protocols, new_inbound_protocols, + ); + peer_info_changed = true; + } + if old_address_types != new_address_types { + info!( + "[LocalNetwork] changed network: address types {:?}->{:?}\n", + old_address_types, new_address_types, + ); + peer_info_changed = true; + } + if old_capabilities != new_capabilities { + info!( + "[PublicInternet] changed network: capabilities {:?}->{:?}\n", + old_capabilities, new_capabilities + ); + peer_info_changed = true; + } + if old_network_class != new_network_class { + info!( + "[LocalNetwork] changed network class: {:?}->{:?}\n", + old_network_class, new_network_class + ); + peer_info_changed = true; + } + }); + + if peer_info_changed { + // Allow signed node info updates at same timestamp for otherwise dead nodes if our network has changed + rti.reset_all_updated_since_last_network_change(); + } + + peer_info_changed + }) + } + + #[instrument(level = "debug", skip(self))] + fn publish(&mut self) { + self.routing_table + .inner + .write() + .publish_peer_info(RoutingDomain::LocalNetwork); + } + + #[instrument(level = "debug", skip(self))] + fn shutdown(&mut self) -> SendPinBoxFutureLifetime<'_, ()> { + Box::pin(async move { + self.clear_dial_info_details(None, None) + .set_network_class(None) + .clear_relay_node() + .commit(true) + .await; + self.routing_table + .inner + .write() + .unpublish_peer_info(RoutingDomain::LocalNetwork); + }) + } +} diff --git a/veilid-core/src/routing_table/routing_table_inner/routing_domains/local_network/mod.rs b/veilid-core/src/routing_table/routing_table_inner/routing_domains/local_network/mod.rs new file mode 100644 index 00000000..c440c666 --- /dev/null +++ b/veilid-core/src/routing_table/routing_table_inner/routing_domains/local_network/mod.rs @@ -0,0 +1,203 @@ +mod editor; + +pub use editor::*; + +use super::*; + +/// Local Network routing domain internals +#[derive(Debug)] +pub struct LocalNetworkRoutingDomainDetail { + /// The local networks this domain will communicate with + local_networks: Vec<(IpAddr, IpAddr)>, + /// Common implementation for all routing domains + common: RoutingDomainDetailCommon, + /// Published peer info for this routing domain + published_peer_info: Mutex>>, +} + +impl Default for LocalNetworkRoutingDomainDetail { + fn default() -> Self { + Self { + local_networks: Default::default(), + common: RoutingDomainDetailCommon::new(RoutingDomain::LocalNetwork), + published_peer_info: Default::default(), + } + } +} + +impl LocalNetworkRoutingDomainDetail { + #[expect(dead_code)] + pub fn local_networks(&self) -> Vec<(IpAddr, IpAddr)> { + self.local_networks.clone() + } + + pub fn set_local_networks(&mut self, mut local_networks: Vec<(IpAddr, IpAddr)>) -> bool { + local_networks.sort(); + if local_networks == self.local_networks { + return false; + } + self.local_networks = local_networks; + true + } +} + +impl RoutingDomainDetailCommonAccessors for LocalNetworkRoutingDomainDetail { + fn common(&self) -> &RoutingDomainDetailCommon { + &self.common + } + fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon { + &mut self.common + } +} + +impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { + fn routing_domain(&self) -> RoutingDomain { + RoutingDomain::LocalNetwork + } + + fn network_class(&self) -> Option { + self.common.network_class() + } + fn outbound_protocols(&self) -> ProtocolTypeSet { + self.common.outbound_protocols() + } + fn inbound_protocols(&self) -> ProtocolTypeSet { + self.common.inbound_protocols() + } + fn address_types(&self) -> AddressTypeSet { + self.common.address_types() + } + fn capabilities(&self) -> Vec { + self.common.capabilities() + } + fn relay_node(&self) -> Option { + self.common.relay_node() + } + fn relay_node_last_keepalive(&self) -> Option { + self.common.relay_node_last_keepalive() + } + fn dial_info_details(&self) -> &Vec { + self.common.dial_info_details() + } + fn has_valid_network_class(&self) -> bool { + self.common.has_valid_network_class() + } + + fn inbound_dial_info_filter(&self) -> DialInfoFilter { + self.common.inbound_dial_info_filter() + } + fn outbound_dial_info_filter(&self) -> DialInfoFilter { + self.common.outbound_dial_info_filter() + } + + fn get_peer_info(&self, rti: &RoutingTableInner) -> Arc { + self.common.get_peer_info(rti) + } + + fn get_published_peer_info(&self) -> Option> { + (*self.published_peer_info.lock()).clone() + } + + fn can_contain_address(&self, address: Address) -> bool { + let ip = address.ip_addr(); + for localnet in &self.local_networks { + if ipaddr_in_network(ip, localnet.0, localnet.1) { + return true; + } + } + false + } + + fn refresh(&self) { + self.common.clear_cache(); + } + + fn publish_peer_info(&self, rti: &RoutingTableInner) -> bool { + let pi = self.get_peer_info(rti); + + // If the network class is not yet determined, don't publish + if pi.signed_node_info().node_info().network_class() == NetworkClass::Invalid { + log_rtab!(debug "[LocalNetwork] Not publishing peer info with invalid network class"); + return false; + } + + // If we need a relay and we don't have one, don't publish yet + if let Some(_relay_kind) = pi.signed_node_info().node_info().requires_relay() { + if pi.signed_node_info().relay_ids().is_empty() { + log_rtab!(debug "[LocalNetwork] Not publishing peer info that wants relay until we have a relay"); + return false; + } + } + + // Don't publish if the peer info hasnt changed from our previous publication + let mut ppi_lock = self.published_peer_info.lock(); + if let Some(old_peer_info) = &*ppi_lock { + if pi.equivalent(old_peer_info) { + log_rtab!(debug "[LocalNetwork] Not publishing peer info because it is equivalent"); + return false; + } + } + + log_rtab!(debug "[LocalNetwork] Published new peer info: {:#?}", pi); + *ppi_lock = Some(pi); + + true + } + + fn unpublish_peer_info(&self) { + let mut ppi_lock = self.published_peer_info.lock(); + log_rtab!(debug "[LocalNetwork] Unpublished peer info"); + *ppi_lock = None; + } + + fn ensure_dial_info_is_valid(&self, dial_info: &DialInfo) -> bool { + let address = dial_info.socket_address().address(); + let can_contain_address = self.can_contain_address(address); + + if !can_contain_address { + log_rtab!(debug "[LocalNetwork] can not add dial info to this routing domain: {:?}", dial_info); + return false; + } + if !dial_info.is_valid() { + log_rtab!(debug + "shouldn't be registering invalid addresses: {:?}", + dial_info + ); + return false; + } + true + } + + fn get_contact_method( + &self, + _rti: &RoutingTableInner, + peer_a: Arc, + peer_b: Arc, + dial_info_filter: DialInfoFilter, + sequencing: Sequencing, + dif_sort: Option>, + ) -> ContactMethod { + // Get the nodeinfos for convenience + let node_a = peer_a.signed_node_info().node_info(); + let node_b = peer_b.signed_node_info().node_info(); + + // Get the node ids that would be used between these peers + let cck = common_crypto_kinds(&peer_a.node_ids().kinds(), &peer_b.node_ids().kinds()); + let Some(_best_ck) = cck.first().copied() else { + // No common crypto kinds between these nodes, can't contact + return ContactMethod::Unreachable; + }; + + if let Some(target_did) = first_filtered_dial_info_detail_between_nodes( + node_a, + node_b, + &dial_info_filter, + sequencing, + dif_sort, + ) { + return ContactMethod::Direct(target_did.dial_info); + } + + ContactMethod::Unreachable + } +} diff --git a/veilid-core/src/routing_table/routing_table_inner/routing_domains/mod.rs b/veilid-core/src/routing_table/routing_table_inner/routing_domains/mod.rs new file mode 100644 index 00000000..7447fbcc --- /dev/null +++ b/veilid-core/src/routing_table/routing_table_inner/routing_domains/mod.rs @@ -0,0 +1,333 @@ +mod editor; +mod local_network; +mod public_internet; + +use super::*; + +pub use editor::*; +pub use local_network::*; +pub use public_internet::*; + +/// General trait for all routing domains +pub(crate) trait RoutingDomainDetail { + // Common accessors + #[expect(dead_code)] + fn routing_domain(&self) -> RoutingDomain; + fn network_class(&self) -> Option; + fn outbound_protocols(&self) -> ProtocolTypeSet; + fn inbound_protocols(&self) -> ProtocolTypeSet; + fn address_types(&self) -> AddressTypeSet; + fn capabilities(&self) -> Vec; + fn relay_node(&self) -> Option; + fn relay_node_last_keepalive(&self) -> Option; + fn dial_info_details(&self) -> &Vec; + fn has_valid_network_class(&self) -> bool; + fn get_published_peer_info(&self) -> Option>; + fn inbound_dial_info_filter(&self) -> DialInfoFilter; + fn outbound_dial_info_filter(&self) -> DialInfoFilter; + fn get_peer_info(&self, rti: &RoutingTableInner) -> Arc; + + /// Can this routing domain contain a particular address + fn can_contain_address(&self, address: Address) -> bool; + fn ensure_dial_info_is_valid(&self, dial_info: &DialInfo) -> bool; + + /// Refresh caches if external data changes + fn refresh(&self); + + /// Publish current peer info to the world + fn publish_peer_info(&self, rti: &RoutingTableInner) -> bool; + fn unpublish_peer_info(&self); + + /// Get the contact method required for node A to reach node B in this routing domain + /// Routing table must be locked for reading to use this function + fn get_contact_method( + &self, + rti: &RoutingTableInner, + peer_a: Arc, + peer_b: Arc, + dial_info_filter: DialInfoFilter, + sequencing: Sequencing, + dif_sort: Option>, + ) -> ContactMethod; +} + +trait RoutingDomainDetailCommonAccessors: RoutingDomainDetail { + #[expect(dead_code)] + fn common(&self) -> &RoutingDomainDetailCommon; + fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon; +} + +fn first_filtered_dial_info_detail_between_nodes( + from_node: &NodeInfo, + to_node: &NodeInfo, + dial_info_filter: &DialInfoFilter, + sequencing: Sequencing, + dif_sort: Option>, +) -> Option { + // Consider outbound capabilities + let dial_info_filter = (*dial_info_filter).filtered( + &DialInfoFilter::all() + .with_address_type_set(from_node.address_types()) + .with_protocol_type_set(from_node.outbound_protocols()), + ); + + // Apply sequencing and get sort + // Include sorting by external dial info sort for rotating through dialinfo + // based on an external preference table, for example the one kept by + // AddressFilter to deprioritize dialinfo that have recently failed to connect + let (ordered, dial_info_filter) = dial_info_filter.apply_sequencing(sequencing); + let sort: Option> = if ordered { + if let Some(dif_sort) = dif_sort { + Some(Box::new(move |a, b| { + let mut ord = dif_sort(a, b); + if ord == core::cmp::Ordering::Equal { + ord = DialInfoDetail::ordered_sequencing_sort(a, b); + } + ord + })) + } else { + Some(Box::new(move |a, b| { + DialInfoDetail::ordered_sequencing_sort(a, b) + })) + } + } else if let Some(dif_sort) = dif_sort { + Some(Box::new(move |a, b| dif_sort(a, b))) + } else { + None + }; + + // If the filter is dead then we won't be able to connect + if dial_info_filter.is_dead() { + return None; + } + + // Get the best match dial info for node B if we have it + let direct_filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); + to_node.first_filtered_dial_info_detail(sort, direct_filter) +} + +#[derive(Debug)] +struct RoutingDomainDetailCommon { + routing_domain: RoutingDomain, + network_class: Option, + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + relay_node: Option, + relay_node_last_keepalive: Option, + capabilities: Vec, + dial_info_details: Vec, + // caches + cached_peer_info: Mutex>>, +} + +impl RoutingDomainDetailCommon { + pub fn new(routing_domain: RoutingDomain) -> Self { + Self { + routing_domain, + network_class: Default::default(), + outbound_protocols: Default::default(), + inbound_protocols: Default::default(), + address_types: Default::default(), + relay_node: Default::default(), + relay_node_last_keepalive: Default::default(), + capabilities: Default::default(), + dial_info_details: Default::default(), + cached_peer_info: Mutex::new(Default::default()), + } + } + + /////////////////////////////////////////////////////////////////////// + // Accessors + + pub fn network_class(&self) -> Option { + self.network_class + } + + pub fn outbound_protocols(&self) -> ProtocolTypeSet { + self.outbound_protocols + } + + pub fn inbound_protocols(&self) -> ProtocolTypeSet { + self.inbound_protocols + } + + pub fn address_types(&self) -> AddressTypeSet { + self.address_types + } + + pub fn capabilities(&self) -> Vec { + self.capabilities.clone() + } + + pub fn relay_node(&self) -> Option { + self.relay_node.as_ref().map(|nr| { + nr.custom_filtered(NodeRefFilter::new().with_routing_domain(self.routing_domain)) + }) + } + + pub fn relay_node_last_keepalive(&self) -> Option { + self.relay_node_last_keepalive + } + + pub fn dial_info_details(&self) -> &Vec { + &self.dial_info_details + } + + pub fn has_valid_network_class(&self) -> bool { + self.network_class.unwrap_or(NetworkClass::Invalid) != NetworkClass::Invalid + } + + pub fn inbound_dial_info_filter(&self) -> DialInfoFilter { + DialInfoFilter::all() + .with_protocol_type_set(self.inbound_protocols) + .with_address_type_set(self.address_types) + } + + pub fn outbound_dial_info_filter(&self) -> DialInfoFilter { + DialInfoFilter::all() + .with_protocol_type_set(self.outbound_protocols) + .with_address_type_set(self.address_types) + } + + pub fn get_peer_info(&self, rti: &RoutingTableInner) -> Arc { + let mut cpi = self.cached_peer_info.lock(); + if cpi.is_none() { + // Regenerate peer info + let pi = self.make_peer_info(rti); + + // Cache the peer info + *cpi = Some(Arc::new(pi)); + } + cpi.as_ref().unwrap().clone() + } + + /////////////////////////////////////////////////////////////////////// + // Mutators + + fn setup_network( + &mut self, + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + capabilities: Vec, + ) { + self.outbound_protocols = outbound_protocols; + self.inbound_protocols = inbound_protocols; + self.address_types = address_types; + self.capabilities = capabilities; + self.clear_cache(); + } + + fn set_network_class(&mut self, network_class: Option) { + self.network_class = network_class; + self.clear_cache(); + } + + fn set_relay_node(&mut self, opt_relay_node: Option) { + self.relay_node = opt_relay_node; + self.relay_node_last_keepalive = None; + self.clear_cache(); + } + fn set_relay_node_last_keepalive(&mut self, ts: Option) { + self.relay_node_last_keepalive = ts; + } + + fn clear_dial_info_details( + &mut self, + address_type: Option, + protocol_type: Option, + ) { + self.dial_info_details.retain_mut(|e| { + let mut remove = true; + if let Some(pt) = protocol_type { + if pt != e.dial_info.protocol_type() { + remove = false; + } + } + if let Some(at) = address_type { + if at != e.dial_info.address_type() { + remove = false; + } + } + !remove + }); + self.clear_cache(); + } + fn add_dial_info_detail(&mut self, did: DialInfoDetail) { + self.dial_info_details.push(did); + self.dial_info_details.sort(); + self.dial_info_details.dedup(); + self.clear_cache(); + } + // fn remove_dial_info_detail(&mut self, did: DialInfoDetail) { + // if let Some(index) = self.dial_info_details.iter().position(|x| *x == did) { + // self.dial_info_details.remove(index); + // } + // self.clear_cache(); + // } + + ////////////////////////////////////////////////////////////////////////////// + // Internal functions + + fn make_peer_info(&self, rti: &RoutingTableInner) -> PeerInfo { + let node_info = NodeInfo::new( + self.network_class.unwrap_or(NetworkClass::Invalid), + self.outbound_protocols, + self.address_types, + VALID_ENVELOPE_VERSIONS.to_vec(), + VALID_CRYPTO_KINDS.to_vec(), + self.capabilities.clone(), + self.dial_info_details.clone(), + ); + + let relay_info = if let Some(rn) = &self.relay_node { + let opt_relay_pi = rn.locked(rti).make_peer_info(self.routing_domain); + if let Some(relay_pi) = opt_relay_pi { + let (_routing_domain, relay_ids, relay_sni) = relay_pi.destructure(); + match relay_sni { + SignedNodeInfo::Direct(d) => Some((relay_ids, d)), + SignedNodeInfo::Relayed(_) => { + warn!("relay node should not have a relay itself! if this happens, a relay updated its signed node info and became a relay, which should cause the relay to be dropped"); + None + } + } + } else { + None + } + } else { + None + }; + + let signed_node_info = match relay_info { + Some((relay_ids, relay_sdni)) => SignedNodeInfo::Relayed( + SignedRelayedNodeInfo::make_signatures( + rti.unlocked_inner.crypto(), + rti.unlocked_inner.node_id_typed_key_pairs(), + node_info, + relay_ids, + relay_sdni, + ) + .unwrap(), + ), + None => SignedNodeInfo::Direct( + SignedDirectNodeInfo::make_signatures( + rti.unlocked_inner.crypto(), + rti.unlocked_inner.node_id_typed_key_pairs(), + node_info, + ) + .unwrap(), + ), + }; + + PeerInfo::new( + self.routing_domain, + rti.unlocked_inner.node_ids(), + signed_node_info, + ) + } + + fn clear_cache(&self) { + *self.cached_peer_info.lock() = None; + } +} diff --git a/veilid-core/src/routing_table/routing_table_inner/routing_domains/public_internet/editor.rs b/veilid-core/src/routing_table/routing_table_inner/routing_domains/public_internet/editor.rs new file mode 100644 index 00000000..a40bb999 --- /dev/null +++ b/veilid-core/src/routing_table/routing_table_inner/routing_domains/public_internet/editor.rs @@ -0,0 +1,266 @@ +use super::*; + +#[derive(Debug)] +enum RoutingDomainChangePublicInternet { + Common(RoutingDomainChangeCommon), +} + +pub struct RoutingDomainEditorPublicInternet { + routing_table: RoutingTable, + changes: Vec, +} + +impl RoutingDomainEditorPublicInternet { + pub(in crate::routing_table) fn new(routing_table: RoutingTable) -> Self { + Self { + routing_table, + changes: Vec::new(), + } + } +} + +impl RoutingDomainEditorCommonTrait for RoutingDomainEditorPublicInternet { + #[instrument(level = "debug", skip(self))] + fn clear_dial_info_details( + &mut self, + address_type: Option, + protocol_type: Option, + ) -> &mut Self { + self.changes.push(RoutingDomainChangePublicInternet::Common( + RoutingDomainChangeCommon::ClearDialInfoDetails { + address_type, + protocol_type, + }, + )); + + self + } + #[instrument(level = "debug", skip(self))] + fn clear_relay_node(&mut self) -> &mut Self { + self.changes.push(RoutingDomainChangePublicInternet::Common( + RoutingDomainChangeCommon::ClearRelayNode, + )); + self + } + #[instrument(level = "debug", skip(self))] + fn set_relay_node(&mut self, relay_node: NodeRef) -> &mut Self { + self.changes.push(RoutingDomainChangePublicInternet::Common( + RoutingDomainChangeCommon::SetRelayNode { relay_node }, + )); + self + } + #[instrument(level = "debug", skip(self))] + fn set_relay_node_keepalive(&mut self, ts: Option) -> &mut Self { + self.changes.push(RoutingDomainChangePublicInternet::Common( + RoutingDomainChangeCommon::SetRelayNodeKeepalive { ts }, + )); + self + } + #[instrument(level = "debug", skip(self))] + fn add_dial_info(&mut self, dial_info: DialInfo, class: DialInfoClass) -> &mut Self { + self.changes.push(RoutingDomainChangePublicInternet::Common( + RoutingDomainChangeCommon::AddDialInfo { + dial_info_detail: DialInfoDetail { + dial_info: dial_info.clone(), + class, + }, + }, + )); + self + } + // #[instrument(level = "debug", skip_all)] + // fn retain_dial_info bool>( + // &mut self, + // closure: F, + // ) -> EyreResult<&mut Self> { + // let dids = self.routing_table.dial_info_details(self.routing_domain); + // for did in dids { + // if !closure(&did.dial_info, did.class) { + // self.changes + // .push(RoutingDomainChangePublicInternet::Common(RoutingDomainChange::RemoveDialInfoDetail { + // dial_info_detail: did, + // })); + // } + // } + + // Ok(self) + // } + + #[instrument(level = "debug", skip(self))] + fn setup_network( + &mut self, + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + capabilities: Vec, + ) -> &mut Self { + self.changes.push(RoutingDomainChangePublicInternet::Common( + RoutingDomainChangeCommon::SetupNetwork { + outbound_protocols, + inbound_protocols, + address_types, + capabilities, + }, + )); + self + } + + #[instrument(level = "debug", skip(self))] + fn set_network_class(&mut self, network_class: Option) -> &mut Self { + self.changes.push(RoutingDomainChangePublicInternet::Common( + RoutingDomainChangeCommon::SetNetworkClass { network_class }, + )); + self + } + + #[instrument(level = "debug", skip(self))] + fn commit(&mut self, pause_tasks: bool) -> SendPinBoxFutureLifetime<'_, bool> { + Box::pin(async move { + // No locking if we have nothing to do + if self.changes.is_empty() { + return false; + } + // Briefly pause routing table ticker while changes are made + let _tick_guard = if pause_tasks { + Some(self.routing_table.pause_tasks().await) + } else { + None + }; + + // Apply changes + let mut peer_info_changed = false; + + let mut rti_lock = self.routing_table.inner.write(); + let rti = &mut rti_lock; + rti.with_public_internet_routing_domain_mut(|detail| { + let old_dial_info_details = detail.dial_info_details().clone(); + let old_relay_node = detail.relay_node(); + let old_outbound_protocols = detail.outbound_protocols(); + let old_inbound_protocols = detail.inbound_protocols(); + let old_address_types = detail.address_types(); + let old_capabilities = detail.capabilities(); + let old_network_class = detail.network_class(); + + for change in self.changes.drain(..) { + match change { + RoutingDomainChangePublicInternet::Common(common_change) => { + detail.apply_common_change(common_change); + } + } + } + + let new_dial_info_details = detail.dial_info_details().clone(); + let new_relay_node = detail.relay_node(); + let new_outbound_protocols = detail.outbound_protocols(); + let new_inbound_protocols = detail.inbound_protocols(); + let new_address_types = detail.address_types(); + let new_capabilities = detail.capabilities(); + let new_network_class = detail.network_class(); + + // Compare and see if peerinfo needs republication + let removed_dial_info = old_dial_info_details + .iter() + .filter(|di| !new_dial_info_details.contains(di)) + .collect::>(); + if !removed_dial_info.is_empty() { + info!( + "[PublicInternet] removed dial info: {:#?}", + removed_dial_info + ); + peer_info_changed = true; + } + let added_dial_info = new_dial_info_details + .iter() + .filter(|di| !old_dial_info_details.contains(di)) + .collect::>(); + if !added_dial_info.is_empty() { + info!("[PublicInternet] added dial info: {:#?}", added_dial_info); + peer_info_changed = true; + } + if let Some(nrn) = new_relay_node { + if let Some(orn) = old_relay_node { + if !nrn.same_entry(&orn) { + info!("[PublicInternet] change relay: {} -> {}", orn, nrn); + peer_info_changed = true; + } + } else { + info!("[PublicInternet] set relay: {}", nrn); + peer_info_changed = true; + } + } + if old_outbound_protocols != new_outbound_protocols { + info!( + "[PublicInternet] changed network: outbound {:?}->{:?}\n", + old_outbound_protocols, new_outbound_protocols + ); + peer_info_changed = true; + } + if old_inbound_protocols != new_inbound_protocols { + info!( + "[PublicInternet] changed network: inbound {:?}->{:?}\n", + old_inbound_protocols, new_inbound_protocols, + ); + peer_info_changed = true; + } + if old_address_types != new_address_types { + info!( + "[PublicInternet] changed network: address types {:?}->{:?}\n", + old_address_types, new_address_types, + ); + peer_info_changed = true; + } + if old_capabilities != new_capabilities { + info!( + "[PublicInternet] changed network: capabilities {:?}->{:?}\n", + old_capabilities, new_capabilities + ); + peer_info_changed = true; + } + if old_network_class != new_network_class { + info!( + "[PublicInternet] changed network class: {:?}->{:?}\n", + old_network_class, new_network_class + ); + peer_info_changed = true; + } + }); + + if peer_info_changed { + // Allow signed node info updates at same timestamp for otherwise dead nodes if our network has changed + rti.reset_all_updated_since_last_network_change(); + } + + peer_info_changed + }) + } + + #[instrument(level = "debug", skip(self))] + fn publish(&mut self) { + let changed = self + .routing_table + .inner + .write() + .publish_peer_info(RoutingDomain::PublicInternet); + + // Clear the routespecstore cache if our PublicInternet dial info has changed + if changed { + let rss = self.routing_table.route_spec_store(); + rss.reset(); + } + } + + #[instrument(level = "debug", skip(self))] + fn shutdown(&mut self) -> SendPinBoxFutureLifetime<'_, ()> { + Box::pin(async move { + self.clear_dial_info_details(None, None) + .set_network_class(None) + .clear_relay_node() + .commit(true) + .await; + self.routing_table + .inner + .write() + .unpublish_peer_info(RoutingDomain::PublicInternet); + }) + } +} diff --git a/veilid-core/src/routing_table/routing_table_inner/routing_domains/public_internet/mod.rs b/veilid-core/src/routing_table/routing_table_inner/routing_domains/public_internet/mod.rs new file mode 100644 index 00000000..f8691f7d --- /dev/null +++ b/veilid-core/src/routing_table/routing_table_inner/routing_domains/public_internet/mod.rs @@ -0,0 +1,369 @@ +mod editor; + +pub use editor::*; + +use super::*; + +/// Public Internet routing domain internals +#[derive(Debug)] +pub struct PublicInternetRoutingDomainDetail { + /// Common implementation for all routing domains + common: RoutingDomainDetailCommon, + /// Published peer info for this routing domain + published_peer_info: Mutex>>, +} + +impl Default for PublicInternetRoutingDomainDetail { + fn default() -> Self { + Self { + common: RoutingDomainDetailCommon::new(RoutingDomain::PublicInternet), + published_peer_info: Default::default(), + } + } +} + +impl RoutingDomainDetailCommonAccessors for PublicInternetRoutingDomainDetail { + fn common(&self) -> &RoutingDomainDetailCommon { + &self.common + } + fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon { + &mut self.common + } +} + +impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { + fn routing_domain(&self) -> RoutingDomain { + RoutingDomain::PublicInternet + } + + fn network_class(&self) -> Option { + self.common.network_class() + } + fn outbound_protocols(&self) -> ProtocolTypeSet { + self.common.outbound_protocols() + } + fn inbound_protocols(&self) -> ProtocolTypeSet { + self.common.inbound_protocols() + } + fn address_types(&self) -> AddressTypeSet { + self.common.address_types() + } + fn capabilities(&self) -> Vec { + self.common.capabilities() + } + fn relay_node(&self) -> Option { + self.common.relay_node() + } + fn relay_node_last_keepalive(&self) -> Option { + self.common.relay_node_last_keepalive() + } + fn dial_info_details(&self) -> &Vec { + self.common.dial_info_details() + } + fn has_valid_network_class(&self) -> bool { + self.common.has_valid_network_class() + } + + fn inbound_dial_info_filter(&self) -> DialInfoFilter { + self.common.inbound_dial_info_filter() + } + fn outbound_dial_info_filter(&self) -> DialInfoFilter { + self.common.outbound_dial_info_filter() + } + + fn get_peer_info(&self, rti: &RoutingTableInner) -> Arc { + self.common.get_peer_info(rti) + } + + fn get_published_peer_info(&self) -> Option> { + (*self.published_peer_info.lock()).clone() + } + + //////////////////////////////////////////////// + + fn can_contain_address(&self, address: Address) -> bool { + address.is_global() + } + + fn refresh(&self) { + self.common.clear_cache(); + } + + fn publish_peer_info(&self, rti: &RoutingTableInner) -> bool { + let pi = self.get_peer_info(rti); + + // If the network class is not yet determined, don't publish + if pi.signed_node_info().node_info().network_class() == NetworkClass::Invalid { + log_rtab!(debug "[PublicInternet] Not publishing peer info with invalid network class"); + return false; + } + + // If we need a relay and we don't have one, don't publish yet + if let Some(_relay_kind) = pi.signed_node_info().node_info().requires_relay() { + if pi.signed_node_info().relay_ids().is_empty() { + log_rtab!(debug "[PublicInternet] Not publishing peer info that wants relay until we have a relay"); + return false; + } + } + + // Don't publish if the peer info hasnt changed from our previous publication + let mut ppi_lock = self.published_peer_info.lock(); + if let Some(old_peer_info) = &*ppi_lock { + if pi.equivalent(old_peer_info) { + log_rtab!(debug "[PublicInternet] Not publishing peer info because it is equivalent"); + return false; + } + } + + log_rtab!(debug "[PublicInternet] Published new peer info: {:#?}", pi); + *ppi_lock = Some(pi); + + true + } + + fn unpublish_peer_info(&self) { + let mut ppi_lock = self.published_peer_info.lock(); + log_rtab!(debug "[PublicInternet] Unpublished peer info"); + *ppi_lock = None; + } + + fn ensure_dial_info_is_valid(&self, dial_info: &DialInfo) -> bool { + let address = dial_info.socket_address().address(); + let can_contain_address = self.can_contain_address(address); + + if !can_contain_address { + log_rtab!(debug "[PublicInternet] can not add dial info to this routing domain: {:?}", dial_info); + return false; + } + if !dial_info.is_valid() { + log_rtab!(debug + "shouldn't be registering invalid addresses: {:?}", + dial_info + ); + return false; + } + true + } + + fn get_contact_method( + &self, + rti: &RoutingTableInner, + peer_a: Arc, + peer_b: Arc, + dial_info_filter: DialInfoFilter, + sequencing: Sequencing, + dif_sort: Option>, + ) -> ContactMethod { + let ip6_prefix_size = rti + .unlocked_inner + .config + .get() + .network + .max_connections_per_ip6_prefix_size as usize; + + // Get the nodeinfos for convenience + let node_a = peer_a.signed_node_info().node_info(); + let node_b = peer_b.signed_node_info().node_info(); + + // Check to see if these nodes are on the same network + let same_ipblock = node_a.node_is_on_same_ipblock(node_b, ip6_prefix_size); + + // Get the node ids that would be used between these peers + let cck = common_crypto_kinds(&peer_a.node_ids().kinds(), &peer_b.node_ids().kinds()); + let Some(best_ck) = cck.first().copied() else { + // No common crypto kinds between these nodes, can't contact + return ContactMethod::Unreachable; + }; + + //let node_a_id = peer_a.node_ids().get(best_ck).unwrap(); + let node_b_id = peer_b.node_ids().get(best_ck).unwrap(); + + // Get the best match dial info for node B if we have it + // Don't try direct inbound at all if the two nodes are on the same ipblock to avoid hairpin NAT issues + // as well avoiding direct traffic between same-network nodes. This would be done in the LocalNetwork RoutingDomain. + if let Some(target_did) = (!same_ipblock) + .then(|| { + first_filtered_dial_info_detail_between_nodes( + node_a, + node_b, + &dial_info_filter, + sequencing, + dif_sort.clone(), + ) + }) + .flatten() + { + // Do we need to signal before going inbound? + if !target_did.class.requires_signal() { + // Go direct without signaling + return ContactMethod::Direct(target_did.dial_info); + } + + // Get the target's inbound relay, it must have one or it is not reachable + if let Some(node_b_relay) = peer_b.signed_node_info().relay_info() { + // Note that relay_peer_info could be node_a, in which case a connection already exists + // and we only get here if the connection had dropped, in which case node_a is unreachable until + // it gets a new relay connection up + if peer_b + .signed_node_info() + .relay_ids() + .contains_any(peer_a.node_ids()) + { + return ContactMethod::Existing; + } + + // Get best node id to contact relay with + let Some(node_b_relay_id) = peer_b.signed_node_info().relay_ids().get(best_ck) + else { + // No best relay id + return ContactMethod::Unreachable; + }; + + // Can node A reach the inbound relay directly? + if first_filtered_dial_info_detail_between_nodes( + node_a, + node_b_relay, + &dial_info_filter, + sequencing, + dif_sort.clone(), + ) + .is_some() + { + // Can node A receive anything inbound ever? + if matches!(node_a.network_class(), NetworkClass::InboundCapable) { + ///////// Reverse connection + + // Get the best match dial info for an reverse inbound connection from node B to node A + if let Some(reverse_did) = first_filtered_dial_info_detail_between_nodes( + node_b, + node_a, + &dial_info_filter, + sequencing, + dif_sort.clone(), + ) { + // Ensure we aren't on the same public IP address (no hairpin nat) + if reverse_did.dial_info.ip_addr() != target_did.dial_info.ip_addr() { + // Can we receive a direct reverse connection? + if !reverse_did.class.requires_signal() { + return ContactMethod::SignalReverse( + node_b_relay_id, + node_b_id, + ); + } + } + } + + ///////// UDP hole-punch + + // Does node B have a direct udp dialinfo node A can reach? + let udp_dial_info_filter = dial_info_filter + .filtered(&DialInfoFilter::all().with_protocol_type(ProtocolType::UDP)); + if let Some(target_udp_did) = first_filtered_dial_info_detail_between_nodes( + node_a, + node_b, + &udp_dial_info_filter, + sequencing, + dif_sort.clone(), + ) { + // Does node A have a direct udp dialinfo that node B can reach? + if let Some(reverse_udp_did) = + first_filtered_dial_info_detail_between_nodes( + node_b, + node_a, + &udp_dial_info_filter, + sequencing, + dif_sort.clone(), + ) + { + // Ensure we aren't on the same public IP address (no hairpin nat) + if reverse_udp_did.dial_info.ip_addr() + != target_udp_did.dial_info.ip_addr() + { + // The target and ourselves have a udp dialinfo that they can reach + return ContactMethod::SignalHolePunch( + node_b_relay_id, + node_b_id, + ); + } + } + } + // Otherwise we have to inbound relay + } + + return ContactMethod::InboundRelay(node_b_relay_id); + } + } + } + // If the node B has no direct dial info or is on the same ipblock, it needs to have an inbound relay + else if let Some(node_b_relay) = peer_b.signed_node_info().relay_info() { + // Note that relay_peer_info could be node_a, in which case a connection already exists + // and we only get here if the connection had dropped, in which case node_b is unreachable until + // it gets a new relay connection up + if peer_b + .signed_node_info() + .relay_ids() + .contains_any(peer_a.node_ids()) + { + return ContactMethod::Existing; + } + + // Get best node id to contact relay with + let Some(node_b_relay_id) = peer_b.signed_node_info().relay_ids().get(best_ck) else { + // No best relay id + return ContactMethod::Unreachable; + }; + + // Can we reach the inbound relay? + if first_filtered_dial_info_detail_between_nodes( + node_a, + node_b_relay, + &dial_info_filter, + sequencing, + dif_sort.clone(), + ) + .is_some() + { + ///////// Reverse connection + + // Get the best match dial info for an reverse inbound connection from node B to node A + // unless both nodes are on the same ipblock + if let Some(reverse_did) = (!same_ipblock) + .then(|| { + first_filtered_dial_info_detail_between_nodes( + node_b, + node_a, + &dial_info_filter, + sequencing, + dif_sort.clone(), + ) + }) + .flatten() + { + // Can we receive a direct reverse connection? + if !reverse_did.class.requires_signal() { + return ContactMethod::SignalReverse(node_b_relay_id, node_b_id); + } + } + + return ContactMethod::InboundRelay(node_b_relay_id); + } + } + + // If node A can't reach the node by other means, it may need to use its outbound relay + if peer_a + .signed_node_info() + .node_info() + .network_class() + .outbound_wants_relay() + { + if let Some(node_a_relay_id) = peer_a.signed_node_info().relay_ids().get(best_ck) { + // Ensure it's not our relay we're trying to reach + if node_a_relay_id != node_b_id { + return ContactMethod::OutboundRelay(node_a_relay_id); + } + } + } + + ContactMethod::Unreachable + } +} diff --git a/veilid-core/src/routing_table/tasks/bootstrap.rs b/veilid-core/src/routing_table/tasks/bootstrap.rs index 474bc7c2..5088ddd9 100644 --- a/veilid-core/src/routing_table/tasks/bootstrap.rs +++ b/veilid-core/src/routing_table/tasks/bootstrap.rs @@ -257,7 +257,7 @@ impl RoutingTable { pub(crate) fn bootstrap_with_peer( self, crypto_kinds: Vec, - pi: PeerInfo, + pi: Arc, unord: &FuturesUnordered>, ) { log_rtab!( @@ -266,7 +266,9 @@ impl RoutingTable { pi.signed_node_info().node_info().dial_info_detail_list() ); - let nr = match self.register_node_with_peer_info(RoutingDomain::PublicInternet, pi, true) { + let routing_domain = pi.routing_domain(); + + let nr = match self.register_node_with_peer_info(pi, true) { Ok(nr) => nr, Err(e) => { log_rtab!(error "failed to register bootstrap peer info: {}", e); @@ -277,14 +279,14 @@ impl RoutingTable { // Add this our futures to process in parallel for crypto_kind in crypto_kinds { // Bootstrap this crypto kind - let nr = nr.clone(); + let nr = nr.unfiltered(); let routing_table = self.clone(); unord.push(Box::pin( async move { // Get what contact method would be used for contacting the bootstrap let bsdi = match routing_table .network_manager() - .get_node_contact_method(nr.clone()) + .get_node_contact_method(nr.default_filtered()) { Ok(NodeContactMethod::Direct(v)) => v, Ok(v) => { @@ -302,10 +304,10 @@ impl RoutingTable { // Need VALID signed peer info, so ask bootstrap to find_node of itself // which will ensure it has the bootstrap's signed peer info as part of the response - let _ = routing_table.find_target(crypto_kind, nr.clone(), vec![]).await; + let _ = routing_table.find_nodes_close_to_node_ref(crypto_kind, nr.clone(), vec![]).await; // Ensure we got the signed peer info - if !nr.signed_node_info_has_valid_signature(RoutingDomain::PublicInternet) { + if !nr.signed_node_info_has_valid_signature(routing_domain) { log_rtab!(warn "bootstrap server is not responding"); log_rtab!(debug "bootstrap server is not responding for dialinfo: {}", bsdi); @@ -324,7 +326,7 @@ impl RoutingTable { #[instrument(level = "trace", skip(self), err)] pub(crate) async fn bootstrap_with_peer_list( self, - peers: Vec, + peers: Vec>, stop_token: StopToken, ) -> EyreResult<()> { log_rtab!(debug " bootstrapped peers: {:?}", &peers); @@ -389,7 +391,7 @@ impl RoutingTable { // Direct bootstrap let network_manager = self.network_manager(); - let mut peer_map = HashMap::::new(); + let mut peer_map = HashMap::>::new(); for bootstrap_di in bootstrap_dialinfos { log_rtab!(debug "direct bootstrap with: {}", bootstrap_di); let peers = network_manager.boot_request(bootstrap_di).await?; @@ -403,7 +405,7 @@ impl RoutingTable { } else { // If not direct, resolve bootstrap servers and recurse their TXT entries let bsrecs = self.resolve_bootstrap(bootstrap).await?; - let peers: Vec = bsrecs + let peers: Vec> = bsrecs .into_iter() .map(|bsrec| { // Get crypto support from list of node ids @@ -413,8 +415,8 @@ impl RoutingTable { let sni = SignedNodeInfo::Direct(SignedDirectNodeInfo::with_no_signature( NodeInfo::new( NetworkClass::InboundCapable, // Bootstraps are always inbound capable - ProtocolTypeSet::only(ProtocolType::UDP), // Bootstraps do not participate in relaying and will not make outbound requests, but will have UDP enabled - AddressTypeSet::all(), // Bootstraps are always IPV4 and IPV6 capable + ProtocolTypeSet::all(), // Bootstraps are always capable of all protocols + AddressTypeSet::all(), // Bootstraps are always IPV4 and IPV6 capable bsrec.envelope_support, // Envelope support is as specified in the bootstrap list crypto_support, // Crypto support is derived from list of node ids vec![], // Bootstrap needs no capabilities @@ -422,7 +424,11 @@ impl RoutingTable { ), )); - PeerInfo::new(bsrec.node_ids, sni) + Arc::new(PeerInfo::new( + RoutingDomain::PublicInternet, + bsrec.node_ids, + sni, + )) }) .collect(); diff --git a/veilid-core/src/routing_table/tasks/closest_peers_refresh.rs b/veilid-core/src/routing_table/tasks/closest_peers_refresh.rs index 22f4e92b..3350f87c 100644 --- a/veilid-core/src/routing_table/tasks/closest_peers_refresh.rs +++ b/veilid-core/src/routing_table/tasks/closest_peers_refresh.rs @@ -56,7 +56,7 @@ impl RoutingTable { self_node_id, filters, |_rti, entry: Option>| { - NodeRef::new(routing_table.clone(), entry.unwrap().clone(), None) + NodeRef::new(routing_table.clone(), entry.unwrap().clone()) }, ) .unwrap(); diff --git a/veilid-core/src/routing_table/tasks/mod.rs b/veilid-core/src/routing_table/tasks/mod.rs index 31a838b7..1b3a4f3f 100644 --- a/veilid-core/src/routing_table/tasks/mod.rs +++ b/veilid-core/src/routing_table/tasks/mod.rs @@ -185,7 +185,13 @@ impl RoutingTable { .closest_peers_refresh_task .tick() .await?; + } + // Only perform these operations if we already have a published peer info + if self + .get_published_peer_info(RoutingDomain::PublicInternet) + .is_some() + { // Run the private route management task self.unlocked_inner .private_route_management_task diff --git a/veilid-core/src/routing_table/tasks/peer_minimum_refresh.rs b/veilid-core/src/routing_table/tasks/peer_minimum_refresh.rs index 34ebfc25..b4fb1e26 100644 --- a/veilid-core/src/routing_table/tasks/peer_minimum_refresh.rs +++ b/veilid-core/src/routing_table/tasks/peer_minimum_refresh.rs @@ -71,7 +71,7 @@ impl RoutingTable { min_peer_count, filters, |_rti, entry: Option>| { - NodeRef::new(routing_table.clone(), entry.unwrap().clone(), None) + NodeRef::new(routing_table.clone(), entry.unwrap().clone()) }, ); diff --git a/veilid-core/src/routing_table/tasks/ping_validator.rs b/veilid-core/src/routing_table/tasks/ping_validator.rs index 077a07d0..90567155 100644 --- a/veilid-core/src/routing_table/tasks/ping_validator.rs +++ b/veilid-core/src/routing_table/tasks/ping_validator.rs @@ -23,9 +23,13 @@ impl RoutingTable { async fn relay_keepalive_public_internet( &self, cur_ts: Timestamp, - relay_nr: NodeRef, futurequeue: &mut VecDeque, ) -> EyreResult<()> { + // Get the PublicInternet relay if we are using one + let Some(relay_nr) = self.relay_node(RoutingDomain::PublicInternet) else { + return Ok(()); + }; + let rpc = self.rpc_processor(); // Get our publicinternet dial info let dids = self.all_filtered_dial_info_details( @@ -45,7 +49,7 @@ impl RoutingTable { return Ok(()); } // Say we're doing this keepalive now - self.edit_routing_domain(RoutingDomain::PublicInternet) + self.edit_public_internet_routing_domain() .set_relay_node_keepalive(Some(cur_ts)) .commit(false) .await; @@ -88,17 +92,14 @@ impl RoutingTable { got_unordered = true; } let dif = did.dial_info.make_filter(); - let relay_nr_filtered = - relay_nr.filtered_clone(NodeRefFilter::new().with_dial_info_filter(dif)); - relay_noderefs.push(relay_nr_filtered); + + relay_noderefs + .push(relay_nr.filtered_clone(NodeRefFilter::new().with_dial_info_filter(dif))); } } // Add noderef filters for ordered or unordered sequencing if we havent already seen those if !got_ordered { - let (_, nrf) = NodeRefFilter::new().with_sequencing(Sequencing::EnsureOrdered); - let mut relay_nr_filtered = relay_nr.filtered_clone(nrf); - relay_nr_filtered.set_sequencing(Sequencing::EnsureOrdered); - relay_noderefs.push(relay_nr_filtered); + relay_noderefs.push(relay_nr.sequencing_clone(Sequencing::EnsureOrdered)); } if !got_unordered { relay_noderefs.push(relay_nr); @@ -158,7 +159,11 @@ impl RoutingTable { log_rtab!("--> Watch ping to {:?}", watch_nr); futurequeue.push_back( - async move { rpc.rpc_call_status(Destination::direct(watch_nr)).await }.boxed(), + async move { + rpc.rpc_call_status(Destination::direct(watch_nr.default_filtered())) + .await + } + .boxed(), ); } Ok(()) @@ -177,14 +182,9 @@ impl RoutingTable { // Get all nodes needing pings in the PublicInternet routing domain let node_refs = self.get_nodes_needing_ping(RoutingDomain::PublicInternet, cur_ts); - // Get the PublicInternet relay if we are using one - let opt_relay_nr = self.relay_node(RoutingDomain::PublicInternet); - - // If this is our relay, let's check for NAT keepalives - if let Some(relay_nr) = opt_relay_nr { - self.relay_keepalive_public_internet(cur_ts, relay_nr, futurequeue) - .await?; - } + // If we have a relay, let's ping for NAT keepalives + self.relay_keepalive_public_internet(cur_ts, futurequeue) + .await?; // Check active watch keepalives self.active_watches_keepalive_public_internet(cur_ts, futurequeue) diff --git a/veilid-core/src/routing_table/tasks/private_route_management.rs b/veilid-core/src/routing_table/tasks/private_route_management.rs index 469a90cf..402cd374 100644 --- a/veilid-core/src/routing_table/tasks/private_route_management.rs +++ b/veilid-core/src/routing_table/tasks/private_route_management.rs @@ -117,7 +117,6 @@ impl RoutingTable { let rss = self.route_spec_store(); #[derive(Default, Debug)] struct TestRouteContext { - failed: bool, dead_routes: Vec, } @@ -130,17 +129,13 @@ impl RoutingTable { unord.push( async move { let success = match rss.test_route(r).await { - Ok(v) => v, - // Route was already removed - Err(VeilidAPIError::InvalidArgument { - context: _, - argument: _, - value: _, - }) => false, - // Other failures + // Test had result + Ok(Some(v)) => v, + // Test could not be performed at this time + Ok(None) => true, + // Test failure Err(e) => { log_rtab!(error "Test route failed: {}", e); - ctx.lock().failed = true; return; } }; diff --git a/veilid-core/src/routing_table/tasks/relay_management.rs b/veilid-core/src/routing_table/tasks/relay_management.rs index 2a88ec96..7827f381 100644 --- a/veilid-core/src/routing_table/tasks/relay_management.rs +++ b/veilid-core/src/routing_table/tasks/relay_management.rs @@ -4,7 +4,7 @@ impl RoutingTable { // Check if a relay is desired or not #[instrument(level = "trace", skip_all)] fn public_internet_wants_relay(&self) -> Option { - let own_peer_info = self.get_own_peer_info(RoutingDomain::PublicInternet); + let own_peer_info = self.get_current_peer_info(RoutingDomain::PublicInternet); let own_node_info = own_peer_info.signed_node_info().node_info(); let network_class = own_node_info.network_class(); @@ -34,7 +34,7 @@ impl RoutingTable { for did in own_node_info.dial_info_detail_list() { inbound_addresses.insert(did.dial_info.to_socket_addr()); } - let own_local_peer_info = self.get_own_peer_info(RoutingDomain::LocalNetwork); + let own_local_peer_info = self.get_current_peer_info(RoutingDomain::LocalNetwork); let own_local_node_info = own_local_peer_info.signed_node_info().node_info(); for ldid in own_local_node_info.dial_info_detail_list() { inbound_addresses.remove(&ldid.dial_info.to_socket_addr()); @@ -59,7 +59,7 @@ impl RoutingTable { let relay_desired = self.public_internet_wants_relay(); // Get routing domain editor - let mut editor = self.edit_routing_domain(RoutingDomain::PublicInternet); + let mut editor = self.edit_public_internet_routing_domain(); // If we already have a relay, see if it is dead, or if we don't need it any more let has_relay = { @@ -107,16 +107,14 @@ impl RoutingTable { let mut got_outbound_relay = false; if matches!(relay_desired, RelayKind::Outbound) { // The outbound relay is the host of the PWA - if let Some(outbound_relay_peerinfo) = intf::get_outbound_relay_peer().await { + if let Some(outbound_relay_peerinfo) = + intf::get_outbound_relay_peer(RoutingDomain::PublicInternet).await + { // Register new outbound relay - match self.register_node_with_peer_info( - RoutingDomain::PublicInternet, - outbound_relay_peerinfo, - false, - ) { + match self.register_node_with_peer_info(outbound_relay_peerinfo, false) { Ok(nr) => { log_rtab!(debug "Outbound relay node selected: {}", nr); - editor.set_relay_node(nr); + editor.set_relay_node(nr.unfiltered()); got_outbound_relay = true; } Err(e) => { @@ -141,7 +139,14 @@ impl RoutingTable { } // Commit the changes - editor.commit(false).await; + if editor.commit(false).await { + // Try to publish the peer info + editor.publish(); + + self.network_manager() + .connection_manager() + .update_protections(); + } Ok(()) } @@ -152,7 +157,7 @@ impl RoutingTable { let outbound_dif = self.get_outbound_dial_info_filter(RoutingDomain::PublicInternet); let mapped_port_info = self.get_low_level_port_info(); let own_node_info = self - .get_own_peer_info(RoutingDomain::PublicInternet) + .get_current_peer_info(RoutingDomain::PublicInternet) .signed_node_info() .node_info() .clone(); @@ -174,11 +179,9 @@ impl RoutingTable { // Exclude any nodes that have 'failed to send' state indicating a // connection drop or inability to reach the node - - // XXX: we should be able to enable this! - // if e.peer_stats().rpc_stats.failed_to_send > 0 { - // return false; - // } + if e.peer_stats().rpc_stats.failed_to_send > 0 { + return false; + } // Until we have a way of reducing a SignedRelayedNodeInfo to a SignedDirectNodeInfo // See https://gitlab.com/veilid/veilid/-/issues/381 @@ -200,7 +203,7 @@ impl RoutingTable { // Disqualify nodes that don't cover all our inbound ports for tcp and udp // as we need to be able to use the relay for keepalives for all nat mappings let mut low_level_protocol_ports = mapped_port_info.low_level_protocol_ports.clone(); - let dids = node_info.all_filtered_dial_info_details(DialInfoDetail::NO_SORT, |did| { + let dids = node_info.filtered_dial_info_details(DialInfoDetail::NO_SORT, |did| { did.matches_filter(&outbound_dif) }); for did in &dids { @@ -282,6 +285,6 @@ impl RoutingTable { Option::<()>::None }); // Return the best inbound relay noderef - best_inbound_relay.map(|e| NodeRef::new(self.clone(), e, None)) + best_inbound_relay.map(|e| NodeRef::new(self.clone(), e)) } } diff --git a/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs b/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs index bd81e8c2..91576a4e 100644 --- a/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs +++ b/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs @@ -65,6 +65,7 @@ pub async fn test_round_trip_peerinfo() { ]), )); let pi: PeerInfo = PeerInfo::new( + RoutingDomain::PublicInternet, tks, SignedNodeInfo::Direct(SignedDirectNodeInfo::new( NodeInfo::new( diff --git a/veilid-core/src/routing_table/types/contact_method.rs b/veilid-core/src/routing_table/types/contact_method.rs new file mode 100644 index 00000000..72885355 --- /dev/null +++ b/veilid-core/src/routing_table/types/contact_method.rs @@ -0,0 +1,20 @@ +use super::*; + +/// Mechanism required to contact another node +#[derive(Clone, Debug)] +pub(crate) enum ContactMethod { + /// Node is not reachable by any means + Unreachable, + /// Connection should have already existed + Existing, + /// Contact the node directly + Direct(DialInfo), + /// Request via signal the node connect back directly (relay, target) + SignalReverse(TypedKey, TypedKey), + /// Request via signal the node negotiate a hole punch (relay, target) + SignalHolePunch(TypedKey, TypedKey), + /// Must use an inbound relay to reach the node + InboundRelay(TypedKey), + /// Must use outbound relay to reach the node + OutboundRelay(TypedKey), +} diff --git a/veilid-core/src/routing_table/types/mod.rs b/veilid-core/src/routing_table/types/mod.rs index 217c5d48..0bf49377 100644 --- a/veilid-core/src/routing_table/types/mod.rs +++ b/veilid-core/src/routing_table/types/mod.rs @@ -1,3 +1,4 @@ +mod contact_method; mod dial_info_detail; mod direction; mod node_info; @@ -10,6 +11,7 @@ mod signed_relayed_node_info; use super::*; +pub(crate) use contact_method::*; pub use dial_info_detail::*; pub use direction::*; pub use node_info::*; diff --git a/veilid-core/src/routing_table/types/node_info.rs b/veilid-core/src/routing_table/types/node_info.rs index 2a05084b..13c84137 100644 --- a/veilid-core/src/routing_table/types/node_info.rs +++ b/veilid-core/src/routing_table/types/node_info.rs @@ -96,7 +96,7 @@ impl NodeInfo { None } - pub fn all_filtered_dial_info_details( + pub fn filtered_dial_info_details( &self, sort: Option, filter: F, diff --git a/veilid-core/src/routing_table/types/peer_info.rs b/veilid-core/src/routing_table/types/peer_info.rs index 7a143790..462f3678 100644 --- a/veilid-core/src/routing_table/types/peer_info.rs +++ b/veilid-core/src/routing_table/types/peer_info.rs @@ -2,14 +2,20 @@ use super::*; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct PeerInfo { + routing_domain: RoutingDomain, node_ids: TypedKeyGroup, signed_node_info: SignedNodeInfo, } impl PeerInfo { - pub fn new(node_ids: TypedKeyGroup, signed_node_info: SignedNodeInfo) -> Self { + pub fn new( + routing_domain: RoutingDomain, + node_ids: TypedKeyGroup, + signed_node_info: SignedNodeInfo, + ) -> Self { assert!(!node_ids.is_empty() && node_ids.len() <= MAX_CRYPTO_KINDS); Self { + routing_domain, node_ids, signed_node_info, } @@ -24,17 +30,20 @@ impl PeerInfo { Ok(()) } + pub fn routing_domain(&self) -> RoutingDomain { + self.routing_domain + } pub fn node_ids(&self) -> &TypedKeyGroup { &self.node_ids } pub fn signed_node_info(&self) -> &SignedNodeInfo { &self.signed_node_info } - pub fn destructure(self) -> (TypedKeyGroup, SignedNodeInfo) { - (self.node_ids, self.signed_node_info) + pub fn destructure(self) -> (RoutingDomain, TypedKeyGroup, SignedNodeInfo) { + (self.routing_domain, self.node_ids, self.signed_node_info) } - pub fn validate_vec(peer_info_vec: &mut Vec, crypto: Crypto) { + pub fn validate_vec(peer_info_vec: &mut Vec>, crypto: Crypto) { let mut n = 0usize; while n < peer_info_vec.len() { let pi = peer_info_vec.get(n).unwrap(); @@ -45,4 +54,16 @@ impl PeerInfo { } } } + + /// Compare this PeerInfo to another one + /// Exclude the signature and timestamp and any other fields that are not + /// semantically valuable + /// If the two are not equivalent they should be considered different + /// enough for republication, but this is not the only criteria required + /// for publication. + pub fn equivalent(&self, other: &PeerInfo) -> bool { + self.routing_domain == other.routing_domain + && self.node_ids == other.node_ids + && self.signed_node_info.equivalent(&other.signed_node_info) + } } diff --git a/veilid-core/src/routing_table/types/signed_direct_node_info.rs b/veilid-core/src/routing_table/types/signed_direct_node_info.rs index c78491a8..77b96cf3 100644 --- a/veilid-core/src/routing_table/types/signed_direct_node_info.rs +++ b/veilid-core/src/routing_table/types/signed_direct_node_info.rs @@ -96,4 +96,13 @@ impl SignedDirectNodeInfo { pub fn signatures(&self) -> &[TypedSignature] { &self.signatures } + + /// Compare this SignedDirectNodeInfo to another one + /// Exclude the signature and timestamp and any other fields that are not + /// semantically valuable + pub fn equivalent(&self, other: &SignedDirectNodeInfo) -> bool { + let a = self.node_info(); + let b = other.node_info(); + a == b + } } diff --git a/veilid-core/src/routing_table/types/signed_node_info.rs b/veilid-core/src/routing_table/types/signed_node_info.rs index d56597ea..0d331218 100644 --- a/veilid-core/src/routing_table/types/signed_node_info.rs +++ b/veilid-core/src/routing_table/types/signed_node_info.rs @@ -49,13 +49,14 @@ impl SignedNodeInfo { SignedNodeInfo::Relayed(r) => Some(r.relay_info().node_info()), } } - pub fn relay_peer_info(&self) -> Option { + pub fn relay_peer_info(&self, routing_domain: RoutingDomain) -> Option> { match self { SignedNodeInfo::Direct(_) => None, - SignedNodeInfo::Relayed(r) => Some(PeerInfo::new( + SignedNodeInfo::Relayed(r) => Some(Arc::new(PeerInfo::new( + routing_domain, r.relay_ids().clone(), SignedNodeInfo::Direct(r.relay_info().clone()), - )), + ))), } } pub fn has_any_dial_info(&self) -> bool { @@ -96,4 +97,20 @@ impl SignedNodeInfo { }) .unwrap_or_default(); } + + /// Compare this SignedNodeInfo to another one + /// Exclude the signature and timestamp and any other fields that are not + /// semantically valuable + pub fn equivalent(&self, other: &SignedNodeInfo) -> bool { + match self { + SignedNodeInfo::Direct(d) => match other { + SignedNodeInfo::Direct(pd) => d.equivalent(pd), + SignedNodeInfo::Relayed(_) => true, + }, + SignedNodeInfo::Relayed(r) => match other { + SignedNodeInfo::Direct(_) => true, + SignedNodeInfo::Relayed(pr) => r.equivalent(pr), + }, + } + } } diff --git a/veilid-core/src/routing_table/types/signed_relayed_node_info.rs b/veilid-core/src/routing_table/types/signed_relayed_node_info.rs index 470903d4..1f8bfd44 100644 --- a/veilid-core/src/routing_table/types/signed_relayed_node_info.rs +++ b/veilid-core/src/routing_table/types/signed_relayed_node_info.rs @@ -141,4 +141,18 @@ impl SignedRelayedNodeInfo { pub fn signatures(&self) -> &[TypedSignature] { &self.signatures } + + /// Compare this SignedRelayedNodeInfo to another one + /// Exclude the signature and timestamp and any other fields that are not + /// semantically valuable + pub fn equivalent(&self, other: &SignedRelayedNodeInfo) -> bool { + let a = self.node_info(); + let b = other.node_info(); + let ari = self.relay_ids(); + let bri = other.relay_ids(); + let ar = self.relay_info(); + let br = other.relay_info(); + + a == b && ari == bri && ar.equivalent(br) + } } diff --git a/veilid-core/src/rpc_processor/coders/mod.rs b/veilid-core/src/rpc_processor/coders/mod.rs index 3778d953..deb1d3ea 100644 --- a/veilid-core/src/rpc_processor/coders/mod.rs +++ b/veilid-core/src/rpc_processor/coders/mod.rs @@ -74,3 +74,8 @@ pub(in crate::rpc_processor) struct RPCValidateContext { // pub rpc_processor: RPCProcessor, pub question_context: Option, } + +#[derive(Clone)] +pub(crate) struct RPCDecodeContext { + pub routing_domain: RoutingDomain, +} diff --git a/veilid-core/src/rpc_processor/coders/operations/answer.rs b/veilid-core/src/rpc_processor/coders/operations/answer.rs index 257efba6..f40962e6 100644 --- a/veilid-core/src/rpc_processor/coders/operations/answer.rs +++ b/veilid-core/src/rpc_processor/coders/operations/answer.rs @@ -12,16 +12,18 @@ impl RPCAnswer { pub fn validate(&mut self, validate_context: &RPCValidateContext) -> Result<(), RPCError> { self.detail.validate(validate_context) } - #[cfg(feature = "verbose-tracing")] pub fn desc(&self) -> &'static str { self.detail.desc() } pub fn destructure(self) -> RPCAnswerDetail { self.detail } - pub fn decode(reader: &veilid_capnp::answer::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::answer::Reader, + ) -> Result { let d_reader = reader.get_detail(); - let detail = RPCAnswerDetail::decode(&d_reader)?; + let detail = RPCAnswerDetail::decode(decode_context, &d_reader)?; Ok(RPCAnswer { detail }) } pub fn encode(&self, builder: &mut veilid_capnp::answer::Builder) -> Result<(), RPCError> { @@ -52,7 +54,6 @@ pub(in crate::rpc_processor) enum RPCAnswerDetail { } impl RPCAnswerDetail { - #[cfg(feature = "verbose-tracing")] pub fn desc(&self) -> &'static str { match self { RPCAnswerDetail::StatusA(_) => "StatusA", @@ -96,73 +97,74 @@ impl RPCAnswerDetail { } } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::answer::detail::Reader, ) -> Result { let which_reader = reader.which().map_err(RPCError::protocol)?; let out = match which_reader { veilid_capnp::answer::detail::StatusA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationStatusA::decode(&op_reader)?; + let out = RPCOperationStatusA::decode(decode_context, &op_reader)?; RPCAnswerDetail::StatusA(Box::new(out)) } veilid_capnp::answer::detail::FindNodeA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationFindNodeA::decode(&op_reader)?; + let out = RPCOperationFindNodeA::decode(decode_context, &op_reader)?; RPCAnswerDetail::FindNodeA(Box::new(out)) } veilid_capnp::answer::detail::AppCallA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationAppCallA::decode(&op_reader)?; + let out = RPCOperationAppCallA::decode(decode_context, &op_reader)?; RPCAnswerDetail::AppCallA(Box::new(out)) } veilid_capnp::answer::detail::GetValueA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationGetValueA::decode(&op_reader)?; + let out = RPCOperationGetValueA::decode(decode_context, &op_reader)?; RPCAnswerDetail::GetValueA(Box::new(out)) } veilid_capnp::answer::detail::SetValueA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationSetValueA::decode(&op_reader)?; + let out = RPCOperationSetValueA::decode(decode_context, &op_reader)?; RPCAnswerDetail::SetValueA(Box::new(out)) } veilid_capnp::answer::detail::WatchValueA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationWatchValueA::decode(&op_reader)?; + let out = RPCOperationWatchValueA::decode(decode_context, &op_reader)?; RPCAnswerDetail::WatchValueA(Box::new(out)) } veilid_capnp::answer::detail::InspectValueA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationInspectValueA::decode(&op_reader)?; + let out = RPCOperationInspectValueA::decode(decode_context, &op_reader)?; RPCAnswerDetail::InspectValueA(Box::new(out)) } #[cfg(feature = "unstable-blockstore")] veilid_capnp::answer::detail::SupplyBlockA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationSupplyBlockA::decode(&op_reader)?; + let out = RPCOperationSupplyBlockA::decode(decode_context, &op_reader)?; RPCAnswerDetail::SupplyBlockA(Box::new(out)) } #[cfg(feature = "unstable-blockstore")] veilid_capnp::answer::detail::FindBlockA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationFindBlockA::decode(&op_reader)?; + let out = RPCOperationFindBlockA::decode(decode_context, &op_reader)?; RPCAnswerDetail::FindBlockA(Box::new(out)) } #[cfg(feature = "unstable-tunnels")] veilid_capnp::answer::detail::StartTunnelA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationStartTunnelA::decode(&op_reader)?; + let out = RPCOperationStartTunnelA::decode(decode_context, &op_reader)?; RPCAnswerDetail::StartTunnelA(Box::new(out)) } #[cfg(feature = "unstable-tunnels")] veilid_capnp::answer::detail::CompleteTunnelA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationCompleteTunnelA::decode(&op_reader)?; + let out = RPCOperationCompleteTunnelA::decode(decode_context, &op_reader)?; RPCAnswerDetail::CompleteTunnelA(Box::new(out)) } #[cfg(feature = "unstable-tunnels")] veilid_capnp::answer::detail::CancelTunnelA(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationCancelTunnelA::decode(&op_reader)?; + let out = RPCOperationCancelTunnelA::decode(decode_context, &op_reader)?; RPCAnswerDetail::CancelTunnelA(Box::new(out)) } }; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation.rs b/veilid-core/src/rpc_processor/coders/operations/operation.rs index 9e070ef1..4a1c2947 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation.rs @@ -8,7 +8,6 @@ pub(in crate::rpc_processor) enum RPCOperationKind { } impl RPCOperationKind { - #[cfg(feature = "verbose-tracing")] pub fn desc(&self) -> &'static str { match self { RPCOperationKind::Question(q) => q.desc(), @@ -25,22 +24,25 @@ impl RPCOperationKind { } } - pub fn decode(kind_reader: &veilid_capnp::operation::kind::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + kind_reader: &veilid_capnp::operation::kind::Reader, + ) -> Result { let which_reader = kind_reader.which().map_err(RPCError::protocol)?; let out = match which_reader { veilid_capnp::operation::kind::Which::Question(r) => { let q_reader = r.map_err(RPCError::protocol)?; - let out = RPCQuestion::decode(&q_reader)?; + let out = RPCQuestion::decode(decode_context, &q_reader)?; RPCOperationKind::Question(Box::new(out)) } veilid_capnp::operation::kind::Which::Statement(r) => { let q_reader = r.map_err(RPCError::protocol)?; - let out = RPCStatement::decode(&q_reader)?; + let out = RPCStatement::decode(decode_context, &q_reader)?; RPCOperationKind::Statement(Box::new(out)) } veilid_capnp::operation::kind::Which::Answer(r) => { let q_reader = r.map_err(RPCError::protocol)?; - let out = RPCAnswer::decode(&q_reader)?; + let out = RPCAnswer::decode(decode_context, &q_reader)?; RPCOperationKind::Answer(Box::new(out)) } }; @@ -63,8 +65,7 @@ impl RPCOperationKind { #[derive(Debug, Clone)] pub(in crate::rpc_processor) struct RPCOperation { op_id: OperationId, - opt_sender_peer_info: Option, - target_node_info_ts: Timestamp, + sender_peer_info: SenderPeerInfo, kind: RPCOperationKind, } @@ -72,16 +73,14 @@ impl RPCOperation { pub fn new_question(question: RPCQuestion, sender_peer_info: SenderPeerInfo) -> Self { Self { op_id: OperationId::new(get_random_u64()), - opt_sender_peer_info: sender_peer_info.opt_sender_peer_info, - target_node_info_ts: sender_peer_info.target_node_info_ts, + sender_peer_info, kind: RPCOperationKind::Question(Box::new(question)), } } pub fn new_statement(statement: RPCStatement, sender_peer_info: SenderPeerInfo) -> Self { Self { op_id: OperationId::new(get_random_u64()), - opt_sender_peer_info: sender_peer_info.opt_sender_peer_info, - target_node_info_ts: sender_peer_info.target_node_info_ts, + sender_peer_info, kind: RPCOperationKind::Statement(Box::new(statement)), } } @@ -93,15 +92,14 @@ impl RPCOperation { ) -> Self { Self { op_id: request.op_id, - opt_sender_peer_info: sender_peer_info.opt_sender_peer_info, - target_node_info_ts: sender_peer_info.target_node_info_ts, + sender_peer_info, kind: RPCOperationKind::Answer(Box::new(answer)), } } pub fn validate(&mut self, validate_context: &RPCValidateContext) -> Result<(), RPCError> { // Validate sender peer info - if let Some(sender_peer_info) = &self.opt_sender_peer_info { + if let Some(sender_peer_info) = &self.sender_peer_info.opt_peer_info { sender_peer_info .validate(validate_context.crypto.clone()) .map_err(RPCError::protocol)?; @@ -115,34 +113,29 @@ impl RPCOperation { self.op_id } - pub fn sender_peer_info(&self) -> Option<&PeerInfo> { - self.opt_sender_peer_info.as_ref() - } - pub fn target_node_info_ts(&self) -> Timestamp { - self.target_node_info_ts + pub fn sender_peer_info(&self) -> SenderPeerInfo { + self.sender_peer_info.clone() } pub fn kind(&self) -> &RPCOperationKind { &self.kind } - pub fn destructure(self) -> (OperationId, Option, Timestamp, RPCOperationKind) { - ( - self.op_id, - self.opt_sender_peer_info, - self.target_node_info_ts, - self.kind, - ) + pub fn destructure(self) -> (OperationId, SenderPeerInfo, RPCOperationKind) { + (self.op_id, self.sender_peer_info, self.kind) } - pub fn decode(operation_reader: &veilid_capnp::operation::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + operation_reader: &veilid_capnp::operation::Reader, + ) -> Result { let op_id = OperationId::new(operation_reader.get_op_id()); - let sender_peer_info = if operation_reader.has_sender_peer_info() { + let opt_peer_info = if operation_reader.has_sender_peer_info() { let pi_reader = operation_reader .get_sender_peer_info() .map_err(RPCError::protocol)?; - let pi = decode_peer_info(&pi_reader)?; + let pi = Arc::new(decode_peer_info(decode_context, &pi_reader)?); Some(pi) } else { None @@ -151,23 +144,25 @@ impl RPCOperation { let target_node_info_ts = Timestamp::new(operation_reader.get_target_node_info_ts()); let kind_reader = operation_reader.get_kind(); - let kind = RPCOperationKind::decode(&kind_reader)?; + let kind = RPCOperationKind::decode(decode_context, &kind_reader)?; Ok(RPCOperation { op_id, - opt_sender_peer_info: sender_peer_info, - target_node_info_ts, + sender_peer_info: SenderPeerInfo { + opt_peer_info, + target_node_info_ts, + }, kind, }) } pub fn encode(&self, builder: &mut veilid_capnp::operation::Builder) -> Result<(), RPCError> { builder.set_op_id(self.op_id.as_u64()); - if let Some(sender_peer_info) = &self.opt_sender_peer_info { + if let Some(sender_peer_info) = &self.sender_peer_info.opt_peer_info { let mut pi_builder = builder.reborrow().init_sender_peer_info(); encode_peer_info(sender_peer_info, &mut pi_builder)?; } - builder.set_target_node_info_ts(self.target_node_info_ts.as_u64()); + builder.set_target_node_info_ts(self.sender_peer_info.target_node_info_ts.as_u64()); let mut k_builder = builder.reborrow().init_kind(); self.kind.encode(&mut k_builder)?; Ok(()) diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs b/veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs index 55c435d3..9b5fb1f3 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs @@ -27,7 +27,10 @@ impl RPCOperationAppCallQ { self.message } - pub fn decode(reader: &veilid_capnp::operation_app_call_q::Reader) -> Result { + pub fn decode( + _decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_app_call_q::Reader, + ) -> Result { let mr = reader.get_message().map_err(RPCError::protocol)?; if mr.len() > MAX_APP_CALL_Q_MESSAGE_LEN { return Err(RPCError::protocol("AppCallQ message too long to set")); @@ -72,7 +75,10 @@ impl RPCOperationAppCallA { self.message } - pub fn decode(reader: &veilid_capnp::operation_app_call_a::Reader) -> Result { + pub fn decode( + _decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_app_call_a::Reader, + ) -> Result { let mr = reader.get_message().map_err(RPCError::protocol)?; if mr.len() > MAX_APP_CALL_A_MESSAGE_LEN { return Err(RPCError::protocol("AppCallA message too long to set")); diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs b/veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs index 13dd7f8b..91155806 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs @@ -26,7 +26,10 @@ impl RPCOperationAppMessage { self.message } - pub fn decode(reader: &veilid_capnp::operation_app_message::Reader) -> Result { + pub fn decode( + _decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_app_message::Reader, + ) -> Result { let mr = reader.get_message().map_err(RPCError::protocol)?; if mr.len() > MAX_APP_MESSAGE_MESSAGE_LEN { return Err(RPCError::protocol("AppMessage message too long to set")); diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs b/veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs index 41c34248..40387286 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs @@ -23,6 +23,7 @@ impl RPCOperationCancelTunnelQ { } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_cancel_tunnel_q::Reader, ) -> Result { let id = TunnelId::new(reader.get_id()); diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs b/veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs index 36a4b695..dc96db35 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs @@ -40,6 +40,7 @@ impl RPCOperationCompleteTunnelQ { } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_complete_tunnel_q::Reader, ) -> Result { let id = TunnelId::new(reader.get_id()); diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs b/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs index 1fe48924..096436ed 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs @@ -26,6 +26,7 @@ impl RPCOperationFindBlockQ { } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_find_block_q::Reader, ) -> Result { let bi_reader = reader.get_block_id().map_err(RPCError::protocol)?; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs b/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs index 26efcc98..cba032ca 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs @@ -30,7 +30,10 @@ impl RPCOperationFindNodeQ { (self.node_id, self.capabilities) } - pub fn decode(reader: &veilid_capnp::operation_find_node_q::Reader) -> Result { + pub fn decode( + _decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_find_node_q::Reader, + ) -> Result { let ni_reader = reader.get_node_id().map_err(RPCError::protocol)?; let node_id = decode_typed_key(&ni_reader)?; let cap_reader = reader @@ -77,11 +80,11 @@ impl RPCOperationFindNodeQ { #[derive(Debug, Clone)] pub(in crate::rpc_processor) struct RPCOperationFindNodeA { - peers: Vec, + peers: Vec>, } impl RPCOperationFindNodeA { - pub fn new(peers: Vec) -> Result { + pub fn new(peers: Vec>) -> Result { if peers.len() > MAX_FIND_NODE_A_PEERS_LEN { return Err(RPCError::protocol( "encoded find node peers length too long", @@ -100,11 +103,12 @@ impl RPCOperationFindNodeA { // &self.peers // } - pub fn destructure(self) -> Vec { + pub fn destructure(self) -> Vec> { self.peers } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_find_node_a::Reader, ) -> Result { let peers_reader = reader.get_peers().map_err(RPCError::protocol)?; @@ -115,15 +119,15 @@ impl RPCOperationFindNodeA { )); } - let mut peers = Vec::::with_capacity( + let mut peers = Vec::>::with_capacity( peers_reader .len() .try_into() .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p)?; - peers.push(peer_info); + let peer_info = decode_peer_info(decode_context, &p)?; + peers.push(Arc::new(peer_info)); } Ok(Self { peers }) diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs b/veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs index 1652b6fb..35366884 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs @@ -52,7 +52,10 @@ impl RPCOperationGetValueQ { (self.key, self.subkey, self.want_descriptor) } - pub fn decode(reader: &veilid_capnp::operation_get_value_q::Reader) -> Result { + pub fn decode( + _decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_get_value_q::Reader, + ) -> Result { let k_reader = reader.reborrow().get_key().map_err(RPCError::protocol)?; let key = decode_typed_key(&k_reader)?; let subkey = reader.reborrow().get_subkey(); @@ -80,14 +83,14 @@ impl RPCOperationGetValueQ { #[derive(Debug, Clone)] pub(in crate::rpc_processor) struct RPCOperationGetValueA { value: Option, - peers: Vec, + peers: Vec>, descriptor: Option, } impl RPCOperationGetValueA { pub fn new( value: Option, - peers: Vec, + peers: Vec>, descriptor: Option, ) -> Result { if peers.len() > MAX_GET_VALUE_A_PEERS_LEN { @@ -171,13 +174,16 @@ impl RPCOperationGetValueA { self, ) -> ( Option, - Vec, + Vec>, Option, ) { (self.value, self.peers, self.descriptor) } - pub fn decode(reader: &veilid_capnp::operation_get_value_a::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_get_value_a::Reader, + ) -> Result { let value = if reader.has_value() { let value_reader = reader.get_value().map_err(RPCError::protocol)?; let value = decode_signed_value_data(&value_reader)?; @@ -192,14 +198,14 @@ impl RPCOperationGetValueA { "decoded GetValueA peers length too long", )); } - let mut peers = Vec::::with_capacity( + let mut peers = Vec::>::with_capacity( peers_reader .len() .try_into() .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p)?; + let peer_info = Arc::new(decode_peer_info(decode_context, &p)?); peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_inspect_value.rs b/veilid-core/src/rpc_processor/coders/operations/operation_inspect_value.rs index 197943e9..20ad0429 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_inspect_value.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_inspect_value.rs @@ -58,6 +58,7 @@ impl RPCOperationInspectValueQ { } pub fn decode( + _decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_inspect_value_q::Reader, ) -> Result { let k_reader = reader.reborrow().get_key().map_err(RPCError::protocol)?; @@ -118,14 +119,14 @@ impl RPCOperationInspectValueQ { #[derive(Debug, Clone)] pub(in crate::rpc_processor) struct RPCOperationInspectValueA { seqs: Vec, - peers: Vec, + peers: Vec>, descriptor: Option, } impl RPCOperationInspectValueA { pub fn new( seqs: Vec, - peers: Vec, + peers: Vec>, descriptor: Option, ) -> Result { if seqs.len() > MAX_INSPECT_VALUE_A_SEQS_LEN { @@ -198,13 +199,14 @@ impl RPCOperationInspectValueA { self, ) -> ( Vec, - Vec, + Vec>, Option, ) { (self.seqs, self.peers, self.descriptor) } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_inspect_value_a::Reader, ) -> Result { let seqs = if reader.has_seqs() { @@ -228,14 +230,14 @@ impl RPCOperationInspectValueA { "decoded InspectValueA peers length too long", )); } - let mut peers = Vec::::with_capacity( + let mut peers = Vec::>::with_capacity( peers_reader .len() .try_into() .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p)?; + let peer_info = Arc::new(decode_peer_info(decode_context, &p)?); peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_return_receipt.rs b/veilid-core/src/rpc_processor/coders/operations/operation_return_receipt.rs index 4c0c5a6d..38bda921 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_return_receipt.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_return_receipt.rs @@ -29,6 +29,7 @@ impl RPCOperationReturnReceipt { } pub fn decode( + _decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_return_receipt::Reader, ) -> Result { let rr = reader.get_receipt().map_err(RPCError::protocol)?; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_route.rs b/veilid-core/src/rpc_processor/coders/operations/operation_route.rs index 510148af..443a6590 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_route.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_route.rs @@ -2,6 +2,7 @@ use super::*; #[derive(Clone)] pub(in crate::rpc_processor) struct RoutedOperation { + routing_domain: RoutingDomain, sequencing: Sequencing, signatures: Vec, nonce: Nonce, @@ -11,6 +12,7 @@ pub(in crate::rpc_processor) struct RoutedOperation { impl fmt::Debug for RoutedOperation { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RoutedOperation") + .field("routing_domain", &self.routing_domain) .field("sequencing", &self.sequencing) .field("signatures.len", &self.signatures.len()) .field("nonce", &self.nonce) @@ -20,8 +22,14 @@ impl fmt::Debug for RoutedOperation { } impl RoutedOperation { - pub fn new(sequencing: Sequencing, nonce: Nonce, data: Vec) -> Self { + pub fn new( + routing_domain: RoutingDomain, + sequencing: Sequencing, + nonce: Nonce, + data: Vec, + ) -> Self { Self { + routing_domain, sequencing, signatures: Vec::new(), nonce, @@ -32,6 +40,9 @@ impl RoutedOperation { //xxx Ok(()) } + pub fn routing_domain(&self) -> RoutingDomain { + self.routing_domain + } pub fn sequencing(&self) -> Sequencing { self.sequencing } @@ -54,7 +65,10 @@ impl RoutedOperation { // (self.sequencing, self.signatures, self.nonce, self.data) // } - pub fn decode(reader: &veilid_capnp::routed_operation::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::routed_operation::Reader, + ) -> Result { let sigs_reader = reader.get_signatures().map_err(RPCError::protocol)?; let mut signatures = Vec::::with_capacity( sigs_reader @@ -73,6 +87,7 @@ impl RoutedOperation { let data = reader.get_data().map_err(RPCError::protocol)?; Ok(Self { + routing_domain: decode_context.routing_domain, sequencing, signatures, nonce, @@ -132,12 +147,15 @@ impl RPCOperationRoute { (self.safety_route, self.operation) } - pub fn decode(reader: &veilid_capnp::operation_route::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_route::Reader, + ) -> Result { let sr_reader = reader.get_safety_route().map_err(RPCError::protocol)?; - let safety_route = decode_safety_route(&sr_reader)?; + let safety_route = decode_safety_route(decode_context, &sr_reader)?; let o_reader = reader.get_operation().map_err(RPCError::protocol)?; - let operation = RoutedOperation::decode(&o_reader)?; + let operation = RoutedOperation::decode(decode_context, &o_reader)?; Ok(Self { safety_route, diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs b/veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs index 8cd58636..3c1aedcb 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs @@ -72,7 +72,10 @@ impl RPCOperationSetValueQ { (self.key, self.subkey, self.value, self.descriptor) } - pub fn decode(reader: &veilid_capnp::operation_set_value_q::Reader) -> Result { + pub fn decode( + _decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_set_value_q::Reader, + ) -> Result { let k_reader = reader.get_key().map_err(RPCError::protocol)?; let key = decode_typed_key(&k_reader)?; let subkey = reader.get_subkey(); @@ -115,14 +118,14 @@ impl RPCOperationSetValueQ { pub(in crate::rpc_processor) struct RPCOperationSetValueA { set: bool, value: Option, - peers: Vec, + peers: Vec>, } impl RPCOperationSetValueA { pub fn new( set: bool, value: Option, - peers: Vec, + peers: Vec>, ) -> Result { if peers.len() > MAX_SET_VALUE_A_PEERS_LEN { return Err(RPCError::protocol( @@ -174,11 +177,14 @@ impl RPCOperationSetValueA { // pub fn peers(&self) -> &[PeerInfo] { // &self.peers // } - pub fn destructure(self) -> (bool, Option, Vec) { + pub fn destructure(self) -> (bool, Option, Vec>) { (self.set, self.value, self.peers) } - pub fn decode(reader: &veilid_capnp::operation_set_value_a::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_set_value_a::Reader, + ) -> Result { let set = reader.get_set(); let value = if reader.has_value() { let v_reader = reader.get_value().map_err(RPCError::protocol)?; @@ -193,14 +199,14 @@ impl RPCOperationSetValueA { "decoded SetValueA peers length too long", )); } - let mut peers = Vec::::with_capacity( + let mut peers = Vec::>::with_capacity( peers_reader .len() .try_into() .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p)?; + let peer_info = Arc::new(decode_peer_info(decode_context, &p)?); peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_signal.rs b/veilid-core/src/rpc_processor/coders/operations/operation_signal.rs index 36167cf5..79fc9b63 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_signal.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_signal.rs @@ -19,8 +19,11 @@ impl RPCOperationSignal { self.signal_info } - pub fn decode(reader: &veilid_capnp::operation_signal::Reader) -> Result { - let signal_info = decode_signal_info(reader)?; + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_signal::Reader, + ) -> Result { + let signal_info = decode_signal_info(decode_context, reader)?; Ok(Self { signal_info }) } pub fn encode( diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs b/veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs index 3b7d8ff4..f79dce52 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs @@ -35,6 +35,7 @@ impl RPCOperationStartTunnelQ { } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_start_tunnel_q::Reader, ) -> Result { let id = TunnelId::new(reader.get_id()); @@ -86,6 +87,7 @@ impl RPCOperationStartTunnelA { } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_start_tunnel_a::Reader, ) -> Result { match reader.which().map_err(RPCError::protocol)? { diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_status.rs b/veilid-core/src/rpc_processor/coders/operations/operation_status.rs index f77f5baf..5cf6638d 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_status.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_status.rs @@ -20,7 +20,10 @@ impl RPCOperationStatusQ { self.node_status } - pub fn decode(reader: &veilid_capnp::operation_status_q::Reader) -> Result { + pub fn decode( + _decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_status_q::Reader, + ) -> Result { let node_status = if reader.has_node_status() { let ns_reader = reader.get_node_status().map_err(RPCError::protocol)?; let node_status = decode_node_status(&ns_reader)?; @@ -72,7 +75,10 @@ impl RPCOperationStatusA { (self.node_status, self.sender_info) } - pub fn decode(reader: &veilid_capnp::operation_status_a::Reader) -> Result { + pub fn decode( + _decode_context: &RPCDecodeContext, + reader: &veilid_capnp::operation_status_a::Reader, + ) -> Result { let node_status = if reader.has_node_status() { let ns_reader = reader.get_node_status().map_err(RPCError::protocol)?; let node_status = decode_node_status(&ns_reader)?; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs b/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs index dbfa1a5c..3b60ca60 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs @@ -24,6 +24,7 @@ impl RPCOperationSupplyBlockQ { } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_supply_block_q::Reader, ) -> Result { let bi_reader = reader.get_block_id().map_err(RPCError::protocol)?; @@ -72,6 +73,7 @@ impl RPCOperationSupplyBlockA { } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_supply_block_a::Reader, ) -> Result { let expiration = reader.get_expiration(); @@ -87,7 +89,7 @@ impl RPCOperationSupplyBlockA { .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p)?; + let peer_info = decode_peer_info(decode_context, &p)?; peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_validate_dial_info.rs b/veilid-core/src/rpc_processor/coders/operations/operation_validate_dial_info.rs index 8a5848fe..47d05b16 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_validate_dial_info.rs @@ -44,6 +44,7 @@ impl RPCOperationValidateDialInfo { } pub fn decode( + _decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_validate_dial_info::Reader, ) -> Result { let di_reader = reader.get_dial_info().map_err(RPCError::protocol)?; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_value_changed.rs b/veilid-core/src/rpc_processor/coders/operations/operation_value_changed.rs index 34fe48ea..bd11b2e6 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_value_changed.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_value_changed.rs @@ -13,7 +13,6 @@ pub(in crate::rpc_processor) struct RPCOperationValueChanged { } impl RPCOperationValueChanged { - #[allow(dead_code)] pub fn new( key: TypedKey, subkeys: ValueSubkeyRangeSet, @@ -58,32 +57,31 @@ impl RPCOperationValueChanged { Ok(()) } - #[allow(dead_code)] + #[expect(dead_code)] pub fn key(&self) -> &TypedKey { &self.key } - #[allow(dead_code)] + #[expect(dead_code)] pub fn subkeys(&self) -> &ValueSubkeyRangeSet { &self.subkeys } - #[allow(dead_code)] + #[expect(dead_code)] pub fn count(&self) -> u32 { self.count } - #[allow(dead_code)] + #[expect(dead_code)] pub fn watch_id(&self) -> u64 { self.watch_id } - #[allow(dead_code)] + #[expect(dead_code)] pub fn value(&self) -> Option<&SignedValueData> { self.value.as_ref() } - #[allow(dead_code)] pub fn destructure( self, ) -> ( @@ -103,6 +101,7 @@ impl RPCOperationValueChanged { } pub fn decode( + _decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_value_changed::Reader, ) -> Result { let k_reader = reader.get_key().map_err(RPCError::protocol)?; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_watch_value.rs b/veilid-core/src/rpc_processor/coders/operations/operation_watch_value.rs index 261d8e36..aaddcf00 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_watch_value.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_watch_value.rs @@ -15,7 +15,6 @@ pub(in crate::rpc_processor) struct RPCOperationWatchValueQ { } impl RPCOperationWatchValueQ { - #[allow(dead_code)] pub fn new( key: TypedKey, subkeys: ValueSubkeyRangeSet, @@ -103,40 +102,39 @@ impl RPCOperationWatchValueQ { Ok(()) } - #[allow(dead_code)] + #[expect(dead_code)] pub fn key(&self) -> &TypedKey { &self.key } - #[allow(dead_code)] + #[expect(dead_code)] pub fn subkeys(&self) -> &ValueSubkeyRangeSet { &self.subkeys } - #[allow(dead_code)] + #[expect(dead_code)] pub fn expiration(&self) -> u64 { self.expiration } - #[allow(dead_code)] + #[expect(dead_code)] pub fn count(&self) -> u32 { self.count } - #[allow(dead_code)] + #[expect(dead_code)] pub fn watch_id(&self) -> Option { self.watch_id } - #[allow(dead_code)] + #[expect(dead_code)] pub fn watcher(&self) -> &PublicKey { &self.watcher } - #[allow(dead_code)] + #[expect(dead_code)] pub fn signature(&self) -> &Signature { &self.signature } - #[allow(dead_code)] pub fn destructure( self, ) -> ( @@ -160,6 +158,7 @@ impl RPCOperationWatchValueQ { } pub fn decode( + _decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_watch_value_q::Reader, ) -> Result { let k_reader = reader.get_key().map_err(RPCError::protocol)?; @@ -248,16 +247,15 @@ impl RPCOperationWatchValueQ { pub(in crate::rpc_processor) struct RPCOperationWatchValueA { accepted: bool, expiration: u64, - peers: Vec, + peers: Vec>, watch_id: u64, } impl RPCOperationWatchValueA { - #[allow(dead_code)] pub fn new( accepted: bool, expiration: u64, - peers: Vec, + peers: Vec>, watch_id: u64, ) -> Result { if peers.len() > MAX_WATCH_VALUE_A_PEERS_LEN { @@ -276,28 +274,28 @@ impl RPCOperationWatchValueA { Ok(()) } - #[allow(dead_code)] + #[expect(dead_code)] pub fn accepted(&self) -> bool { self.accepted } - #[allow(dead_code)] + #[expect(dead_code)] pub fn expiration(&self) -> u64 { self.expiration } - #[allow(dead_code)] - pub fn peers(&self) -> &[PeerInfo] { + #[expect(dead_code)] + pub fn peers(&self) -> &[Arc] { &self.peers } - #[allow(dead_code)] + #[expect(dead_code)] pub fn watch_id(&self) -> u64 { self.watch_id } - #[allow(dead_code)] - pub fn destructure(self) -> (bool, u64, Vec, u64) { + pub fn destructure(self) -> (bool, u64, Vec>, u64) { (self.accepted, self.expiration, self.peers, self.watch_id) } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_watch_value_a::Reader, ) -> Result { let accepted = reader.get_accepted(); @@ -306,14 +304,14 @@ impl RPCOperationWatchValueA { if peers_reader.len() as usize > MAX_WATCH_VALUE_A_PEERS_LEN { return Err(RPCError::protocol("WatchValueA peers length too long")); } - let mut peers = Vec::::with_capacity( + let mut peers = Vec::>::with_capacity( peers_reader .len() .try_into() .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p)?; + let peer_info = Arc::new(decode_peer_info(decode_context, &p)?); peers.push(peer_info); } let watch_id = reader.get_watch_id(); diff --git a/veilid-core/src/rpc_processor/coders/operations/question.rs b/veilid-core/src/rpc_processor/coders/operations/question.rs index bf36b35e..da9bd51c 100644 --- a/veilid-core/src/rpc_processor/coders/operations/question.rs +++ b/veilid-core/src/rpc_processor/coders/operations/question.rs @@ -20,18 +20,20 @@ impl RPCQuestion { pub fn detail(&self) -> &RPCQuestionDetail { &self.detail } - #[cfg(feature = "verbose-tracing")] pub fn desc(&self) -> &'static str { self.detail.desc() } pub fn destructure(self) -> (RespondTo, RPCQuestionDetail) { (self.respond_to, self.detail) } - pub fn decode(reader: &veilid_capnp::question::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::question::Reader, + ) -> Result { let rt_reader = reader.get_respond_to(); - let respond_to = RespondTo::decode(&rt_reader)?; + let respond_to = RespondTo::decode(decode_context, &rt_reader)?; let d_reader = reader.get_detail(); - let detail = RPCQuestionDetail::decode(&d_reader)?; + let detail = RPCQuestionDetail::decode(decode_context, &d_reader)?; Ok(RPCQuestion { respond_to, detail }) } pub fn encode(&self, builder: &mut veilid_capnp::question::Builder) -> Result<(), RPCError> { @@ -64,7 +66,6 @@ pub(in crate::rpc_processor) enum RPCQuestionDetail { } impl RPCQuestionDetail { - #[cfg(feature = "verbose-tracing")] pub fn desc(&self) -> &'static str { match self { RPCQuestionDetail::StatusQ(_) => "StatusQ", @@ -109,73 +110,74 @@ impl RPCQuestionDetail { } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::question::detail::Reader, ) -> Result { let which_reader = reader.which().map_err(RPCError::protocol)?; let out = match which_reader { veilid_capnp::question::detail::StatusQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationStatusQ::decode(&op_reader)?; + let out = RPCOperationStatusQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::StatusQ(Box::new(out)) } veilid_capnp::question::detail::FindNodeQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationFindNodeQ::decode(&op_reader)?; + let out = RPCOperationFindNodeQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::FindNodeQ(Box::new(out)) } veilid_capnp::question::detail::Which::AppCallQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationAppCallQ::decode(&op_reader)?; + let out = RPCOperationAppCallQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::AppCallQ(Box::new(out)) } veilid_capnp::question::detail::GetValueQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationGetValueQ::decode(&op_reader)?; + let out = RPCOperationGetValueQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::GetValueQ(Box::new(out)) } veilid_capnp::question::detail::SetValueQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationSetValueQ::decode(&op_reader)?; + let out = RPCOperationSetValueQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::SetValueQ(Box::new(out)) } veilid_capnp::question::detail::WatchValueQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationWatchValueQ::decode(&op_reader)?; + let out = RPCOperationWatchValueQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::WatchValueQ(Box::new(out)) } veilid_capnp::question::detail::InspectValueQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationInspectValueQ::decode(&op_reader)?; + let out = RPCOperationInspectValueQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::InspectValueQ(Box::new(out)) } #[cfg(feature = "unstable-blockstore")] veilid_capnp::question::detail::SupplyBlockQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationSupplyBlockQ::decode(&op_reader)?; + let out = RPCOperationSupplyBlockQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::SupplyBlockQ(Box::new(out)) } #[cfg(feature = "unstable-blockstore")] veilid_capnp::question::detail::FindBlockQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationFindBlockQ::decode(&op_reader)?; + let out = RPCOperationFindBlockQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::FindBlockQ(Box::new(out)) } #[cfg(feature = "unstable-tunnels")] veilid_capnp::question::detail::StartTunnelQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationStartTunnelQ::decode(&op_reader)?; + let out = RPCOperationStartTunnelQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::StartTunnelQ(Box::new(out)) } #[cfg(feature = "unstable-tunnels")] veilid_capnp::question::detail::CompleteTunnelQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationCompleteTunnelQ::decode(&op_reader)?; + let out = RPCOperationCompleteTunnelQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::CompleteTunnelQ(Box::new(out)) } #[cfg(feature = "unstable-tunnels")] veilid_capnp::question::detail::CancelTunnelQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationCancelTunnelQ::decode(&op_reader)?; + let out = RPCOperationCancelTunnelQ::decode(decode_context, &op_reader)?; RPCQuestionDetail::CancelTunnelQ(Box::new(out)) } }; diff --git a/veilid-core/src/rpc_processor/coders/operations/respond_to.rs b/veilid-core/src/rpc_processor/coders/operations/respond_to.rs index b9dec598..dbe13649 100644 --- a/veilid-core/src/rpc_processor/coders/operations/respond_to.rs +++ b/veilid-core/src/rpc_processor/coders/operations/respond_to.rs @@ -30,12 +30,15 @@ impl RespondTo { Ok(()) } - pub fn decode(reader: &veilid_capnp::question::respond_to::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::question::respond_to::Reader, + ) -> Result { let respond_to = match reader.which().map_err(RPCError::protocol)? { veilid_capnp::question::respond_to::Sender(()) => RespondTo::Sender, veilid_capnp::question::respond_to::PrivateRoute(pr_reader) => { let pr_reader = pr_reader.map_err(RPCError::protocol)?; - let pr = decode_private_route(&pr_reader)?; + let pr = decode_private_route(decode_context, &pr_reader)?; RespondTo::PrivateRoute(pr) } }; diff --git a/veilid-core/src/rpc_processor/coders/operations/statement.rs b/veilid-core/src/rpc_processor/coders/operations/statement.rs index 0395afa6..904c2c86 100644 --- a/veilid-core/src/rpc_processor/coders/operations/statement.rs +++ b/veilid-core/src/rpc_processor/coders/operations/statement.rs @@ -15,16 +15,18 @@ impl RPCStatement { pub fn detail(&self) -> &RPCStatementDetail { &self.detail } - #[cfg(feature = "verbose-tracing")] pub fn desc(&self) -> &'static str { self.detail.desc() } pub fn destructure(self) -> RPCStatementDetail { self.detail } - pub fn decode(reader: &veilid_capnp::statement::Reader) -> Result { + pub fn decode( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::statement::Reader, + ) -> Result { let d_reader = reader.get_detail(); - let detail = RPCStatementDetail::decode(&d_reader)?; + let detail = RPCStatementDetail::decode(decode_context, &d_reader)?; Ok(RPCStatement { detail }) } pub fn encode(&self, builder: &mut veilid_capnp::statement::Builder) -> Result<(), RPCError> { @@ -44,7 +46,6 @@ pub(in crate::rpc_processor) enum RPCStatementDetail { } impl RPCStatementDetail { - #[cfg(feature = "verbose-tracing")] pub fn desc(&self) -> &'static str { match self { RPCStatementDetail::ValidateDialInfo(_) => "ValidateDialInfo", @@ -66,38 +67,39 @@ impl RPCStatementDetail { } } pub fn decode( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::statement::detail::Reader, ) -> Result { let which_reader = reader.which().map_err(RPCError::protocol)?; let out = match which_reader { veilid_capnp::statement::detail::ValidateDialInfo(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationValidateDialInfo::decode(&op_reader)?; + let out = RPCOperationValidateDialInfo::decode(decode_context, &op_reader)?; RPCStatementDetail::ValidateDialInfo(Box::new(out)) } veilid_capnp::statement::detail::Route(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationRoute::decode(&op_reader)?; + let out = RPCOperationRoute::decode(decode_context, &op_reader)?; RPCStatementDetail::Route(Box::new(out)) } veilid_capnp::statement::detail::ValueChanged(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationValueChanged::decode(&op_reader)?; + let out = RPCOperationValueChanged::decode(decode_context, &op_reader)?; RPCStatementDetail::ValueChanged(Box::new(out)) } veilid_capnp::statement::detail::Signal(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationSignal::decode(&op_reader)?; + let out = RPCOperationSignal::decode(decode_context, &op_reader)?; RPCStatementDetail::Signal(Box::new(out)) } veilid_capnp::statement::detail::ReturnReceipt(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationReturnReceipt::decode(&op_reader)?; + let out = RPCOperationReturnReceipt::decode(decode_context, &op_reader)?; RPCStatementDetail::ReturnReceipt(Box::new(out)) } veilid_capnp::statement::detail::AppMessage(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationAppMessage::decode(&op_reader)?; + let out = RPCOperationAppMessage::decode(decode_context, &op_reader)?; RPCStatementDetail::AppMessage(Box::new(out)) } }; diff --git a/veilid-core/src/rpc_processor/coders/peer_info.rs b/veilid-core/src/rpc_processor/coders/peer_info.rs index b8707994..68daac18 100644 --- a/veilid-core/src/rpc_processor/coders/peer_info.rs +++ b/veilid-core/src/rpc_processor/coders/peer_info.rs @@ -27,7 +27,10 @@ pub fn encode_peer_info( Ok(()) } -pub fn decode_peer_info(reader: &veilid_capnp::peer_info::Reader) -> Result { +pub fn decode_peer_info( + decode_context: &RPCDecodeContext, + reader: &veilid_capnp::peer_info::Reader, +) -> Result { let nids_reader = reader .reborrow() .get_node_ids() @@ -44,5 +47,9 @@ pub fn decode_peer_info(reader: &veilid_capnp::peer_info::Reader) -> Result Result { let n_reader = reader.reborrow().get_node(); @@ -78,8 +79,8 @@ pub(crate) fn decode_route_hop( } veilid_capnp::route_hop::node::Which::PeerInfo(pi) => { let pi_reader = pi.map_err(RPCError::protocol)?; - RouteNode::PeerInfo(Box::new( - decode_peer_info(&pi_reader) + RouteNode::PeerInfo(Arc::new( + decode_peer_info(decode_context, &pi_reader) .map_err(RPCError::map_protocol("invalid peer info in route hop"))?, )) } @@ -126,6 +127,7 @@ pub(crate) fn encode_private_route( } pub(crate) fn decode_private_route( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::private_route::Reader, ) -> Result { let public_key = decode_typed_key(&reader.get_public_key().map_err( @@ -136,7 +138,7 @@ pub(crate) fn decode_private_route( let hops = match reader.get_hops().which().map_err(RPCError::protocol)? { veilid_capnp::private_route::hops::Which::FirstHop(rh_reader) => { let rh_reader = rh_reader.map_err(RPCError::protocol)?; - PrivateRouteHops::FirstHop(Box::new(decode_route_hop(&rh_reader)?)) + PrivateRouteHops::FirstHop(Box::new(decode_route_hop(decode_context, &rh_reader)?)) } veilid_capnp::private_route::hops::Which::Data(rhd_reader) => { let rhd_reader = rhd_reader.map_err(RPCError::protocol)?; @@ -179,6 +181,7 @@ pub(crate) fn encode_safety_route( } pub(crate) fn decode_safety_route( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::safety_route::Reader, ) -> Result { let public_key = decode_typed_key( @@ -194,7 +197,7 @@ pub(crate) fn decode_safety_route( } veilid_capnp::safety_route::hops::Which::Private(pr_reader) => { let pr_reader = pr_reader.map_err(RPCError::protocol)?; - SafetyRouteHops::Private(decode_private_route(&pr_reader)?) + SafetyRouteHops::Private(decode_private_route(decode_context, &pr_reader)?) } }; diff --git a/veilid-core/src/rpc_processor/coders/signal_info.rs b/veilid-core/src/rpc_processor/coders/signal_info.rs index 5e9edc84..2965ec49 100644 --- a/veilid-core/src/rpc_processor/coders/signal_info.rs +++ b/veilid-core/src/rpc_processor/coders/signal_info.rs @@ -33,6 +33,7 @@ pub fn encode_signal_info( } pub fn decode_signal_info( + decode_context: &RPCDecodeContext, reader: &veilid_capnp::operation_signal::Reader, ) -> Result { Ok( @@ -52,7 +53,7 @@ pub fn decode_signal_info( let pi_reader = r.get_peer_info().map_err(RPCError::map_protocol( "invalid peer info in hole punch signal info", ))?; - let peer_info = decode_peer_info(&pi_reader)?; + let peer_info = Arc::new(decode_peer_info(decode_context, &pi_reader)?); SignalInfo::HolePunch { receipt, peer_info } } @@ -68,7 +69,7 @@ pub fn decode_signal_info( let pi_reader = r.get_peer_info().map_err(RPCError::map_protocol( "invalid peer info in reverse connect signal info", ))?; - let peer_info = decode_peer_info(&pi_reader)?; + let peer_info = Arc::new(decode_peer_info(decode_context, &pi_reader)?); SignalInfo::ReverseConnect { receipt, peer_info } } diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index c6c18baa..3ce53fa6 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -6,14 +6,14 @@ pub(crate) enum Destination { /// Send to node directly Direct { /// The node to send to - node: NodeRef, + node: FilteredNodeRef, /// Require safety route or not safety_selection: SafetySelection, }, /// Send to node for relay purposes Relay { /// The relay to send to - relay: NodeRef, + relay: FilteredNodeRef, /// The final destination the relay should send to node: NodeRef, /// Require safety route or not @@ -37,31 +37,14 @@ pub struct UnsafeRoutingInfo { } impl Destination { - pub fn node(&self) -> Option { - match self { - Destination::Direct { - node: target, - safety_selection: _, - } => Some(target.clone()), - Destination::Relay { - relay: _, - node: target, - safety_selection: _, - } => Some(target.clone()), - Destination::PrivateRoute { - private_route: _, - safety_selection: _, - } => None, - } - } - pub fn direct(node: NodeRef) -> Self { + pub fn direct(node: FilteredNodeRef) -> Self { let sequencing = node.sequencing(); Self::Direct { node, safety_selection: SafetySelection::Unsafe(sequencing), } } - pub fn relay(relay: NodeRef, node: NodeRef) -> Self { + pub fn relay(relay: FilteredNodeRef, node: NodeRef) -> Self { let sequencing = relay.sequencing().max(node.sequencing()); Self::Relay { relay, @@ -122,13 +105,31 @@ impl Destination { } } + pub fn get_target_node_ids(&self) -> Option { + match self { + Destination::Direct { + node, + safety_selection: _, + } => Some(node.node_ids()), + Destination::Relay { + relay: _, + node, + safety_selection: _, + } => Some(node.node_ids()), + Destination::PrivateRoute { + private_route: _, + safety_selection: _, + } => None, + } + } + pub fn get_target(&self, rss: RouteSpecStore) -> Result { match self { Destination::Direct { node, safety_selection: _, - } - | Destination::Relay { + } => Ok(Target::NodeId(node.best_node_id())), + Destination::Relay { relay: _, node, safety_selection: _, @@ -173,7 +174,7 @@ impl Destination { // Only a stale connection or no connection exists log_rpc!(debug "No routing domain for node: node={}", node); }; - (Some(node.clone()), None, opt_routing_domain) + (Some(node.unfiltered()), None, opt_routing_domain) } Destination::Relay { relay, @@ -207,7 +208,11 @@ impl Destination { log_rpc!(debug "Unexpected relay used for node: relay={}, node={}", relay, node); }; - (Some(node.clone()), Some(relay.clone()), opt_routing_domain) + ( + Some(node.clone()), + Some(relay.unfiltered()), + opt_routing_domain, + ) } Destination::PrivateRoute { private_route: _, @@ -277,14 +282,14 @@ impl RPCProcessor { match target { Target::NodeId(node_id) => { // Resolve node - let mut nr = match self.resolve_node(node_id, safety_selection).await? { + let nr = match self.resolve_node(node_id, safety_selection).await? { Some(nr) => nr, None => { return Err(RPCError::network("could not resolve node id")); } }; // Apply sequencing to match safety selection - nr.set_sequencing(safety_selection.get_sequencing()); + let nr = nr.sequencing_filtered(safety_selection.get_sequencing()); Ok(rpc_processor::Destination::Direct { node: nr, @@ -385,21 +390,23 @@ impl RPCProcessor { match safety_selection { SafetySelection::Unsafe(_) => { // Sent to a private route with no safety route, use a stub safety route for the response - if !routing_table.has_valid_network_class(RoutingDomain::PublicInternet) { + + let Some(published_peer_info) = + routing_table.get_published_peer_info(RoutingDomain::PublicInternet) + else { return Ok(NetworkResult::service_unavailable( - "Own node info must be valid to use private route", + "Own node info must be published to use private route", )); - } + }; // Determine if we can use optimized nodeinfo let route_node = if rss.has_remote_private_route_seen_our_node_info( &private_route.public_key.value, + &published_peer_info, ) { RouteNode::NodeId(routing_table.node_id(crypto_kind).value) } else { - let own_peer_info = - routing_table.get_own_peer_info(RoutingDomain::PublicInternet); - RouteNode::PeerInfo(Box::new(own_peer_info)) + RouteNode::PeerInfo(published_peer_info) }; Ok(NetworkResult::value(RespondTo::PrivateRoute( diff --git a/veilid-core/src/rpc_processor/fanout_call.rs b/veilid-core/src/rpc_processor/fanout_call.rs index 16f212cf..cac06d32 100644 --- a/veilid-core/src/rpc_processor/fanout_call.rs +++ b/veilid-core/src/rpc_processor/fanout_call.rs @@ -58,7 +58,12 @@ pub(crate) fn debug_fanout_results(results: &[FanoutResult]) -> String { out } -pub(crate) type FanoutCallReturnType = RPCNetworkResult>; +#[derive(Debug)] +pub(crate) struct FanoutCallOutput { + pub peer_info_list: Vec>, +} + +pub(crate) type FanoutCallResult = RPCNetworkResult; pub(crate) type FanoutNodeInfoFilter = Arc bool + Send + Sync>; pub(crate) fn empty_fanout_node_info_filter() -> FanoutNodeInfoFilter { @@ -89,12 +94,11 @@ pub(crate) fn capability_fanout_node_info_filter(caps: Vec) -> Fanou pub(crate) struct FanoutCall where R: Unpin, - F: Future, + F: Future, C: Fn(NodeRef) -> F, D: Fn(&[NodeRef]) -> Option, { routing_table: RoutingTable, - crypto_kind: CryptoKind, node_id: TypedKey, context: Mutex>, node_count: usize, @@ -108,7 +112,7 @@ where impl FanoutCall where R: Unpin, - F: Future, + F: Future, C: Fn(NodeRef) -> F, D: Fn(&[NodeRef]) -> Option, { @@ -131,7 +135,6 @@ where Arc::new(Self { routing_table, node_id, - crypto_kind: node_id.kind, context, node_count, fanout, @@ -198,7 +201,8 @@ where match (self.call_routine)(next_node.clone()).await { Ok(NetworkResult::Value(v)) => { // Filter returned nodes - let filtered_v: Vec = v + let filtered_v: Vec> = v + .peer_info_list .into_iter() .filter(|pi| { let node_ids = pi.node_ids().to_vec(); @@ -216,7 +220,7 @@ where // Register the returned nodes and add them to the fanout queue in sorted order let new_nodes = self .routing_table - .register_find_node_answer(self.crypto_kind, filtered_v); + .register_nodes_with_peer_info_list(filtered_v); self.clone().add_to_fanout_queue(&new_nodes); } #[allow(unused_variables)] @@ -273,7 +277,7 @@ where let filters = VecDeque::from([filter]); let transform = |_rti: &RoutingTableInner, v: Option>| { - NodeRef::new(routing_table.clone(), v.unwrap().clone(), None) + NodeRef::new(routing_table.clone(), v.unwrap().clone()) }; routing_table diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 70dafc8e..55ada4e0 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -55,8 +55,8 @@ struct RPCMessageHeaderDetailDirect { envelope: Envelope, /// The noderef of the peer that sent the message (not the original sender). /// Ensures node doesn't get evicted from routing table until we're done with it - /// Should be filted to the routing domain of the peer that we received from - peer_noderef: NodeRef, + /// Should be filtered to the routing domain of the peer that we received from + peer_noderef: FilteredNodeRef, /// The flow from the peer sent the message (not the original sender) flow: Flow, /// The routing domain of the peer that we received from @@ -226,7 +226,7 @@ struct RenderedOperation { /// Destination node we're sending to destination_node_ref: NodeRef, /// Node to send envelope to (may not be destination node in case of relay) - node_ref: NodeRef, + node_ref: FilteredNodeRef, /// Total safety + private route hop count + 1 hop for the initial send hop_count: usize, /// The safety route used to send the message @@ -255,20 +255,20 @@ impl fmt::Debug for RenderedOperation { #[derive(Default, Debug, Clone)] pub struct SenderPeerInfo { /// The current peer info of the sender if required - opt_sender_peer_info: Option, + opt_peer_info: Option>, /// The last timestamp of the target's node info to assist remote node with sending its latest node info target_node_info_ts: Timestamp, } impl SenderPeerInfo { pub fn new_no_peer_info(target_node_info_ts: Timestamp) -> Self { Self { - opt_sender_peer_info: None, + opt_peer_info: None, target_node_info_ts, } } - pub fn new(sender_peer_info: PeerInfo, target_node_info_ts: Timestamp) -> Self { + pub fn new(peer_info: Arc, target_node_info_ts: Timestamp) -> Self { Self { - opt_sender_peer_info: Some(sender_peer_info), + opt_peer_info: Some(peer_info), target_node_info_ts, } } @@ -294,7 +294,7 @@ struct RPCProcessorUnlockedInner { queue_size: u32, concurrency: u32, max_route_hop_count: usize, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] validate_dial_info_receipt_time_ms: u32, update_callback: UpdateCallback, waiting_rpc_table: OperationWaiter>, @@ -458,7 +458,7 @@ impl RPCProcessor { ////////////////////////////////////////////////////////////////////// /// Get waiting app call id for debugging purposes - pub fn get_app_call_ids(&self) -> Vec { + pub(crate) fn get_app_call_ids(&self) -> Vec { self.unlocked_inner .waiting_app_call_table .get_operation_ids() @@ -479,12 +479,65 @@ impl RPCProcessor { .has_all_capabilities(capabilities) } + /// Incorporate 'sender peer info' sent along with an RPC message + #[instrument(level = "trace", target = "rpc", skip_all)] + fn process_sender_peer_info( + &self, + routing_domain: RoutingDomain, + sender_node_id: TypedKey, + sender_peer_info: &SenderPeerInfo, + ) -> RPCNetworkResult> { + let Some(peer_info) = sender_peer_info.opt_peer_info.clone() else { + return Ok(NetworkResult::value(None)); + }; + let address_filter = self.network_manager.address_filter(); + + // Ensure the sender peer info is for the actual sender specified in the envelope + if !peer_info.node_ids().contains(&sender_node_id) { + // Attempted to update peer info for the wrong node id + address_filter.punish_node_id(sender_node_id, PunishmentReason::WrongSenderPeerInfo); + + return Ok(NetworkResult::invalid_message( + "attempt to update peer info for non-sender node id", + )); + } + + // Sender PeerInfo was specified, update our routing table with it + if !self.verify_node_info(routing_domain, peer_info.signed_node_info(), &[]) { + log_network_result!(debug "Dropping invalid PeerInfo in {:?} for id {}: {:?}", routing_domain, sender_node_id, peer_info); + // Don't punish for this because in the case of hairpin NAT + // you can legally get LocalNetwork PeerInfo when you expect PublicInternet PeerInfo + // + // address_filter.punish_node_id( + // sender_node_id, + // PunishmentReason::FailedToVerifySenderPeerInfo, + // ); + return Ok(NetworkResult::value(None)); + } + let sender_nr = match self + .routing_table() + .register_node_with_peer_info(peer_info.clone(), false) + { + Ok(v) => v.unfiltered(), + Err(e) => { + address_filter.punish_node_id( + sender_node_id, + PunishmentReason::FailedToRegisterSenderPeerInfo, + ); + return Ok(NetworkResult::invalid_message(e)); + } + }; + + Ok(NetworkResult::value(Some(sender_nr))) + } + ////////////////////////////////////////////////////////////////////// - /// Search the network for a single node and add it to the routing table and return the node reference + /// Search the public internet routing domain for a single node and add + /// it to the routing table and return the node reference /// If no node was found in the timeout, this returns None #[instrument(level = "trace", target = "rpc", skip_all)] - async fn search_for_node_id( + async fn public_internet_peer_search( &self, node_id: TypedKey, count: usize, @@ -493,6 +546,7 @@ impl RPCProcessor { safety_selection: SafetySelection, ) -> TimeoutOr, RPCError>> { let routing_table = self.routing_table(); + let routing_domain = RoutingDomain::PublicInternet; // Ignore own node if routing_table.matches_own_node_id(&[node_id]) { @@ -506,13 +560,16 @@ impl RPCProcessor { let v = network_result_try!( this.clone() .rpc_call_find_node( - Destination::direct(next_node).with_safety(safety_selection), + Destination::direct(next_node.routing_domain_filtered(routing_domain)) + .with_safety(safety_selection), node_id, vec![], ) .await? ); - Ok(NetworkResult::value(v.answer)) + Ok(NetworkResult::value(FanoutCallOutput { + peer_info_list: v.answer, + })) } }; @@ -547,7 +604,8 @@ impl RPCProcessor { fanout_call.run(vec![]).await } - /// Search the DHT for a specific node corresponding to a key unless we have that node in our routing table already, and return the node reference + /// Search the DHT for a specific node corresponding to a key unless we + /// have that node in our routing table already, and return the node reference /// Note: This routine can possibly be recursive, hence the SendPinBoxFuture async form #[instrument(level = "trace", target = "rpc", skip_all)] pub fn resolve_node( @@ -567,10 +625,13 @@ impl RPCProcessor { let routing_table = this.routing_table(); // First see if we have the node in our routing table already + let mut existing_nr = None; if let Some(nr) = routing_table .lookup_node_ref(node_id) .map_err(RPCError::internal)? { + existing_nr = Some(nr.clone()); + // ensure we have some dial info for the entry already, // and that the node is still alive // if not, we should do the find_node anyway @@ -590,9 +651,16 @@ impl RPCProcessor { ) }; - // Search in preferred cryptosystem order + // Search routing domains for peer + // xxx: Eventually add other routing domains here let nr = match this - .search_for_node_id(node_id, node_count, fanout, timeout, safety_selection) + .public_internet_peer_search( + node_id, + node_count, + fanout, + timeout, + safety_selection, + ) .await { TimeoutOr::Timeout => None, @@ -602,7 +670,8 @@ impl RPCProcessor { } }; - Ok(nr) + // Either return the node we just resolved or a dead one we found in the routing table to try again + Ok(nr.or(existing_nr)) } .in_current_span(), ) @@ -692,6 +761,7 @@ impl RPCProcessor { #[instrument(level = "trace", target = "rpc", skip_all)] fn wrap_with_route( &self, + routing_domain: RoutingDomain, safety_selection: SafetySelection, remote_private_route: PrivateRoute, reply_private_route: Option, @@ -729,8 +799,12 @@ impl RPCProcessor { .map_err(RPCError::map_internal("encryption failed"))?; // Make the routed operation - let operation = - RoutedOperation::new(safety_selection.get_sequencing(), nonce, enc_msg_data); + let operation = RoutedOperation::new( + routing_domain, + safety_selection.get_sequencing(), + nonce, + enc_msg_data, + ); // Prepare route operation let sr_hop_count = compiled_route.safety_route.hop_count; @@ -753,7 +827,7 @@ impl RPCProcessor { let out = RenderedOperation { message: out_message, - destination_node_ref: compiled_route.first_hop.clone(), + destination_node_ref: compiled_route.first_hop.unfiltered(), node_ref: compiled_route.first_hop, hop_count: out_hop_count, safety_route: if sr_is_stub { None } else { Some(sr_pubkey) }, @@ -816,7 +890,7 @@ impl RPCProcessor { { (node_ref.clone(), target.clone()) } else { - (node_ref.clone(), node_ref.clone()) + (node_ref.clone(), node_ref.unfiltered()) }; // Handle the existence of safety route @@ -827,10 +901,6 @@ impl RPCProcessor { if sequencing > node_ref.sequencing() { node_ref.set_sequencing(sequencing) } - let mut destination_node_ref = destination_node_ref.clone(); - if sequencing > destination_node_ref.sequencing() { - destination_node_ref.set_sequencing(sequencing) - } // Reply private route should be None here, even for questions assert!(reply_private_route.is_none()); @@ -848,26 +918,28 @@ impl RPCProcessor { }); } SafetySelection::Safe(_) => { + // For now we only private-route over PublicInternet + let routing_domain = RoutingDomain::PublicInternet; + // No private route was specified for the request // but we are using a safety route, so we must create an empty private route // Destination relay is ignored for safety routed operations - let peer_info = match destination_node_ref - .make_peer_info(RoutingDomain::PublicInternet) - { + let peer_info = match destination_node_ref.make_peer_info(routing_domain) { None => { return Ok(NetworkResult::no_connection_other( - "No PublicInternet peer info for stub private route", + "No peer info for stub private route", )) } Some(pi) => pi, }; let private_route = PrivateRoute::new_stub( destination_node_ref.best_node_id(), - RouteNode::PeerInfo(Box::new(peer_info)), + RouteNode::PeerInfo(Arc::new(peer_info)), ); // Wrap with safety route out = self.wrap_with_route( + routing_domain, safety_selection, private_route, reply_private_route, @@ -880,10 +952,14 @@ impl RPCProcessor { private_route, safety_selection, } => { + // For now we only private-route over PublicInternet + let routing_domain = RoutingDomain::PublicInternet; + // Send to private route // --------------------- // Reply with 'route' operation out = self.wrap_with_route( + routing_domain, safety_selection, private_route, reply_private_route, @@ -930,17 +1006,17 @@ impl RPCProcessor { // Return whatever peer info we have even if the network class is not yet valid // That away we overwrite any prior existing valid-network-class nodeinfo in the remote routing table let routing_table = self.routing_table(); - let own_peer_info = routing_table.get_own_peer_info(routing_domain); - // Get our node info timestamp - let our_node_info_ts = own_peer_info.signed_node_info().timestamp(); + if let Some(published_peer_info) = routing_table.get_published_peer_info(routing_domain) { + // Get our node info timestamp + let our_node_info_ts = published_peer_info.signed_node_info().timestamp(); - // If the target has seen our node info already don't send it again - if node.has_seen_our_node_info_ts(routing_domain, our_node_info_ts) { - return SenderPeerInfo::new_no_peer_info(target_node_info_ts); + // If the target has not yet seen our published peer info, send it along if we have it + if !node.has_seen_our_node_info_ts(routing_domain, our_node_info_ts) { + return SenderPeerInfo::new(published_peer_info, target_node_info_ts); + } } - - SenderPeerInfo::new(own_peer_info, target_node_info_ts) + SenderPeerInfo::new_no_peer_info(target_node_info_ts) } /// Record failure to send to node or route @@ -960,7 +1036,7 @@ impl RPCProcessor { node_ref.stats_failed_to_send(send_ts, wants_answer); // Also clear the last_connections for the entry so we make a new connection next time - node_ref.clear_last_connections(); + node_ref.clear_last_flows(); return; } @@ -993,7 +1069,7 @@ impl RPCProcessor { node_ref.stats_question_lost(); // Also clear the last_connections for the entry so we make a new connection next time - node_ref.clear_last_connections(); + node_ref.clear_last_flows(); return; } @@ -1212,7 +1288,6 @@ impl RPCProcessor { let op_id = operation.op_id(); // Log rpc send - #[cfg(feature = "verbose-tracing")] debug!(target: "rpc_message", dir = "send", kind = "question", op_id = op_id.as_u64(), desc = operation.kind().desc(), ?dest); // Produce rendered operation @@ -1254,7 +1329,7 @@ impl RPCProcessor { self.record_send_failure( RPCKind::Question, send_ts, - node_ref.clone(), + node_ref.unfiltered(), safety_route, remote_private_route, ); @@ -1262,7 +1337,7 @@ impl RPCProcessor { })?; let send_data_method = network_result_value_or_log!( res => [ format!(": node_ref={}, destination_node_ref={}, message.len={}", node_ref, destination_node_ref, message_len) ] { // If we couldn't send we're still cleaning up - self.record_send_failure(RPCKind::Question, send_ts, node_ref.clone(), safety_route, remote_private_route); + self.record_send_failure(RPCKind::Question, send_ts, node_ref.unfiltered(), safety_route, remote_private_route); network_result_raise!(res); } ); @@ -1272,7 +1347,7 @@ impl RPCProcessor { RPCKind::Question, send_ts, bytes, - node_ref.clone(), + node_ref.unfiltered(), safety_route, remote_private_route, ); @@ -1288,7 +1363,7 @@ impl RPCProcessor { Ok(NetworkResult::value(WaitableReply { handle, timeout_us, - node_ref, + node_ref: node_ref.unfiltered(), send_ts, send_data_method, safety_route, @@ -1308,7 +1383,6 @@ impl RPCProcessor { let operation = RPCOperation::new_statement(statement, spi); // Log rpc send - #[cfg(feature = "verbose-tracing")] debug!(target: "rpc_message", dir = "send", kind = "statement", op_id = operation.op_id().as_u64(), desc = operation.kind().desc(), ?dest); // Produce rendered operation @@ -1340,7 +1414,7 @@ impl RPCProcessor { self.record_send_failure( RPCKind::Statement, send_ts, - node_ref.clone(), + node_ref.unfiltered(), safety_route, remote_private_route, ); @@ -1348,7 +1422,7 @@ impl RPCProcessor { })?; let _send_data_method = network_result_value_or_log!( res => [ format!(": node_ref={}, destination_node_ref={}, message.len={}", node_ref, destination_node_ref, message_len) ] { // If we couldn't send we're still cleaning up - self.record_send_failure(RPCKind::Statement, send_ts, node_ref.clone(), safety_route, remote_private_route); + self.record_send_failure(RPCKind::Statement, send_ts, node_ref.unfiltered(), safety_route, remote_private_route); network_result_raise!(res); } ); @@ -1358,7 +1432,7 @@ impl RPCProcessor { RPCKind::Statement, send_ts, bytes, - node_ref, + node_ref.unfiltered(), safety_route, remote_private_route, ); @@ -1379,7 +1453,6 @@ impl RPCProcessor { let operation = RPCOperation::new_answer(&request.operation, answer, spi); // Log rpc send - #[cfg(feature = "verbose-tracing")] debug!(target: "rpc_message", dir = "send", kind = "answer", op_id = operation.op_id().as_u64(), desc = operation.kind().desc(), ?dest); // Produce rendered operation @@ -1411,7 +1484,7 @@ impl RPCProcessor { self.record_send_failure( RPCKind::Answer, send_ts, - node_ref.clone(), + node_ref.unfiltered(), safety_route, remote_private_route, ); @@ -1419,7 +1492,7 @@ impl RPCProcessor { })?; let _send_data_kind = network_result_value_or_log!( res => [ format!(": node_ref={}, destination_node_ref={}, message.len={}", node_ref, destination_node_ref, message_len) ] { // If we couldn't send we're still cleaning up - self.record_send_failure(RPCKind::Answer, send_ts, node_ref.clone(), safety_route, remote_private_route); + self.record_send_failure(RPCKind::Answer, send_ts, node_ref.unfiltered(), safety_route, remote_private_route); network_result_raise!(res); } ); @@ -1429,7 +1502,7 @@ impl RPCProcessor { RPCKind::Answer, send_ts, bytes, - node_ref, + node_ref.unfiltered(), safety_route, remote_private_route, ); @@ -1449,7 +1522,10 @@ impl RPCProcessor { let op_reader = reader .get_root::() .map_err(RPCError::protocol)?; - let mut operation = RPCOperation::decode(&op_reader)?; + let decode_context = RPCDecodeContext { + routing_domain: encoded_msg.header.routing_domain(), + }; + let mut operation = RPCOperation::decode(&decode_context, &op_reader)?; // Validate the RPC message self.validate_rpc_operation(&mut operation)?; @@ -1536,49 +1612,13 @@ impl RPCProcessor { let routing_domain = detail.routing_domain; // Get the sender noderef, incorporating sender's peer info - let mut opt_sender_nr: Option = None; - if let Some(sender_peer_info) = operation.sender_peer_info() { - // Ensure the sender peer info is for the actual sender specified in the envelope - if !sender_peer_info.node_ids().contains(&sender_node_id) { - // Attempted to update peer info for the wrong node id - address_filter - .punish_node_id(sender_node_id, PunishmentReason::WrongSenderPeerInfo); - return Ok(NetworkResult::invalid_message( - "attempt to update peer info for non-sender node id", - )); - } - - // Sender PeerInfo was specified, update our routing table with it - if !self.verify_node_info( - routing_domain, - sender_peer_info.signed_node_info(), - &[], - ) { - address_filter.punish_node_id( - sender_node_id, - PunishmentReason::FailedToVerifySenderPeerInfo, - ); - return Ok(NetworkResult::invalid_message(format!( - "sender peerinfo has invalid peer scope: {:?}", - sender_peer_info.signed_node_info() - ))); - } - opt_sender_nr = match self.routing_table().register_node_with_peer_info( - routing_domain, - sender_peer_info.clone(), - false, - ) { - Ok(v) => Some(v), - Err(e) => { - address_filter.punish_node_id( - sender_node_id, - PunishmentReason::FailedToRegisterSenderPeerInfo, - ); - return Ok(NetworkResult::invalid_message(e)); - } - } - } - + let sender_peer_info = operation.sender_peer_info(); + let mut opt_sender_nr: Option = network_result_try!(self + .process_sender_peer_info(routing_domain, sender_node_id, &sender_peer_info)? => { + log_network_result!(debug "Sender PeerInfo: {:?}", sender_peer_info); + log_network_result!(debug "From Operation: {:?}", operation.kind()); + log_network_result!(debug "With Detail: {:?}", detail); + }); // look up sender node, in case it's different than our peer due to relaying if opt_sender_nr.is_none() { opt_sender_nr = match self.routing_table().lookup_node_ref(sender_node_id) { @@ -1586,6 +1626,7 @@ impl RPCProcessor { Err(e) => { // If this fails it's not the other node's fault. We should be able to look up a // node ref for a registered sender node id that just sent a message to us + // because it is registered with an existing connection at the very least. return Ok(NetworkResult::no_connection_other(e)); } } @@ -1594,8 +1635,10 @@ impl RPCProcessor { // Update the 'seen our node info' timestamp to determine if this node needs a // 'node info update' ping if let Some(sender_nr) = &opt_sender_nr { - sender_nr - .set_seen_our_node_info_ts(routing_domain, operation.target_node_info_ts()); + sender_nr.set_seen_our_node_info_ts( + routing_domain, + sender_peer_info.target_node_info_ts, + ); } // Make the RPC message @@ -1638,8 +1681,7 @@ impl RPCProcessor { } // Log rpc receive - #[cfg(feature = "verbose-tracing")] - debug!(target: "rpc_message", dir = "recv", kind = "question", op_id = msg.operation.op_id().as_u64(), desc = msg.operation.kind().desc(), header = ?msg.header); + debug!(target: "rpc_message", dir = "recv", kind = "question", op_id = msg.operation.op_id().as_u64(), desc = msg.operation.kind().desc(), header = ?msg.header, operation = ?msg.operation.kind()); } RPCOperationKind::Statement(_) => { if let Some(sender_nr) = msg.opt_sender_nr.clone() { @@ -1647,15 +1689,13 @@ impl RPCProcessor { } // Log rpc receive - #[cfg(feature = "verbose-tracing")] - debug!(target: "rpc_message", dir = "recv", kind = "statement", op_id = msg.operation.op_id().as_u64(), desc = msg.operation.kind().desc(), header = ?msg.header); + debug!(target: "rpc_message", dir = "recv", kind = "statement", op_id = msg.operation.op_id().as_u64(), desc = msg.operation.kind().desc(), header = ?msg.header, operation = ?msg.operation.kind()); } RPCOperationKind::Answer(_) => { // Answer stats are processed in wait_for_reply // Log rpc receive - #[cfg(feature = "verbose-tracing")] - debug!(target: "rpc_message", dir = "recv", kind = "answer", op_id = msg.operation.op_id().as_u64(), desc = msg.operation.kind().desc(), header = ?msg.header); + debug!(target: "rpc_message", dir = "recv", kind = "answer", op_id = msg.operation.op_id().as_u64(), desc = msg.operation.kind().desc(), header = ?msg.header, operation = ?msg.operation.kind()); } }; @@ -1749,7 +1789,7 @@ impl RPCProcessor { pub fn enqueue_direct_message( &self, envelope: Envelope, - peer_noderef: NodeRef, + peer_noderef: FilteredNodeRef, flow: Flow, routing_domain: RoutingDomain, body: Vec, @@ -1760,6 +1800,10 @@ impl RPCProcessor { .enter() .map_err(RPCError::map_try_again("not started up"))?; + if peer_noderef.routing_domain_set() != routing_domain { + bail!("routing domain should match peer noderef filter"); + } + let header = RPCMessageHeader { detail: RPCMessageHeaderDetail::Direct(RPCMessageHeaderDetailDirect { envelope, diff --git a/veilid-core/src/rpc_processor/rpc_app_call.rs b/veilid-core/src/rpc_processor/rpc_app_call.rs index e5e4eaf6..c5d7b1fa 100644 --- a/veilid-core/src/rpc_processor/rpc_app_call.rs +++ b/veilid-core/src/rpc_processor/rpc_app_call.rs @@ -36,7 +36,7 @@ impl RPCProcessor { }; // Get the right answer type - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let app_call_a = match kind { RPCOperationKind::Answer(a) => match a.destructure() { RPCAnswerDetail::AppCallA(a) => a, @@ -65,12 +65,15 @@ impl RPCProcessor { // Ignore if disabled let routing_table = self.routing_table(); - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - if !opi - .signed_node_info() - .node_info() - .has_capability(CAP_APPMESSAGE) - { + let has_capability_app_message = routing_table + .get_published_peer_info(msg.header.routing_domain()) + .map(|ppi| { + ppi.signed_node_info() + .node_info() + .has_capability(CAP_APPMESSAGE) + }) + .unwrap_or(false); + if !has_capability_app_message { return Ok(NetworkResult::service_unavailable( "app call is not available", )); @@ -95,7 +98,7 @@ impl RPCProcessor { }; // Get the question - let (op_id, _, _, kind) = msg.operation.clone().destructure(); + let (op_id, _, kind) = msg.operation.clone().destructure(); let app_call_q = match kind { RPCOperationKind::Question(q) => match q.destructure() { (_, RPCQuestionDetail::AppCallQ(q)) => q, diff --git a/veilid-core/src/rpc_processor/rpc_app_message.rs b/veilid-core/src/rpc_processor/rpc_app_message.rs index b265c9eb..57c817e1 100644 --- a/veilid-core/src/rpc_processor/rpc_app_message.rs +++ b/veilid-core/src/rpc_processor/rpc_app_message.rs @@ -28,12 +28,15 @@ impl RPCProcessor { pub(crate) async fn process_app_message(&self, msg: RPCMessage) -> RPCNetworkResult<()> { // Ignore if disabled let routing_table = self.routing_table(); - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - if !opi - .signed_node_info() - .node_info() - .has_capability(CAP_APPMESSAGE) - { + let has_capability_app_message = routing_table + .get_published_peer_info(msg.header.routing_domain()) + .map(|ppi| { + ppi.signed_node_info() + .node_info() + .has_capability(CAP_APPMESSAGE) + }) + .unwrap_or(false); + if !has_capability_app_message { return Ok(NetworkResult::service_unavailable( "app message is not available", )); @@ -58,7 +61,7 @@ impl RPCProcessor { }; // Get the statement - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let app_message = match kind { RPCOperationKind::Statement(s) => match s.destructure() { RPCStatementDetail::AppMessage(s) => s, diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index e6f8c376..73425f73 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -13,7 +13,7 @@ impl RPCProcessor { dest: Destination, node_id: TypedKey, capabilities: Vec, - ) -> RPCNetworkResult>> { + ) -> RPCNetworkResult>>> { let _guard = self .unlocked_inner .startup_lock @@ -56,7 +56,7 @@ impl RPCProcessor { }; // Get the right answer type - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let find_node_a = match kind { RPCOperationKind::Answer(a) => match a.destructure() { RPCAnswerDetail::FindNodeA(a) => a, @@ -70,7 +70,7 @@ impl RPCProcessor { for peer_info in &peers { if !self.verify_node_info( - RoutingDomain::PublicInternet, + peer_info.routing_domain(), peer_info.signed_node_info(), &capabilities, ) { @@ -114,8 +114,13 @@ impl RPCProcessor { // Get a chunk of the routing table near the requested node id let routing_table = self.routing_table(); - let closest_nodes = - network_result_try!(routing_table.find_preferred_closest_peers(node_id, &capabilities)); + let routing_domain = msg.header.routing_domain(); + + let closest_nodes = network_result_try!(routing_table.find_preferred_closest_peers( + routing_domain, + node_id, + &capabilities + )); // Make FindNode answer let find_node_a = RPCOperationFindNodeA::new(closest_nodes)?; diff --git a/veilid-core/src/rpc_processor/rpc_get_value.rs b/veilid-core/src/rpc_processor/rpc_get_value.rs index 46f481bc..04150a1e 100644 --- a/veilid-core/src/rpc_processor/rpc_get_value.rs +++ b/veilid-core/src/rpc_processor/rpc_get_value.rs @@ -4,7 +4,7 @@ use crate::storage_manager::{SignedValueData, SignedValueDescriptor}; #[derive(Clone, Debug)] pub struct GetValueAnswer { pub value: Option, - pub peers: Vec, + pub peers: Vec>, pub descriptor: Option, } @@ -38,7 +38,7 @@ impl RPCProcessor { // Ensure destination never has a private route // and get the target noderef so we can validate the response - let Some(target) = dest.node() else { + let Some(target_node_ids) = dest.get_target_node_ids() else { return Err(RPCError::internal( "Never send get value requests over private routes", )); @@ -48,7 +48,7 @@ impl RPCProcessor { let Some(vcrypto) = self.crypto.get(key.kind) else { return Err(RPCError::internal("unsupported cryptosystem")); }; - let Some(target_node_id) = target.node_ids().get(key.kind) else { + let Some(target_node_id) = target_node_ids.get(key.kind) else { return Err(RPCError::internal("No node id for crypto kind")); }; @@ -94,7 +94,7 @@ impl RPCProcessor { }; // Get the right answer type - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let get_value_a = match kind { RPCOperationKind::Answer(a) => match a.destructure() { RPCAnswerDetail::GetValueA(a) => a, @@ -188,10 +188,15 @@ impl RPCProcessor { )) } } - // Ignore if disabled let routing_table = self.routing_table(); - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - if !opi.signed_node_info().node_info().has_capability(CAP_DHT) { + let routing_domain = msg.header.routing_domain(); + + // Ignore if disabled + let has_capability_dht = routing_table + .get_published_peer_info(msg.header.routing_domain()) + .map(|ppi| ppi.signed_node_info().node_info().has_capability(CAP_DHT)) + .unwrap_or(false); + if !has_capability_dht { return Ok(NetworkResult::service_unavailable("dht is not available")); } @@ -209,9 +214,8 @@ impl RPCProcessor { let (key, subkey, want_descriptor) = get_value_q.destructure(); // Get the nodes that we know about that are closer to the the key than our own node - let routing_table = self.routing_table(); let closer_to_key_peers = network_result_try!( - routing_table.find_preferred_peers_closer_to_key(key, vec![CAP_DHT]) + routing_table.find_preferred_peers_closer_to_key(routing_domain, key, vec![CAP_DHT]) ); if debug_target_enabled!("dht") { diff --git a/veilid-core/src/rpc_processor/rpc_inspect_value.rs b/veilid-core/src/rpc_processor/rpc_inspect_value.rs index 8d57c462..59319dd6 100644 --- a/veilid-core/src/rpc_processor/rpc_inspect_value.rs +++ b/veilid-core/src/rpc_processor/rpc_inspect_value.rs @@ -4,7 +4,7 @@ use crate::storage_manager::SignedValueDescriptor; #[derive(Clone, Debug)] pub struct InspectValueAnswer { pub seqs: Vec, - pub peers: Vec, + pub peers: Vec>, pub descriptor: Option, } @@ -40,7 +40,7 @@ impl RPCProcessor { // Ensure destination never has a private route // and get the target noderef so we can validate the response - let Some(target) = dest.node() else { + let Some(target_node_ids) = dest.get_target_node_ids() else { return Err(RPCError::internal( "Never send get value requests over private routes", )); @@ -50,7 +50,7 @@ impl RPCProcessor { let Some(vcrypto) = self.crypto.get(key.kind) else { return Err(RPCError::internal("unsupported cryptosystem")); }; - let Some(target_node_id) = target.node_ids().get(key.kind) else { + let Some(target_node_id) = target_node_ids.get(key.kind) else { return Err(RPCError::internal("No node id for crypto kind")); }; @@ -97,7 +97,7 @@ impl RPCProcessor { }; // Get the right answer type - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let inspect_value_a = match kind { RPCOperationKind::Answer(a) => match a.destructure() { RPCAnswerDetail::InspectValueA(a) => a, @@ -169,10 +169,15 @@ impl RPCProcessor { )) } } - // Ignore if disabled let routing_table = self.routing_table(); - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - if !opi.signed_node_info().node_info().has_capability(CAP_DHT) { + let routing_domain = msg.header.routing_domain(); + + // Ignore if disabled + let has_capability_dht = routing_table + .get_published_peer_info(msg.header.routing_domain()) + .map(|ppi| ppi.signed_node_info().node_info().has_capability(CAP_DHT)) + .unwrap_or(false); + if !has_capability_dht { return Ok(NetworkResult::service_unavailable("dht is not available")); } @@ -190,9 +195,8 @@ impl RPCProcessor { let (key, subkeys, want_descriptor) = inspect_value_q.destructure(); // Get the nodes that we know about that are closer to the the key than our own node - let routing_table = self.routing_table(); let closer_to_key_peers = network_result_try!( - routing_table.find_preferred_peers_closer_to_key(key, vec![CAP_DHT]) + routing_table.find_preferred_peers_closer_to_key(routing_domain, key, vec![CAP_DHT]) ); if debug_target_enabled!("dht") { diff --git a/veilid-core/src/rpc_processor/rpc_return_receipt.rs b/veilid-core/src/rpc_processor/rpc_return_receipt.rs index 6519d404..fa9acc7f 100644 --- a/veilid-core/src/rpc_processor/rpc_return_receipt.rs +++ b/veilid-core/src/rpc_processor/rpc_return_receipt.rs @@ -30,7 +30,7 @@ impl RPCProcessor { #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_return_receipt(&self, msg: RPCMessage) -> RPCNetworkResult<()> { // Get the statement - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let receipt = match kind { RPCOperationKind::Statement(s) => match s.destructure() { RPCStatementDetail::ReturnReceipt(s) => s.destructure(), diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 601fa699..bb5f34de 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -26,18 +26,18 @@ impl RPCProcessor { } // Get next hop node ref - let Some(mut next_hop_nr) = route_hop + let Some(next_hop_nr) = route_hop .node .node_ref(self.routing_table.clone(), safety_route.public_key.kind) else { - return Err(RPCError::network(format!( + return Ok(NetworkResult::invalid_message(format!( "could not get route node hop ref: {}", route_hop.node.describe(safety_route.public_key.kind) ))); }; // Apply sequencing preference - next_hop_nr.set_sequencing(routed_operation.sequencing()); + let next_hop_nr = next_hop_nr.sequencing_filtered(routed_operation.sequencing()); // Pass along the route let next_hop_route = RPCOperationRoute::new( @@ -72,17 +72,17 @@ impl RPCProcessor { } // Get next hop node ref - let Some(mut next_hop_nr) = + let Some(next_hop_nr) = next_route_node.node_ref(self.routing_table.clone(), safety_route_public_key.kind) else { - return Err(RPCError::network(format!( + return Ok(NetworkResult::invalid_message(format!( "could not get route node hop ref: {}", next_route_node.describe(safety_route_public_key.kind) ))); }; // Apply sequencing preference - next_hop_nr.set_sequencing(routed_operation.sequencing()); + let next_hop_nr = next_hop_nr.sequencing_filtered(routed_operation.sequencing()); // Pass along the route let next_hop_route = RPCOperationRoute::new( @@ -253,10 +253,7 @@ impl RPCProcessor { ) } } - #[cfg_attr( - feature = "verbose-tracing", - instrument(level = "trace", skip_all, err) - )] + #[instrument(level = "trace", target = "rpc", skip_all)] async fn process_private_route_first_hop( &self, @@ -327,7 +324,7 @@ impl RPCProcessor { &self, route_hop_data: &RouteHopData, pr_pubkey: &TypedKey, - route_operation: &mut RoutedOperation, + routed_operation: &mut RoutedOperation, ) -> RPCNetworkResult { // Get crypto kind let crypto_kind = pr_pubkey.kind; @@ -363,7 +360,10 @@ impl RPCProcessor { let rh_reader = dec_blob_reader .get_root::() .map_err(RPCError::protocol)?; - decode_route_hop(&rh_reader)? + let decode_context = RPCDecodeContext { + routing_domain: routed_operation.routing_domain(), + }; + decode_route_hop(&decode_context, &rh_reader)? }; // Validate the RouteHop @@ -377,9 +377,9 @@ impl RPCProcessor { let node_id = self.routing_table.node_id(crypto_kind); let node_id_secret = self.routing_table.node_id_secret_key(crypto_kind); let sig = vcrypto - .sign(&node_id.value, &node_id_secret, route_operation.data()) + .sign(&node_id.value, &node_id_secret, routed_operation.data()) .map_err(RPCError::internal)?; - route_operation.add_signature(sig); + routed_operation.add_signature(sig); } Ok(NetworkResult::value(route_hop)) @@ -389,14 +389,20 @@ impl RPCProcessor { pub(crate) async fn process_route(&self, msg: RPCMessage) -> RPCNetworkResult<()> { // Ignore if disabled let routing_table = self.routing_table(); - if !routing_table.has_valid_network_class(msg.header.routing_domain()) { - return Ok(NetworkResult::service_unavailable( - "can't route without valid network class", - )); - } - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - if !opi.signed_node_info().node_info().has_capability(CAP_ROUTE) { + let Some(published_peer_info) = + routing_table.get_published_peer_info(msg.header.routing_domain()) + else { + return Ok(NetworkResult::service_unavailable( + "Own node info must be published to route", + )); + }; + + if !published_peer_info + .signed_node_info() + .node_info() + .has_capability(CAP_ROUTE) + { return Ok(NetworkResult::service_unavailable("route is not available")); } @@ -411,7 +417,7 @@ impl RPCProcessor { }; // Get the statement - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let route = match kind { RPCOperationKind::Statement(s) => match s.destructure() { RPCStatementDetail::Route(s) => s, @@ -475,7 +481,11 @@ impl RPCProcessor { "failed to get private route reader for blob", )); }; - let Ok(private_route) = decode_private_route(&pr_reader) else { + let decode_context = RPCDecodeContext { + routing_domain: routed_operation.routing_domain(), + }; + let Ok(private_route) = decode_private_route(&decode_context, &pr_reader) + else { return Ok(NetworkResult::invalid_message( "failed to decode private route", )); @@ -509,7 +519,10 @@ impl RPCProcessor { "failed to get route hop reader for blob", )); }; - let Ok(route_hop) = decode_route_hop(&rh_reader) else { + let decode_context = RPCDecodeContext { + routing_domain: routed_operation.routing_domain(), + }; + let Ok(route_hop) = decode_route_hop(&decode_context, &rh_reader) else { return Ok(NetworkResult::invalid_message( "failed to decode route hop", )); diff --git a/veilid-core/src/rpc_processor/rpc_set_value.rs b/veilid-core/src/rpc_processor/rpc_set_value.rs index ad8d64e7..bb6c01db 100644 --- a/veilid-core/src/rpc_processor/rpc_set_value.rs +++ b/veilid-core/src/rpc_processor/rpc_set_value.rs @@ -4,7 +4,7 @@ use super::*; pub struct SetValueAnswer { pub set: bool, pub value: Option, - pub peers: Vec, + pub peers: Vec>, } impl RPCProcessor { @@ -42,7 +42,7 @@ impl RPCProcessor { // Ensure destination never has a private route // and get the target noderef so we can validate the response - let Some(target) = dest.node() else { + let Some(target_node_ids) = dest.get_target_node_ids() else { return Err(RPCError::internal( "Never send set value requests over private routes", )); @@ -52,7 +52,7 @@ impl RPCProcessor { let Some(vcrypto) = self.crypto.get(key.kind) else { return Err(RPCError::internal("unsupported cryptosystem")); }; - let Some(target_node_id) = target.node_ids().get(key.kind) else { + let Some(target_node_id) = target_node_ids.get(key.kind) else { return Err(RPCError::internal("No node id for crypto kind")); }; @@ -106,7 +106,7 @@ impl RPCProcessor { }; // Get the right answer type - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let set_value_a = match kind { RPCOperationKind::Answer(a) => match a.destructure() { RPCAnswerDetail::SetValueA(a) => a, @@ -189,24 +189,26 @@ impl RPCProcessor { #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_set_value_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> { - // Ignore if disabled - let routing_table = self.routing_table(); - let rss = routing_table.route_spec_store(); - - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - if !opi.signed_node_info().node_info().has_capability(CAP_DHT) { - return Ok(NetworkResult::service_unavailable("dht is not available")); - } - // Ensure this never came over a private route, safety route is okay though match &msg.header.detail { RPCMessageHeaderDetail::Direct(_) | RPCMessageHeaderDetail::SafetyRouted(_) => {} RPCMessageHeaderDetail::PrivateRouted(_) => { return Ok(NetworkResult::invalid_message( - "not processing set value request over private route", + "not processing get value request over private route", )) } } + let routing_table = self.routing_table(); + let routing_domain = msg.header.routing_domain(); + + // Ignore if disabled + let has_capability_dht = routing_table + .get_published_peer_info(msg.header.routing_domain()) + .map(|ppi| ppi.signed_node_info().node_info().has_capability(CAP_DHT)) + .unwrap_or(false); + if !has_capability_dht { + return Ok(NetworkResult::service_unavailable("dht is not available")); + } // Get the question let kind = msg.operation.kind().clone(); @@ -223,12 +225,12 @@ impl RPCProcessor { // Get target for ValueChanged notifications let dest = network_result_try!(self.get_respond_to_destination(&msg)); + let rss = routing_table.route_spec_store(); let target = dest.get_target(rss)?; // Get the nodes that we know about that are closer to the the key than our own node - let routing_table = self.routing_table(); let closer_to_key_peers = network_result_try!( - routing_table.find_preferred_peers_closer_to_key(key, vec![CAP_DHT]) + routing_table.find_preferred_peers_closer_to_key(routing_domain, key, vec![CAP_DHT]) ); let debug_string = format!( diff --git a/veilid-core/src/rpc_processor/rpc_signal.rs b/veilid-core/src/rpc_processor/rpc_signal.rs index b5c95c35..5670a433 100644 --- a/veilid-core/src/rpc_processor/rpc_signal.rs +++ b/veilid-core/src/rpc_processor/rpc_signal.rs @@ -41,12 +41,16 @@ impl RPCProcessor { pub(crate) async fn process_signal(&self, msg: RPCMessage) -> RPCNetworkResult<()> { // Ignore if disabled let routing_table = self.routing_table(); - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - if !opi - .signed_node_info() - .node_info() - .has_capability(CAP_SIGNAL) - { + + let has_capability_signal = routing_table + .get_published_peer_info(msg.header.routing_domain()) + .map(|ppi| { + ppi.signed_node_info() + .node_info() + .has_capability(CAP_SIGNAL) + }) + .unwrap_or(false); + if !has_capability_signal { return Ok(NetworkResult::service_unavailable( "signal is not available", )); @@ -62,7 +66,7 @@ impl RPCProcessor { }; // Get the statement - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let signal = match kind { RPCOperationKind::Statement(s) => match s.destructure() { RPCStatementDetail::Signal(s) => s, diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 05f5d4e6..11c500ed 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -83,7 +83,7 @@ impl RPCProcessor { }; // Get the right answer type - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let status_a = match kind { RPCOperationKind::Answer(a) => match a.destructure() { RPCAnswerDetail::StatusA(a) => a, @@ -126,13 +126,13 @@ impl RPCProcessor { .report_public_internet_socket_address( sender_info.socket_address, send_data_method.unique_flow.flow, - target, + target.unfiltered(), ), RoutingDomain::LocalNetwork => { self.network_manager().report_local_network_socket_address( sender_info.socket_address, send_data_method.unique_flow.flow, - target, + target.unfiltered(), ) } } diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 23a5d2b9..a45053a3 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -3,7 +3,6 @@ use super::*; impl RPCProcessor { // Can only be sent directly, not via relays or routes #[instrument(level = "trace", target = "rpc", skip(self), ret, err)] - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] pub async fn rpc_call_validate_dial_info( self, peer: NodeRef, @@ -38,7 +37,7 @@ impl RPCProcessor { // Send the validate_dial_info request // This can only be sent directly, as relays can not validate dial info - network_result_value_or_log!(self.statement(Destination::direct(peer), statement) + network_result_value_or_log!(self.statement(Destination::direct(peer.default_filtered()), statement) .await? => [ format!(": peer={} statement={:?}", peer, statement) ] { return Ok(false); } @@ -82,14 +81,7 @@ impl RPCProcessor { #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_validate_dial_info(&self, msg: RPCMessage) -> RPCNetworkResult<()> { - let routing_table = self.routing_table(); - if !routing_table.has_valid_network_class(msg.header.routing_domain()) { - return Ok(NetworkResult::service_unavailable( - "can't validate dial info without valid network class", - )); - } - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - + // Ensure this never came over a private route, safety route is okay though let detail = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { @@ -100,20 +92,26 @@ impl RPCProcessor { }; // Ignore if disabled - let ni = opi.signed_node_info().node_info(); - if !opi - .signed_node_info() - .node_info() - .has_capability(CAP_VALIDATE_DIAL_INFO) - || !ni.is_fully_direct_inbound() - { + let routing_table = self.routing_table(); + let routing_domain = detail.routing_domain; + + let has_capability_validate_dial_info = routing_table + .get_published_peer_info(routing_domain) + .map(|ppi| { + ppi.signed_node_info() + .node_info() + .has_capability(CAP_VALIDATE_DIAL_INFO) + && ppi.signed_node_info().node_info().is_fully_direct_inbound() + }) + .unwrap_or(false); + if !has_capability_validate_dial_info { return Ok(NetworkResult::service_unavailable( "validate dial info is not available", )); } // Get the statement - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let (dial_info, receipt, redirect) = match kind { RPCOperationKind::Statement(s) => match s.destructure() { RPCStatementDetail::ValidateDialInfo(s) => s.destructure(), @@ -161,7 +159,11 @@ impl RPCProcessor { ]); // Find nodes matching filter to redirect this to - let peers = routing_table.find_fast_public_nodes_filtered(node_count, filters); + let peers = routing_table.find_fast_non_local_nodes_filtered( + routing_domain, + node_count, + filters, + ); if peers.is_empty() { return Ok(NetworkResult::no_connection_other(format!( "no peers able to reach dialinfo '{:?}'", @@ -183,7 +185,7 @@ impl RPCProcessor { // Send the validate_dial_info request // This can only be sent directly, as relays can not validate dial info - network_result_value_or_log!(self.statement(Destination::direct(peer), statement) + network_result_value_or_log!(self.statement(Destination::direct(peer.default_filtered()), statement) .await? => [ format!(": peer={} statement={:?}", peer, statement) ] { continue; } @@ -196,8 +198,7 @@ impl RPCProcessor { )); }; - // Otherwise send a return receipt directly - // Possibly from an alternate port + // Otherwise send a return receipt directly from a system-allocated random port let network_manager = self.network_manager(); network_manager .send_out_of_band_receipt(dial_info.clone(), receipt) diff --git a/veilid-core/src/rpc_processor/rpc_value_changed.rs b/veilid-core/src/rpc_processor/rpc_value_changed.rs index e89cfeb9..37f3bfa0 100644 --- a/veilid-core/src/rpc_processor/rpc_value_changed.rs +++ b/veilid-core/src/rpc_processor/rpc_value_changed.rs @@ -38,7 +38,7 @@ impl RPCProcessor { #[instrument(level = "trace", target = "rpc", skip_all)] pub(crate) async fn process_value_changed(&self, msg: RPCMessage) -> RPCNetworkResult<()> { // Get the statement - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let (key, subkeys, count, watch_id, value) = match kind { RPCOperationKind::Statement(s) => match s.destructure() { RPCStatementDetail::ValueChanged(s) => s.destructure(), diff --git a/veilid-core/src/rpc_processor/rpc_watch_value.rs b/veilid-core/src/rpc_processor/rpc_watch_value.rs index 76fcc915..b5bd7b6b 100644 --- a/veilid-core/src/rpc_processor/rpc_watch_value.rs +++ b/veilid-core/src/rpc_processor/rpc_watch_value.rs @@ -4,7 +4,7 @@ use super::*; pub struct WatchValueAnswer { pub accepted: bool, pub expiration_ts: Timestamp, - pub peers: Vec, + pub peers: Vec>, pub watch_id: u64, } @@ -40,7 +40,7 @@ impl RPCProcessor { // Ensure destination never has a private route // and get the target noderef so we can validate the response - let Some(target) = dest.node() else { + let Some(target_node_ids) = dest.get_target_node_ids() else { return Err(RPCError::internal( "Never send watch value requests over private routes", )); @@ -50,7 +50,7 @@ impl RPCProcessor { let Some(vcrypto) = self.crypto.get(key.kind) else { return Err(RPCError::internal("unsupported cryptosystem")); }; - let Some(target_node_id) = target.node_ids().get(key.kind) else { + let Some(target_node_id) = target_node_ids.get(key.kind) else { return Err(RPCError::internal("No node id for crypto kind")); }; @@ -99,7 +99,7 @@ impl RPCProcessor { }; // Get the right answer type - let (_, _, _, kind) = msg.operation.destructure(); + let (_, _, kind) = msg.operation.destructure(); let watch_value_a = match kind { RPCOperationKind::Answer(a) => match a.destructure() { RPCAnswerDetail::WatchValueA(a) => a, @@ -186,9 +186,6 @@ impl RPCProcessor { #[instrument(level = "trace", target = "rpc", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_watch_value_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> { - let routing_table = self.routing_table(); - let rss = routing_table.route_spec_store(); - // Ensure this never came over a private route, safety route is okay though match &msg.header.detail { RPCMessageHeaderDetail::Direct(_) | RPCMessageHeaderDetail::SafetyRouted(_) => {} @@ -201,17 +198,19 @@ impl RPCProcessor { // Ignore if disabled let routing_table = self.routing_table(); - let opi = routing_table.get_own_peer_info(msg.header.routing_domain()); - if !opi.signed_node_info().node_info().has_capability(CAP_DHT) { - return Ok(NetworkResult::service_unavailable("dht is not available")); - } - if !opi - .signed_node_info() - .node_info() - .has_capability(CAP_DHT_WATCH) - { + let routing_domain = msg.header.routing_domain(); + + let has_capability_dht_and_watch = routing_table + .get_published_peer_info(msg.header.routing_domain()) + .map(|ppi| { + ppi.signed_node_info() + .node_info() + .has_all_capabilities(&[CAP_DHT, CAP_DHT_WATCH]) + }) + .unwrap_or(false); + if !has_capability_dht_and_watch { return Ok(NetworkResult::service_unavailable( - "dht watch is not available", + "dht and dht_watch are not available", )); } @@ -231,6 +230,7 @@ impl RPCProcessor { // Get target for ValueChanged notifications let dest = network_result_try!(self.get_respond_to_destination(&msg)); + let rss = routing_table.route_spec_store(); let target = dest.get_target(rss)?; if debug_target_enabled!("dht") { @@ -253,9 +253,8 @@ impl RPCProcessor { } // Get the nodes that we know about that are closer to the the key than our own node - let closer_to_key_peers = network_result_try!( - routing_table.find_preferred_peers_closer_to_key(key, vec![CAP_DHT, CAP_DHT_WATCH]) - ); + let closer_to_key_peers = network_result_try!(routing_table + .find_preferred_peers_closer_to_key(routing_domain, key, vec![CAP_DHT, CAP_DHT_WATCH])); // See if we would have accepted this as a set, same set_value_count for watches let set_value_count = { diff --git a/veilid-core/src/storage_manager/get_value.rs b/veilid-core/src/storage_manager/get_value.rs index 4a6563c0..0c276d20 100644 --- a/veilid-core/src/storage_manager/get_value.rs +++ b/veilid-core/src/storage_manager/get_value.rs @@ -35,6 +35,7 @@ impl StorageManager { last_get_result: GetResult, ) -> VeilidAPIResult>> { let routing_table = rpc_processor.routing_table(); + let routing_domain = RoutingDomain::PublicInternet; // Get the DHT parameters for 'GetValue' let (key_count, consensus_count, fanout, timeout_us) = { @@ -50,7 +51,16 @@ impl StorageManager { // Get the nodes we know are caching this value to seed the fanout let init_fanout_queue = { let inner = self.inner.lock().await; - inner.get_value_nodes(key)?.unwrap_or_default() + inner + .get_value_nodes(key)? + .unwrap_or_default() + .into_iter() + .filter(|x| { + x.node_info(routing_domain) + .map(|ni| ni.has_all_capabilities(&[CAP_DHT])) + .unwrap_or_default() + }) + .collect() }; // Parse the schema @@ -85,7 +95,7 @@ impl StorageManager { rpc_processor .clone() .rpc_call_get_value( - Destination::direct(next_node.clone()) + Destination::direct(next_node.routing_domain_filtered(routing_domain)) .with_safety(safety_selection), key, subkey, @@ -115,7 +125,7 @@ impl StorageManager { // Return peers if we have some log_network_result!(debug "GetValue returned no value, fanout call returned peers {}", gva.answer.peers.len()); - return Ok(NetworkResult::value(gva.answer.peers)) + return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list: gva.answer.peers})) }; log_dht!(debug "GetValue got value back: len={} seq={}", value.value_data().data().len(), value.value_data().seq()); @@ -180,7 +190,7 @@ impl StorageManager { // Return peers if we have some log_network_result!(debug "GetValue fanout call returned peers {}", gva.answer.peers.len()); - Ok(NetworkResult::value(gva.answer.peers)) + Ok(NetworkResult::value(FanoutCallOutput{peer_info_list: gva.answer.peers})) }.instrument(tracing::trace_span!("outbound_get_value fanout routine")) } }; diff --git a/veilid-core/src/storage_manager/inspect_value.rs b/veilid-core/src/storage_manager/inspect_value.rs index 7483caaf..fc54330d 100644 --- a/veilid-core/src/storage_manager/inspect_value.rs +++ b/veilid-core/src/storage_manager/inspect_value.rs @@ -60,6 +60,7 @@ impl StorageManager { use_set_scope: bool, ) -> VeilidAPIResult { let routing_table = rpc_processor.routing_table(); + let routing_domain = RoutingDomain::PublicInternet; // Get the DHT parameters for 'InspectValue' // Can use either 'get scope' or 'set scope' depending on the purpose of the inspection @@ -86,7 +87,16 @@ impl StorageManager { // Get the nodes we know are caching this value to seed the fanout let init_fanout_queue = { let inner = self.inner.lock().await; - inner.get_value_nodes(key)?.unwrap_or_default() + inner + .get_value_nodes(key)? + .unwrap_or_default() + .into_iter() + .filter(|x| { + x.node_info(routing_domain) + .map(|ni| ni.has_all_capabilities(&[CAP_DHT])) + .unwrap_or_default() + }) + .collect() }; // Make do-inspect-value answer context @@ -120,7 +130,7 @@ impl StorageManager { rpc_processor .clone() .rpc_call_inspect_value( - Destination::direct(next_node.clone()).with_safety(safety_selection), + Destination::direct(next_node.routing_domain_filtered(routing_domain)).with_safety(safety_selection), key, subkeys.clone(), opt_descriptor.map(|x| (*x).clone()), @@ -227,7 +237,7 @@ impl StorageManager { // Return peers if we have some log_network_result!(debug "InspectValue fanout call returned peers {}", answer.peers.len()); - Ok(NetworkResult::value(answer.peers)) + Ok(NetworkResult::value(FanoutCallOutput { peer_info_list: answer.peers})) }.instrument(tracing::trace_span!("outbound_inspect_value fanout call")) }; diff --git a/veilid-core/src/storage_manager/mod.rs b/veilid-core/src/storage_manager/mod.rs index bcfb0179..6de9facf 100644 --- a/veilid-core/src/storage_manager/mod.rs +++ b/veilid-core/src/storage_manager/mod.rs @@ -9,7 +9,6 @@ mod types; mod watch_value; use super::*; -use network_manager::*; use record_store::*; use routing_table::*; use rpc_processor::*; @@ -154,17 +153,19 @@ impl StorageManager { pub async fn terminate(&self) { log_stor!(debug "starting storage manager shutdown"); + // Stop the background ticker process { let mut inner = self.inner.lock().await; - inner.terminate().await; + inner.stop_ticker().await; } // Cancel all tasks self.cancel_tasks().await; - // Release the storage manager + // Terminate and release the storage manager { let mut inner = self.inner.lock().await; + inner.terminate().await; *inner = Self::new_inner(self.unlocked_inner.clone()); } @@ -190,25 +191,9 @@ impl StorageManager { } fn online_ready_inner(inner: &StorageManagerInner) -> Option { - if let Some(rpc_processor) = { inner.opt_rpc_processor.clone() } { - if let Some(network_class) = rpc_processor - .routing_table() - .get_network_class(RoutingDomain::PublicInternet) - { - // If our PublicInternet network class is valid we're ready to talk - if network_class != NetworkClass::Invalid { - Some(rpc_processor) - } else { - None - } - } else { - // If we haven't gotten a network class yet we shouldnt try to use the DHT - None - } - } else { - // If we aren't attached, we won't have an rpc processor - None - } + let routing_table = inner.opt_routing_table.clone()?; + routing_table.get_published_peer_info(RoutingDomain::PublicInternet)?; + inner.opt_rpc_processor.clone() } async fn online_writes_ready(&self) -> EyreResult> { @@ -872,7 +857,7 @@ impl StorageManager { let offline_subkey_writes = inner .offline_subkey_writes .get(&key) - .map(|o| o.subkeys.clone()) + .map(|o| o.subkeys.union(&o.subkeys_in_flight)) .unwrap_or_default() .intersect(&subkeys); @@ -1015,8 +1000,20 @@ impl StorageManager { match fanout_result.kind { FanoutResultKind::Partial => false, FanoutResultKind::Timeout => { - log_stor!(debug "timeout in set_value, adding offline subkey: {}:{}", key, subkey); - true + let get_consensus = + self.unlocked_inner.config.get().network.dht.get_value_count as usize; + let value_node_count = fanout_result.value_nodes.len(); + if value_node_count < get_consensus { + log_stor!(debug "timeout with insufficient consensus ({}<{}), adding offline subkey: {}:{}", + value_node_count, get_consensus, + key, subkey); + true + } else { + log_stor!(debug "timeout with sufficient consensus ({}>={}): set_value {}:{}", + value_node_count, get_consensus, + key, subkey); + false + } } FanoutResultKind::Exhausted => { let get_consensus = @@ -1028,6 +1025,9 @@ impl StorageManager { key, subkey); true } else { + log_stor!(debug "exhausted with sufficient consensus ({}>={}): set_value {}:{}", + value_node_count, get_consensus, + key, subkey); false } } diff --git a/veilid-core/src/storage_manager/set_value.rs b/veilid-core/src/storage_manager/set_value.rs index 6821f5cb..0472031c 100644 --- a/veilid-core/src/storage_manager/set_value.rs +++ b/veilid-core/src/storage_manager/set_value.rs @@ -36,12 +36,14 @@ impl StorageManager { descriptor: Arc, ) -> VeilidAPIResult>> { let routing_table = rpc_processor.routing_table(); + let routing_domain = RoutingDomain::PublicInternet; // Get the DHT parameters for 'SetValue' - let (key_count, consensus_count, fanout, timeout_us) = { + let (key_count, get_consensus_count, set_consensus_count, fanout, timeout_us) = { let c = self.unlocked_inner.config.get(); ( c.network.dht.max_find_node_count as usize, + c.network.dht.get_value_count as usize, c.network.dht.set_value_count as usize, c.network.dht.set_value_fanout as usize, TimestampDuration::from(ms_to_us(c.network.dht.set_value_timeout_ms)), @@ -51,7 +53,16 @@ impl StorageManager { // Get the nodes we know are caching this value to seed the fanout let init_fanout_queue = { let inner = self.inner.lock().await; - inner.get_value_nodes(key)?.unwrap_or_default() + inner + .get_value_nodes(key)? + .unwrap_or_default() + .into_iter() + .filter(|x| { + x.node_info(routing_domain) + .map(|ni| ni.has_all_capabilities(&[CAP_DHT])) + .unwrap_or_default() + }) + .collect() }; // Make the return channel @@ -90,7 +101,7 @@ impl StorageManager { rpc_processor .clone() .rpc_call_set_value( - Destination::direct(next_node.clone()) + Destination::direct(next_node.routing_domain_filtered(routing_domain)) .with_safety(safety_selection), key, subkey, @@ -108,7 +119,7 @@ impl StorageManager { // Return peers if we have some log_network_result!(debug "SetValue missed: {}, fanout call returned peers {}", ctx.missed_since_last_set, sva.answer.peers.len()); - return Ok(NetworkResult::value(sva.answer.peers)); + return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers})); } // See if we got a value back @@ -123,7 +134,7 @@ impl StorageManager { // Return peers if we have some log_network_result!(debug "SetValue returned no value, fanout call returned peers {}", sva.answer.peers.len()); - return Ok(NetworkResult::value(sva.answer.peers)); + return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers})); }; // Keep the value if we got one and it is newer and it passes schema validation @@ -153,7 +164,7 @@ impl StorageManager { ctx.send_partial_update = true; } - return Ok(NetworkResult::value(sva.answer.peers)); + return Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers})); } // We have a prior value, ensure this is a newer sequence number @@ -175,7 +186,7 @@ impl StorageManager { // Send an update since the value changed ctx.send_partial_update = true; - Ok(NetworkResult::value(sva.answer.peers)) + Ok(NetworkResult::value(FanoutCallOutput{peer_info_list:sva.answer.peers})) }.instrument(tracing::trace_span!("fanout call_routine")) } }; @@ -207,15 +218,16 @@ impl StorageManager { } } - // If we have reached sufficient consensus, return done - if ctx.value_nodes.len() >= consensus_count { + // If we have reached set consensus (the max consensus we care about), return done + if ctx.value_nodes.len() >= set_consensus_count { return Some(()); } - // If we have missed more than our consensus count since our last set, return done + + // If we have missed get_consensus count (the minimum consensus we care about) or more since our last set, return done // This keeps the traversal from searching too many nodes when we aren't converging - // Only do this if we have gotten at least half our desired sets. - if ctx.value_nodes.len() >= ((consensus_count + 1) / 2) - && ctx.missed_since_last_set >= consensus_count + // Only do this if we have gotten at least the get_consensus (the minimum consensus we care about) + if ctx.value_nodes.len() >= get_consensus_count + && ctx.missed_since_last_set >= get_consensus_count { return Some(()); } diff --git a/veilid-core/src/storage_manager/storage_manager_inner.rs b/veilid-core/src/storage_manager/storage_manager_inner.rs index b8c1893a..93934739 100644 --- a/veilid-core/src/storage_manager/storage_manager_inner.rs +++ b/veilid-core/src/storage_manager/storage_manager_inner.rs @@ -7,6 +7,8 @@ const OFFLINE_SUBKEY_WRITES: &[u8] = b"offline_subkey_writes"; pub(super) struct OfflineSubkeyWrite { pub safety_selection: SafetySelection, pub subkeys: ValueSubkeyRangeSet, + #[serde(default)] + pub subkeys_in_flight: ValueSubkeyRangeSet, } /// Locked structure for storage manager @@ -148,14 +150,16 @@ impl StorageManagerInner { Ok(()) } - pub async fn terminate(&mut self) { - self.update_callback = None; - + pub async fn stop_ticker(&mut self) { // Stop ticker let tick_future = self.tick_future.take(); if let Some(f) = tick_future { f.await; } + } + + pub async fn terminate(&mut self) { + self.update_callback = None; // Stop deferred result processor self.deferred_result_processor.terminate().await; @@ -729,6 +733,7 @@ impl StorageManagerInner { .or_insert(OfflineSubkeyWrite { safety_selection, subkeys: ValueSubkeyRangeSet::single(subkey), + subkeys_in_flight: ValueSubkeyRangeSet::new(), }); } diff --git a/veilid-core/src/storage_manager/tasks/offline_subkey_writes.rs b/veilid-core/src/storage_manager/tasks/offline_subkey_writes.rs index d565b030..cf330fe1 100644 --- a/veilid-core/src/storage_manager/tasks/offline_subkey_writes.rs +++ b/veilid-core/src/storage_manager/tasks/offline_subkey_writes.rs @@ -1,7 +1,247 @@ use super::*; use futures_util::*; +use stop_token::future::FutureExt as _; + +#[derive(Debug)] +enum OfflineSubkeyWriteResult { + Finished(set_value::OutboundSetValueResult), + Cancelled, + Dropped, +} + +#[derive(Debug)] +struct WorkItem { + key: TypedKey, + safety_selection: SafetySelection, + subkeys: ValueSubkeyRangeSet, +} + +#[derive(Debug)] +struct WorkItemResult { + key: TypedKey, + written_subkeys: ValueSubkeyRangeSet, + fanout_results: Vec<(ValueSubkey, FanoutResult)>, +} impl StorageManager { + // Write a single offline subkey + #[instrument(level = "trace", target = "stor", skip_all, err)] + async fn write_single_offline_subkey( + self, + stop_token: StopToken, + key: TypedKey, + subkey: ValueSubkey, + safety_selection: SafetySelection, + ) -> EyreResult { + let Some(rpc_processor) = self.online_writes_ready().await? else { + // Cancel this operation because we're offline + return Ok(OfflineSubkeyWriteResult::Cancelled); + }; + let get_result = { + let mut inner = self.lock().await?; + inner.handle_get_local_value(key, subkey, true).await + }; + let Ok(get_result) = get_result else { + log_stor!(debug "Offline subkey write had no subkey result: {}:{}", key, subkey); + // drop this one + return Ok(OfflineSubkeyWriteResult::Dropped); + }; + let Some(value) = get_result.opt_value else { + log_stor!(debug "Offline subkey write had no subkey value: {}:{}", key, subkey); + // drop this one + return Ok(OfflineSubkeyWriteResult::Dropped); + }; + let Some(descriptor) = get_result.opt_descriptor else { + log_stor!(debug "Offline subkey write had no descriptor: {}:{}", key, subkey); + return Ok(OfflineSubkeyWriteResult::Dropped); + }; + log_stor!(debug "Offline subkey write: {}:{} len={}", key, subkey, value.value_data().data().len()); + let osvres = self + .outbound_set_value( + rpc_processor, + key, + subkey, + safety_selection, + value.clone(), + descriptor, + ) + .await; + match osvres { + Ok(res_rx) => { + while let Ok(Ok(res)) = res_rx.recv_async().timeout_at(stop_token.clone()).await { + match res { + Ok(result) => { + let partial = result.fanout_result.kind.is_partial(); + // Skip partial results in offline subkey write mode + if partial { + continue; + } + + // Set the new value if it differs from what was asked to set + if result.signed_value_data.value_data() != value.value_data() { + // Record the newer value and send and update since it is different than what we just set + let mut inner = self.lock().await?; + inner + .handle_set_local_value( + key, + subkey, + result.signed_value_data.clone(), + WatchUpdateMode::UpdateAll, + ) + .await?; + } + + return Ok(OfflineSubkeyWriteResult::Finished(result)); + } + Err(e) => { + log_stor!(debug "failed to get offline subkey write result: {}:{} {}", key, subkey, e); + return Ok(OfflineSubkeyWriteResult::Cancelled); + } + } + } + log_stor!(debug "writing offline subkey did not complete {}:{}", key, subkey); + return Ok(OfflineSubkeyWriteResult::Cancelled); + } + Err(e) => { + log_stor!(debug "failed to write offline subkey: {}:{} {}", key, subkey, e); + return Ok(OfflineSubkeyWriteResult::Cancelled); + } + } + } + + // Write a set of subkeys of the same key + #[instrument(level = "trace", target = "stor", skip_all, err)] + async fn process_work_item( + self, + stop_token: StopToken, + work_item: WorkItem, + ) -> EyreResult { + let mut written_subkeys = ValueSubkeyRangeSet::new(); + let mut fanout_results = Vec::<(ValueSubkey, FanoutResult)>::new(); + + for subkey in work_item.subkeys.iter() { + if poll!(stop_token.clone()).is_ready() { + break; + } + + let result = match self + .clone() + .write_single_offline_subkey( + stop_token.clone(), + work_item.key, + subkey, + work_item.safety_selection, + ) + .await? + { + OfflineSubkeyWriteResult::Finished(r) => r, + OfflineSubkeyWriteResult::Cancelled => { + // Stop now and return what we have + break; + } + OfflineSubkeyWriteResult::Dropped => { + // Don't process this item any more but continue + written_subkeys.insert(subkey); + continue; + } + }; + + // Process non-partial setvalue result + let was_offline = + self.check_fanout_set_offline(work_item.key, subkey, &result.fanout_result); + if !was_offline { + written_subkeys.insert(subkey); + } + fanout_results.push((subkey, result.fanout_result)); + } + + Ok(WorkItemResult { + key: work_item.key, + written_subkeys, + fanout_results, + }) + } + + // Process all results + fn prepare_all_work( + offline_subkey_writes: HashMap, + ) -> VecDeque { + offline_subkey_writes + .into_iter() + .map(|(key, v)| WorkItem { + key, + safety_selection: v.safety_selection, + subkeys: v.subkeys_in_flight, + }) + .collect() + } + + // Process all results + #[instrument(level = "trace", target = "stor", skip_all)] + fn process_single_result_inner(inner: &mut StorageManagerInner, result: WorkItemResult) { + // Debug print the result + log_stor!(debug "Offline write result: {:?}", result); + + // Get the offline subkey write record + match inner.offline_subkey_writes.entry(result.key) { + std::collections::hash_map::Entry::Occupied(mut o) => { + let finished = { + let osw = o.get_mut(); + + // Mark in-flight subkeys as having been completed + let subkeys_still_offline = + osw.subkeys_in_flight.difference(&result.written_subkeys); + // Now any left over are still offline, so merge them with any subkeys that have been added while we were working + osw.subkeys = osw.subkeys.union(&subkeys_still_offline); + // And clear the subkeys in flight since we're done with this key for now + osw.subkeys_in_flight.clear(); + + osw.subkeys.is_empty() + }; + if finished { + log_stor!(debug "Offline write finished key {}", result.key); + o.remove(); + } + } + std::collections::hash_map::Entry::Vacant(_) => { + panic!( + "offline write work items should always be on offline_subkey_writes entries that exist" + ) + } + } + + // Keep the list of nodes that returned a value for later reference + inner.process_fanout_results( + result.key, + result.fanout_results.iter().map(|x| (x.0, &x.1)), + true, + ); + } + + #[instrument(level = "trace", target = "stor", skip_all, err)] + pub(crate) async fn process_offline_subkey_writes( + self, + stop_token: StopToken, + work_items: Arc>>, + ) -> EyreResult<()> { + // Process all work items + loop { + let Some(work_item) = work_items.lock().pop_front() else { + break; + }; + let result = self + .clone() + .process_work_item(stop_token.clone(), work_item) + .await?; + { + let mut inner = self.lock().await?; + Self::process_single_result_inner(&mut inner, result); + } + } + + Ok(()) + } + // Best-effort write subkeys to the network that were written offline #[instrument(level = "trace", target = "stor", skip_all, err)] pub(crate) async fn offline_subkey_writes_task_routine( @@ -10,138 +250,37 @@ impl StorageManager { _last_ts: Timestamp, _cur_ts: Timestamp, ) -> EyreResult<()> { - let mut offline_subkey_writes = { + // Operate on a copy of the offline subkey writes map + let work_items = { let mut inner = self.lock().await?; - let out = inner.offline_subkey_writes.clone(); - inner.offline_subkey_writes.clear(); - out + // Move the current set of writes to 'in flight' + for osw in &mut inner.offline_subkey_writes { + osw.1.subkeys_in_flight = mem::take(&mut osw.1.subkeys); + } + + // Prepare items to work on + Arc::new(Mutex::new(Self::prepare_all_work( + inner.offline_subkey_writes.clone(), + ))) }; - for (key, osw) in offline_subkey_writes.iter_mut() { - if poll!(stop_token.clone()).is_ready() { - log_stor!(debug "Offline subkey writes cancelled."); - break; - } - let Some(rpc_processor) = self.online_writes_ready().await? else { - log_stor!(debug "Offline subkey writes stopped for network."); - break; - }; + // Process everything + let res = self + .clone() + .process_offline_subkey_writes(stop_token, work_items) + .await; - let mut fanout_results = vec![]; - - let mut written_subkeys = ValueSubkeyRangeSet::new(); - for subkey in osw.subkeys.iter() { - let get_result = { - let mut inner = self.lock().await?; - inner.handle_get_local_value(*key, subkey, true).await - }; - let Ok(get_result) = get_result else { - log_stor!(debug "Offline subkey write had no subkey result: {}:{}", key, subkey); - // drop this one - written_subkeys.insert(subkey); - continue; - }; - let Some(value) = get_result.opt_value else { - log_stor!(debug "Offline subkey write had no subkey value: {}:{}", key, subkey); - // drop this one - written_subkeys.insert(subkey); - continue; - }; - let Some(descriptor) = get_result.opt_descriptor else { - log_stor!(debug "Offline subkey write had no descriptor: {}:{}", key, subkey); - // drop this one - written_subkeys.insert(subkey); - continue; - }; - log_stor!(debug "Offline subkey write: {}:{} len={}", key, subkey, value.value_data().data().len()); - let osvres = self - .outbound_set_value( - rpc_processor.clone(), - *key, - subkey, - osw.safety_selection, - value.clone(), - descriptor, - ) - .await; - match osvres { - Ok(res_rx) => { - while let Ok(res) = res_rx.recv_async().await { - match res { - Ok(result) => { - let partial = result.fanout_result.kind.is_partial(); - // Skip partial results in offline subkey write mode - if partial { - continue; - } - - // Process non-partial setvalue result - let was_offline = self.check_fanout_set_offline( - *key, - subkey, - &result.fanout_result, - ); - if !was_offline { - written_subkeys.insert(subkey); - } - - // Set the new value if it differs from what was asked to set - if result.signed_value_data.value_data() != value.value_data() { - // Record the newer value and send and update since it is different than what we just set - let mut inner = self.lock().await?; - inner - .handle_set_local_value( - *key, - subkey, - result.signed_value_data.clone(), - WatchUpdateMode::UpdateAll, - ) - .await?; - } - - fanout_results.push((subkey, result.fanout_result)); - break; - } - Err(e) => { - log_stor!(debug "failed to get offline subkey write result: {}:{} {}", key, subkey, e); - break; - } - } - } - } - Err(e) => { - log_stor!(debug "failed to write offline subkey: {}:{} {}", key, subkey, e); - } - } - } - - osw.subkeys = osw.subkeys.difference(&written_subkeys); - - // Keep the list of nodes that returned a value for later reference - { - let mut inner = self.lock().await?; - inner.process_fanout_results( - *key, - fanout_results.iter().map(|x| (x.0, &x.1)), - true, - ); + // Ensure nothing is left in-flight when returning even due to an error + { + let mut inner = self.lock().await?; + for osw in &mut inner.offline_subkey_writes { + osw.1.subkeys = osw + .1 + .subkeys + .union(&mem::take(&mut osw.1.subkeys_in_flight)); } } - // Add any subkeys back in that were not successfully written - let mut inner = self.lock().await?; - for (key, osw) in offline_subkey_writes { - if !osw.subkeys.is_empty() { - inner - .offline_subkey_writes - .entry(key) - .and_modify(|x| { - x.subkeys = x.subkeys.union(&osw.subkeys); - }) - .or_insert(osw); - } - } - - Ok(()) + res } } diff --git a/veilid-core/src/storage_manager/watch_value.rs b/veilid-core/src/storage_manager/watch_value.rs index e7a7f5aa..5412560d 100644 --- a/veilid-core/src/storage_manager/watch_value.rs +++ b/veilid-core/src/storage_manager/watch_value.rs @@ -33,6 +33,8 @@ impl StorageManager { watch_id: u64, watch_node: NodeRef, ) -> VeilidAPIResult> { + let routing_domain = RoutingDomain::PublicInternet; + // Get the appropriate watcher key, if anonymous use a static anonymous watch key // which lives for the duration of the app's runtime let watcher = opt_watcher.unwrap_or_else(|| { @@ -47,7 +49,8 @@ impl StorageManager { rpc_processor .clone() .rpc_call_watch_value( - Destination::direct(watch_node.clone()).with_safety(safety_selection), + Destination::direct(watch_node.routing_domain_filtered(routing_domain)) + .with_safety(safety_selection), key, subkeys, Timestamp::default(), @@ -87,6 +90,8 @@ impl StorageManager { watch_id: u64, watch_node: NodeRef, ) -> VeilidAPIResult> { + let routing_domain = RoutingDomain::PublicInternet; + if count == 0 { apibail_internal!("cancel should be done with outbound_watch_value_cancel"); } @@ -108,7 +113,8 @@ impl StorageManager { rpc_processor .clone() .rpc_call_watch_value( - Destination::direct(watch_node.clone()).with_safety(safety_selection), + Destination::direct(watch_node.routing_domain_filtered(routing_domain)) + .with_safety(safety_selection), key, subkeys, expiration, @@ -204,6 +210,7 @@ impl StorageManager { } let routing_table = rpc_processor.routing_table(); + let routing_domain = RoutingDomain::PublicInternet; // Get the DHT parameters for 'WatchValue', some of which are the same for 'SetValue' operations let (key_count, timeout_us, set_value_count) = { @@ -233,7 +240,7 @@ impl StorageManager { .unwrap_or_default() .into_iter() .filter(|x| { - x.node_info(RoutingDomain::PublicInternet) + x.node_info(routing_domain) .map(|ni| ni.has_all_capabilities(&[CAP_DHT, CAP_DHT_WATCH])) .unwrap_or_default() }) @@ -256,7 +263,7 @@ impl StorageManager { rpc_processor .clone() .rpc_call_watch_value( - Destination::direct(next_node.clone()).with_safety(safety_selection), + Destination::direct(next_node.routing_domain_filtered(routing_domain)).with_safety(safety_selection), key, subkeys, expiration, @@ -293,7 +300,7 @@ impl StorageManager { // Return peers if we have some log_network_result!(debug "WatchValue fanout call returned peers {} ({})", wva.answer.peers.len(), next_node); - Ok(NetworkResult::value(wva.answer.peers)) + Ok(NetworkResult::value(FanoutCallOutput{peer_info_list: wva.answer.peers})) }.instrument(tracing::trace_span!("outbound_watch_value call routine")) }; diff --git a/veilid-core/src/tests/android/mod.rs b/veilid-core/src/tests/android/mod.rs index 405e942e..83ca0ed7 100644 --- a/veilid-core/src/tests/android/mod.rs +++ b/veilid-core/src/tests/android/mod.rs @@ -20,7 +20,7 @@ pub extern "system" fn Java_com_veilid_veilid_1core_1android_1tests_MainActivity pub fn veilid_core_setup_android_tests(env: JNIEnv, ctx: JObject) { // Set up subscriber and layers - let filter = VeilidLayerFilter::new(VeilidConfigLogLevel::Info, None); + let filter = VeilidLayerFilter::new(VeilidConfigLogLevel::Info, &[]); let layer = paranoid_android::layer("veilid-core"); tracing_subscriber::registry() .with(layer.with_filter(filter)) diff --git a/veilid-core/src/tests/ios/mod.rs b/veilid-core/src/tests/ios/mod.rs index 7317d399..ce86f263 100644 --- a/veilid-core/src/tests/ios/mod.rs +++ b/veilid-core/src/tests/ios/mod.rs @@ -16,7 +16,7 @@ pub extern "C" fn run_veilid_core_tests() { pub fn veilid_core_setup_ios_tests() { // Set up subscriber and layers - let filter = VeilidLayerFilter::new(VeilidConfigLogLevel::Info, None); + let filter = VeilidLayerFilter::new(VeilidConfigLogLevel::Info, &[]); tracing_subscriber::registry() .with(OsLogger::new("com.veilid.veilidcore-tests", "").with_filter(filter)) .init(); diff --git a/veilid-core/src/veilid_api/api.rs b/veilid-core/src/veilid_api/api.rs index 54f8b6e4..b116e672 100644 --- a/veilid-core/src/veilid_api/api.rs +++ b/veilid-core/src/veilid_api/api.rs @@ -277,7 +277,7 @@ impl VeilidAPI { /// /// Returns a route id and 'blob' that can be published over some means (DHT or otherwise) to be /// imported by another Veilid node. - #[instrument(target = "veilid_api", level = "debug", skip(self), ret, err)] + #[instrument(target = "veilid_api", level = "debug", skip(self), ret)] pub async fn new_custom_private_route( &self, crypto_kinds: &[CryptoKind], @@ -310,9 +310,18 @@ impl VeilidAPI { &[], false, )?; - if !rss.test_route(route_id).await? { - rss.release_route(route_id); - apibail_generic!("allocated route failed to test"); + match rss.test_route(route_id).await? { + Some(true) => { + // route tested okay + } + Some(false) => { + rss.release_route(route_id); + apibail_generic!("allocated route failed to test"); + } + None => { + rss.release_route(route_id); + apibail_generic!("allocated route could not be tested"); + } } let private_routes = rss.assemble_private_routes(&route_id, Some(true))?; let blob = match RouteSpecStore::private_routes_to_blob(&private_routes) { diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 9a769b3a..359449c2 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -195,8 +195,9 @@ fn get_safety_selection(routing_table: RoutingTable) -> impl Fn(&str) -> Option< } } -fn get_node_ref_modifiers(mut node_ref: NodeRef) -> impl FnOnce(&str) -> Option { +fn get_node_ref_modifiers(node_ref: NodeRef) -> impl FnOnce(&str) -> Option { move |text| { + let mut node_ref = node_ref.default_filtered(); for m in text.split('/') { if let Some(pt) = get_protocol_type(m) { node_ref.merge_filter(NodeRefFilter::new().with_protocol_type(pt)); @@ -273,10 +274,10 @@ fn get_dht_key( } } -fn resolve_node_ref( +fn resolve_filtered_node_ref( routing_table: RoutingTable, safety_selection: SafetySelection, -) -> impl FnOnce(&str) -> SendPinBoxFuture> { +) -> impl FnOnce(&str) -> SendPinBoxFuture> { move |text| { let text = text.to_owned(); Box::pin(async move { @@ -285,7 +286,7 @@ fn resolve_node_ref( .map(|x| (x.0, Some(x.1))) .unwrap_or((&text, None)); - let mut nr = if let Some(key) = get_public_key(text) { + let nr = if let Some(key) = get_public_key(text) { let node_id = TypedKey::new(best_crypto_kind(), key); routing_table .rpc_processor() @@ -304,21 +305,37 @@ fn resolve_node_ref( return None; }; if let Some(mods) = mods { - nr = get_node_ref_modifiers(nr)(mods)?; + Some(get_node_ref_modifiers(nr)(mods)?) + } else { + Some(nr.default_filtered()) } - Some(nr) }) } } fn get_node_ref(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option { + move |text| { + let nr = if let Some(key) = get_public_key(text) { + routing_table.lookup_any_node_ref(key).ok().flatten()? + } else if let Some(node_id) = get_typed_key(text) { + routing_table.lookup_node_ref(node_id).ok().flatten()? + } else { + return None; + }; + Some(nr) + } +} + +fn get_filtered_node_ref( + routing_table: RoutingTable, +) -> impl FnOnce(&str) -> Option { move |text| { let (text, mods) = text .split_once('/') .map(|x| (x.0, Some(x.1))) .unwrap_or((text, None)); - let mut nr = if let Some(key) = get_public_key(text) { + let nr = if let Some(key) = get_public_key(text) { routing_table.lookup_any_node_ref(key).ok().flatten()? } else if let Some(node_id) = get_typed_key(text) { routing_table.lookup_node_ref(node_id).ok().flatten()? @@ -326,9 +343,10 @@ fn get_node_ref(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option Option { } } +fn get_published(text: &str) -> Option { + let ptext = text.to_ascii_lowercase(); + if ptext == "published" { + Some(true) + } else if ptext == "current" { + Some(false) + } else { + None + } +} + fn get_debug_argument Option>( value: &str, context: &str, @@ -470,8 +499,12 @@ pub fn print_data(data: &[u8], truncate_len: Option) -> String { } } - let (data, truncated) = if truncate_len.is_some() && data.len() > truncate_len.unwrap() { - (&data[0..64], true) + let (data, truncated) = if let Some(truncate_len) = truncate_len { + if data.len() > truncate_len { + (&data[0..truncate_len], true) + } else { + (data, false) + } } else { (data, false) }; @@ -521,17 +554,31 @@ impl VeilidAPI { let args: Vec = args.split_whitespace().map(|s| s.to_owned()).collect(); let routing_table = self.network_manager()?.routing_table(); - let routing_domain = get_debug_argument_at( - &args, - 0, - "debug_peerinfo", - "routing_domain", - get_routing_domain, - ) - .ok() - .unwrap_or(RoutingDomain::PublicInternet); + let mut ai = 0; + let mut opt_routing_domain = None; + let mut opt_published = None; - Ok(routing_table.debug_info_peerinfo(routing_domain)) + while ai < args.len() { + if let Ok(routing_domain) = get_debug_argument_at( + &args, + ai, + "debug_peerinfo", + "routing_domain", + get_routing_domain, + ) { + opt_routing_domain = Some(routing_domain); + } else if let Ok(published) = + get_debug_argument_at(&args, ai, "debug_peerinfo", "published", get_published) + { + opt_published = Some(published); + } + ai += 1; + } + + let routing_domain = opt_routing_domain.unwrap_or(RoutingDomain::PublicInternet); + let published = opt_published.unwrap_or(true); + + Ok(routing_table.debug_info_peerinfo(routing_domain, published)) } async fn debug_txtrecord(&self, _args: String) -> VeilidAPIResult { @@ -623,11 +670,21 @@ impl VeilidAPI { // Dump routing table entry let routing_table = self.network_manager()?.routing_table(); - routing_table - .edit_routing_domain(routing_domain) - .set_relay_node(relay_node) - .commit(true) - .await; + match routing_domain { + RoutingDomain::LocalNetwork => { + let mut editor = routing_table.edit_local_network_routing_domain(); + if editor.set_relay_node(relay_node).commit(true).await { + editor.publish(); + } + } + RoutingDomain::PublicInternet => { + let mut editor = routing_table.edit_public_internet_routing_domain(); + if editor.set_relay_node(relay_node).commit(true).await { + editor.publish(); + } + } + } + Ok("Relay changed".to_owned()) } @@ -662,6 +719,13 @@ impl VeilidAPI { Ok(format!("{}\n\n{}\n\n{}\n\n", nodeinfo, peertable, connman)) } + async fn debug_nodeid(&self, _args: String) -> VeilidAPIResult { + // Dump routing table entry + let routing_table = self.network_manager()?.routing_table(); + let nodeid = routing_table.debug_info_nodeid(); + Ok(nodeid) + } + async fn debug_config(&self, args: String) -> VeilidAPIResult { let mut args = args.as_str(); let mut config = self.config()?; @@ -817,7 +881,7 @@ impl VeilidAPI { 0, "debug_contact", "node_ref", - get_node_ref(routing_table), + get_filtered_node_ref(routing_table), )?; let cm = network_manager @@ -852,7 +916,7 @@ impl VeilidAPI { } => Ok(format!( "Destination: {:#?}\nTarget Entry:\n{}\n", &dest, - routing_table.debug_info_entry(target.clone()) + routing_table.debug_info_entry(target.unfiltered()) )), Destination::Relay { relay, @@ -862,7 +926,7 @@ impl VeilidAPI { "Destination: {:#?}\nTarget Entry:\n{}\nRelay Entry:\n{}\n", &dest, routing_table.clone().debug_info_entry(target.clone()), - routing_table.debug_info_entry(relay.clone()) + routing_table.debug_info_entry(relay.unfiltered()) )), Destination::PrivateRoute { private_route: _, @@ -1261,10 +1325,10 @@ impl VeilidAPI { .await .map_err(VeilidAPIError::internal)?; - let out = if success { - "SUCCESS".to_owned() - } else { - "FAILED".to_owned() + let out = match success { + Some(true) => "SUCCESS".to_owned(), + Some(false) => "FAILED".to_owned(), + None => "UNTESTED".to_owned(), }; Ok(out) @@ -1912,49 +1976,63 @@ impl VeilidAPI { /// Get the help text for 'internal debug' commands. pub async fn debug_help(&self, _args: String) -> VeilidAPIResult { - Ok(r#"buckets [dead|reliable] -dialinfo -peerinfo [routingdomain] -entries [dead|reliable] [] -entry -nodeinfo -config [insecure] [configkey [new value]] -txtrecord -keypair -purge -attach -detach -restart network -contact [] -resolve -ping -appmessage -appcall -appreply [#id] -relay [public|local] -punish list - clear -route allocate [ord|*ord] [rel] [] [in|out] - release - publish [full] - unpublish - print - list - import - test -record list - purge [bytes] - create [ []] - open [+] [] - close [] - set [] - get [] [force] - delete - info [] [subkey] - watch [] [ [ []]] - cancel [] [] - inspect [] [ []] -table list + Ok(r#"Node Information: + nodeid - display a node's id(s) + nodeinfo - display detailed information about this node + dialinfo - display the dialinfo in the routing domains of this node + peerinfo [routingdomain] [published|current] - display the full PeerInfo for a routing domain of this node + +Routing: + buckets [dead|reliable] - Display the routing table bucket statistics (default is only non-dead nodes) + entries [dead|reliable] [] - Display the index of nodes in the routing table + entry - Display all the details about a particular node in the routing table + contact [] - Explain what mechanism would be used to contact a particular node + resolve - Search the network for a particular node or private route + relay [public|local] - Change the relay in use for this node + punish list - List all punishments this node has assigned to other nodes / networks + clear - Clear all punishments from this node + route allocate [ord|*ord] [rel] [] [in|out] - Allocate a route + release - Release a route + publish [full] - Publish a route 'blob' that can be imported on another machine + unpublish - Mark a route as 'no longer published' + print - Display details about a route + list - List allocated routes + import - Import a remote route blob generated by another node's 'publish' command. + test - Test an allocated or imported remote route + +Utilities: + config [insecure] [configkey [new value]] - Display or temporarily change the node config + (most values should not be changed this way, careful!) + txtrecord - Generate a TXT record for making this node into a bootstrap node capable of DNS bootstrap + keypair [cryptokind] - Generate and display a random public/private keypair + purge - Throw away the node's routing table, connections, or routes + attach - Attach the node to the network if it is detached + detach - Detach the node from the network if it is attached + restart network - Restart the low level network + +RPC Operations: + ping - Send a 'Status' RPC question to a destination node and display the returned ping status + appmessage - Send an 'App Message' RPC statement to a destination node + appcall - Send a 'App Call' RPC question to a destination node and display the answer + appreply [#id] - Reply to an 'App Call' RPC received by this node + +DHT Operations: + record list - display the dht records in the store + purge [bytes] - clear all dht records optionally down to some total size + create [ []] - create a new dht record + open [+] [] - open an existing dht record + close [] - close an opened/created dht record + set [] - write a value to a dht record subkey + get [] [force] - read a value from a dht record subkey + delete - delete the local copy of a dht record (not from the network) + info [] [subkey] - display information about a dht record or subkey + watch [] [ [ []]] - watch a record for changes + cancel [] [] - cancel a dht record watch + inspect [] [ []] - display a dht record's subkey status + +TableDB Operations: + table list - list the names of all the tables in the TableDB + -------------------------------------------------------------------- is: VLD0:GsgXCRPrzSK6oBNgxhNpm-rTYFd02R0ySx6j9vbQBG4 * also , , , @@ -2002,6 +2080,8 @@ table list if arg == "help" { self.debug_help(rest).await + } else if arg == "nodeid" { + self.debug_nodeid(rest).await } else if arg == "buckets" { self.debug_buckets(rest).await } else if arg == "dialinfo" { @@ -2051,7 +2131,7 @@ table list } else if arg == "table" { self.debug_table(rest).await } else { - Err(VeilidAPIError::generic("Unknown server debug command")) + Err(VeilidAPIError::generic("Unknown debug command")) } }; res @@ -2101,42 +2181,33 @@ table list private_route, ss.unwrap_or(SafetySelection::Unsafe(Sequencing::default())), )) - } else { - let (text, mods) = text - .split_once('/') - .map(|x| (x.0, Some(x.1))) - .unwrap_or((text, None)); - if let Some((first, second)) = text.split_once('@') { - // Relay - let mut relay_nr = get_node_ref(routing_table.clone())(second)?; - let target_nr = get_node_ref(routing_table)(first)?; + } else if let Some((first, second)) = text.split_once('@') { + // Relay + let relay_nr = resolve_filtered_node_ref( + routing_table.clone(), + ss.unwrap_or_default(), + )(second) + .await?; + let target_nr = get_node_ref(routing_table)(first)?; - if let Some(mods) = mods { - relay_nr = get_node_ref_modifiers(relay_nr)(mods)?; - } - - let mut d = Destination::relay(relay_nr, target_nr); - if let Some(ss) = ss { - d = d.with_safety(ss) - } - - Some(d) - } else { - // Direct - let mut target_nr = - resolve_node_ref(routing_table, ss.unwrap_or_default())(text).await?; - - if let Some(mods) = mods { - target_nr = get_node_ref_modifiers(target_nr)(mods)?; - } - - let mut d = Destination::direct(target_nr); - if let Some(ss) = ss { - d = d.with_safety(ss) - } - - Some(d) + let mut d = Destination::relay(relay_nr, target_nr); + if let Some(ss) = ss { + d = d.with_safety(ss) } + + Some(d) + } else { + // Direct + let target_nr = + resolve_filtered_node_ref(routing_table, ss.unwrap_or_default())(text) + .await?; + + let mut d = Destination::direct(target_nr); + if let Some(ss) = ss { + d = d.with_safety(ss) + } + + Some(d) } }) } diff --git a/veilid-core/src/veilid_api/error.rs b/veilid-core/src/veilid_api/error.rs index cf402321..bbe4cf17 100644 --- a/veilid-core/src/veilid_api/error.rs +++ b/veilid-core/src/veilid_api/error.rs @@ -236,6 +236,29 @@ impl VeilidAPIError { pub type VeilidAPIResult = Result; +pub trait OkVeilidAPIResult { + fn ok_try_again(self) -> VeilidAPIResult>; + fn ok_try_again_timeout(self) -> VeilidAPIResult>; +} + +impl OkVeilidAPIResult for VeilidAPIResult { + fn ok_try_again(self) -> VeilidAPIResult> { + match self { + Ok(v) => Ok(Some(v)), + Err(VeilidAPIError::TryAgain { message: _ }) => Ok(None), + Err(e) => Err(e), + } + } + fn ok_try_again_timeout(self) -> VeilidAPIResult> { + match self { + Ok(v) => Ok(Some(v)), + Err(VeilidAPIError::TryAgain { message: _ }) => Ok(None), + Err(VeilidAPIError::Timeout) => Ok(None), + Err(e) => Err(e), + } + } +} + impl From for VeilidAPIError { fn from(e: std::io::Error) -> Self { match e.kind() { diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 35786407..c8f09030 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod api; mod debug; mod error; diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index 0f21d2ae..39303f33 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -28,7 +28,6 @@ pub struct RoutingContextUnlockedInner { pub struct RoutingContext { /// Veilid API handle. api: VeilidAPI, - inner: Arc>, unlocked_inner: Arc, } @@ -50,7 +49,6 @@ impl RoutingContext { Ok(Self { api, - inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { safety_selection: SafetySelection::Safe(SafetySpec { preferred_route: None, @@ -96,7 +94,6 @@ impl RoutingContext { Ok(Self { api: self.api.clone(), - inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { safety_selection }), }) } @@ -109,7 +106,6 @@ impl RoutingContext { Self { api: self.api.clone(), - inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { safety_selection: match self.unlocked_inner.safety_selection { SafetySelection::Unsafe(_) => SafetySelection::Unsafe(sequencing), @@ -129,7 +125,8 @@ impl RoutingContext { self.unlocked_inner.safety_selection } - fn sequencing(&self) -> Sequencing { + /// Get the sequencing used by this routing context + pub fn sequencing(&self) -> Sequencing { match self.unlocked_inner.safety_selection { SafetySelection::Unsafe(sequencing) => sequencing, SafetySelection::Safe(safety_spec) => safety_spec.sequencing, @@ -335,7 +332,7 @@ impl RoutingContext { /// /// Returns `None` if the value was successfully put. /// Returns `Some(data)` if the value put was older than the one available on the network. - #[instrument(target = "veilid_api", level = "debug", ret, err)] + #[instrument(target = "veilid_api", level = "debug", skip(data), fields(data = print_data(&data, Some(64))), ret, err)] pub async fn set_dht_value( &self, key: TypedKey, @@ -344,7 +341,7 @@ impl RoutingContext { writer: Option, ) -> VeilidAPIResult> { event!(target: "veilid_api", Level::DEBUG, - "RoutingContext::set_dht_value(self: {:?}, key: {:?}, subkey: {:?}, data: {:?}, writer: {:?})", self, key, subkey, data, writer); + "RoutingContext::set_dht_value(self: {:?}, key: {:?}, subkey: {:?}, data: len={}, writer: {:?})", self, key, subkey, data.len(), writer); Crypto::validate_crypto_kind(key.kind)?; let storage_manager = self.api.storage_manager()?; diff --git a/veilid-core/src/veilid_api/types/dht/value_data.rs b/veilid-core/src/veilid_api/types/dht/value_data.rs index f89a1856..2e4dca5a 100644 --- a/veilid-core/src/veilid_api/types/dht/value_data.rs +++ b/veilid-core/src/veilid_api/types/dht/value_data.rs @@ -72,7 +72,7 @@ impl fmt::Debug for ValueData { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fmt.debug_struct("ValueData") .field("seq", &self.seq) - .field("data", &print_data(&self.data, None)) + .field("data", &print_data(&self.data, Some(64))) .field("writer", &self.writer) .finish() } diff --git a/veilid-flutter/example/pubspec.lock b/veilid-flutter/example/pubspec.lock index fd3dac35..76c2c073 100644 --- a/veilid-flutter/example/pubspec.lock +++ b/veilid-flutter/example/pubspec.lock @@ -347,10 +347,10 @@ packages: dependency: transitive description: name: quiver - sha256: b1c1ac5ce6688d77f65f3375a9abb9319b3cb32486bdc7a1e0fdf004d7ba4e47 + sha256: ea0b925899e64ecdfbf9c7becb60d5b50e706ade44a85b2363be2a22d88117d2 url: "https://pub.dev" source: hosted - version: "3.2.1" + version: "3.2.2" sky_engine: dependency: transitive description: flutter @@ -462,10 +462,10 @@ packages: dependency: transitive description: name: vm_service - sha256: f652077d0bdf60abe4c1f6377448e8655008eef28f128bc023f7b5e8dfeb48fc + sha256: "5c5f338a667b4c644744b661f309fb8080bb94b18a7e91ef1dbd343bed00ed6d" url: "https://pub.dev" source: hosted - version: "14.2.4" + version: "14.2.5" webdriver: dependency: transitive description: diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index 05ca7683..75e1d35a 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -8,6 +8,8 @@ repository = "https://gitlab.com/veilid/veilid" authors = ["Veilid Team "] license = "MPL-2.0" edition = "2021" +rust-version = "1.81.0" +resolver = "2" [lib] crate-type = ["cdylib", "staticlib", "rlib"] diff --git a/veilid-python/tests/test_dht.py b/veilid-python/tests/test_dht.py index 1436fa40..e0c39ae0 100644 --- a/veilid-python/tests/test_dht.py +++ b/veilid-python/tests/test_dht.py @@ -207,6 +207,8 @@ async def test_open_writer_dht_value(api_connection: veilid.VeilidAPI): await rc.delete_dht_record(key) +# @pytest.mark.skipif(os.getenv("INTEGRATION") != "1", reason="integration test requires two servers running") +@pytest.mark.skip(reason = "don't work yet") @pytest.mark.asyncio async def test_watch_dht_values(): @@ -225,6 +227,9 @@ async def test_watch_dht_values(): # So we can pretend to be a different node and get the watch updates # Normally they would not get sent if the set comes from the same target # as the watch's target + + # XXX: this logic doesn't work because our node still suppresses updates + # XXX: if the value hasn't changed in the local record store rcWatch = await api.new_routing_context() rcSet = await (await api.new_routing_context()).with_safety(veilid.SafetySelection.unsafe()) @@ -396,7 +401,7 @@ async def test_dht_integration_writer_reader(): if len(rr.offline_subkeys) == 0: await rc0.close_dht_record(desc0.key) break - time.sleep(0.1) + time.sleep(1) # read dht records on server 1 print(f'reading {COUNT} records') @@ -433,7 +438,7 @@ async def test_dht_write_read_local(): # Previously COUNT was set to 500, which causes these tests to take # 10s of minutes on slow connections or debug veilid-server builds - COUNT = 10 + COUNT = 100 TEST_DATA = b"ABCD"*1024 TEST_DATA2 = b"ABCD"*4096 diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 6b1255a5..cf01ec78 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Veilid Team "] license = "MPL-2.0" edition = "2021" resolver = "2" +rust-version = "1.81.0" [[bin]] name = "veilid-server" @@ -44,7 +45,7 @@ debug-locks = ["veilid-core/debug-locks"] [dependencies] veilid-core = { path = "../veilid-core", default-features = false } tracing = { version = "^0.1.40", features = ["log", "attributes"] } -tracing-subscriber = { version = "^0.3.18", features = ["env-filter"] } +tracing-subscriber = { version = "^0.3.18", features = ["env-filter", "time"] } tracing-appender = "^0.2.3" tracing-opentelemetry = "^0.24.0" # Buggy: tracing-error = "^0" @@ -83,7 +84,8 @@ stop-token = { version = "^0", default-features = false } sysinfo = { version = "^0.30.13" } wg = { version = "^0.9.1", features = ["future"] } tracing-flame = "0.2.0" -tracing-perfetto = "0.1.1" +time = { version = "0.3.36", features = ["local-offset"] } +chrono = "0.4.38" [target.'cfg(windows)'.dependencies] windows-service = "^0" @@ -94,6 +96,7 @@ daemonize = "^0.5.0" signal-hook = "^0.3.17" signal-hook-async-std = "^0.2.2" nix = "^0.29.0" +tracing-perfetto = "0.1.1" [target.'cfg(target_os = "linux")'.dependencies] tracing-journald = "^0.3.0" diff --git a/veilid-server/src/main.rs b/veilid-server/src/main.rs index e26e47a6..e8ef7970 100644 --- a/veilid-server/src/main.rs +++ b/veilid-server/src/main.rs @@ -89,11 +89,12 @@ pub struct CmdlineArgs { flame: Option, /// Turn on perfetto tracing (experimental) + #[cfg(unix)] #[arg(long, hide = true, value_name = "PATH", num_args=0..=1, require_equals=true, default_missing_value = "")] perfetto: Option, /// Run as an extra daemon on the same machine for testing purposes, specify a number greater than zero to offset the listening ports - #[arg(long)] + #[arg(short('n'), long)] subnode_index: Option, /// Only generate a new keypair and print it @@ -235,6 +236,7 @@ fn main() -> EyreResult<()> { settingsrw.logging.flame.enabled = true; settingsrw.logging.flame.path = flame; } + #[cfg(unix)] if let Some(perfetto) = args.perfetto { let perfetto = if perfetto.is_empty() { Settings::get_default_perfetto_path(settingsrw.testing.subnode_index) @@ -418,13 +420,11 @@ fn main() -> EyreResult<()> { run_veilid_server(settings, server_mode, veilid_logs).await }) - .map(|v| { + .inspect(|_v| { println!("{}", success); - v }) - .map_err(|e| { + .inspect_err(|_e| { println!("{}", failure); - e }); } diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index f6a8aca2..5c4cce3e 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -884,6 +884,7 @@ impl Settings { } /// Determine default perfetto output path + #[cfg(unix)] pub fn get_default_perfetto_path(subnode_index: u16) -> PathBuf { std::env::temp_dir().join(if subnode_index == 0 { "veilid-server.pftrace".to_owned() @@ -892,7 +893,7 @@ impl Settings { }) } - #[allow(dead_code)] + #[cfg_attr(windows, expect(dead_code))] fn get_or_create_private_directory>(path: P, group_read: bool) -> bool { let path = path.as_ref(); if !path.is_dir() @@ -904,7 +905,7 @@ impl Settings { true } - #[allow(dead_code)] + #[cfg_attr(windows, expect(dead_code))] fn get_default_directory(subpath: &str) -> PathBuf { #[cfg(unix)] { diff --git a/veilid-server/src/veilid_logs.rs b/veilid-server/src/veilid_logs.rs index 5d9db470..a0e912e5 100644 --- a/veilid-server/src/veilid_logs.rs +++ b/veilid-server/src/veilid_logs.rs @@ -6,7 +6,6 @@ use console_subscriber::ConsoleLayer; cfg_if::cfg_if! { if #[cfg(feature = "opentelemetry-otlp")] { - use opentelemetry::*; use opentelemetry_sdk::*; use opentelemetry_otlp::WithExportConfig; } @@ -18,6 +17,7 @@ use std::path::*; use std::sync::Arc; use tracing_appender::*; use tracing_flame::FlameLayer; +#[cfg(unix)] use tracing_perfetto::PerfettoLayer; use tracing_subscriber::prelude::*; use tracing_subscriber::*; @@ -63,12 +63,27 @@ impl VeilidLogs { // Terminal logger if settingsr.logging.terminal.enabled { + let timer = time::format_description::parse("[hour]:[minute]:[second]") + .expect("invalid time format"); + + // Get time offset for local timezone from UTC + // let time_offset = + // time::UtcOffset::current_local_offset().unwrap_or(time::UtcOffset::UTC); + // nerd fight: https://www.reddit.com/r/learnrust/comments/1bgc4p7/time_crate_never_manages_to_get_local_time/ + // Use chrono instead of time crate to get local offset + let offset_in_sec = chrono::Local::now().offset().local_minus_utc(); + let time_offset = + time::UtcOffset::from_whole_seconds(offset_in_sec).expect("invalid utc offset"); + let timer = fmt::time::OffsetTime::new(time_offset, timer); + let filter = veilid_core::VeilidLayerFilter::new( convert_loglevel(settingsr.logging.terminal.level), &settingsr.logging.terminal.ignore_log_targets, ); let layer = fmt::Layer::new() .compact() + .with_timer(timer) + .with_ansi(true) .with_writer(std::io::stdout) .with_filter(filter.clone()); filters.insert("terminal", filter); @@ -96,6 +111,7 @@ impl VeilidLogs { } // Perfetto logger + #[cfg(unix)] if settingsr.logging.perfetto.enabled { let filter = veilid_core::VeilidLayerFilter::new_no_default( veilid_core::VeilidConfigLogLevel::Trace, @@ -140,7 +156,7 @@ impl VeilidLogs { .tracing() .with_exporter(exporter) .with_trace_config(opentelemetry_sdk::trace::Config::default().with_resource( - Resource::new(vec![KeyValue::new( + Resource::new(vec![opentelemetry::KeyValue::new( opentelemetry_semantic_conventions::resource::SERVICE_NAME, format!( "veilid_server:{}", diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index 7cf86a47..41d726a9 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -8,6 +8,8 @@ repository = "https://gitlab.com/veilid/veilid" authors = ["Veilid Team "] license = "MPL-2.0" edition = "2021" +rust-version = "1.81.0" +resolver = "2" [lib] # staticlib for iOS tests, cydlib for android tests, rlib for everything else diff --git a/veilid-tools/src/network_interfaces/mod.rs b/veilid-tools/src/network_interfaces/mod.rs index 957f1968..2fce46b7 100644 --- a/veilid-tools/src/network_interfaces/mod.rs +++ b/veilid-tools/src/network_interfaces/mod.rs @@ -1,20 +1,22 @@ -mod apple; -mod netlink; -mod openbsd; -mod sockaddr_tools; mod tools; -mod windows; use crate::*; cfg_if::cfg_if! { if #[cfg(any(target_os = "linux", target_os = "android"))] { + mod netlink; use self::netlink::PlatformSupportNetlink as PlatformSupport; } else if #[cfg(target_os = "windows")] { + mod windows; + mod sockaddr_tools; use self::windows::PlatformSupportWindows as PlatformSupport; } else if #[cfg(any(target_os = "macos", target_os = "ios"))] { + mod apple; + mod sockaddr_tools; use self::apple::PlatformSupportApple as PlatformSupport; } else if #[cfg(target_os = "openbsd")] { + mod openbsd; + mod sockaddr_tools; use self::openbsd::PlatformSupportOpenBSD as PlatformSupport; } else { compile_error!("No network interfaces support for this platform!"); @@ -27,7 +29,6 @@ pub enum IfAddr { V6(Ifv6Addr), } -#[allow(dead_code)] impl IfAddr { pub fn ip(&self) -> IpAddr { match *self { @@ -197,7 +198,6 @@ impl PartialOrd for InterfaceAddress { } } -#[allow(dead_code)] impl InterfaceAddress { pub fn new(if_addr: IfAddr, flags: AddressFlags) -> Self { Self { if_addr, flags } @@ -248,7 +248,6 @@ impl fmt::Debug for NetworkInterface { Ok(()) } } -#[allow(dead_code)] impl NetworkInterface { pub fn new(name: String, flags: InterfaceFlags) -> Self { Self { diff --git a/veilid-tools/src/network_interfaces/sockaddr_tools.rs b/veilid-tools/src/network_interfaces/sockaddr_tools.rs index 3b61a985..91451714 100644 --- a/veilid-tools/src/network_interfaces/sockaddr_tools.rs +++ b/veilid-tools/src/network_interfaces/sockaddr_tools.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] // Copyright 2018 MaidSafe.net limited. // // This SAFE Network Software is licensed to you under the MIT license "] license = "MPL-2.0" edition = "2021" +rust-version = "1.81.0" +resolver = "2" [lib] crate-type = ["cdylib", "rlib"]