Merge branch 'dht-consensus' into 'main'

improve dht consensus checking and low level networking

See merge request veilid/veilid!318

CHANGELOG.md

@@ -1,4 +1,4 @@
-**Changed in Veilid 0.3.3**
+**Changed in Veilid 0.3.4**
 - Crates updates
   - Update crates to newer versions
   - Remove veilid-async-tungstenite and veilid-async-tls crates as they are no longer needed

Cargo.lock

@@ -5886,6 +5886,7 @@ dependencies = [
  "sharded-slab",
  "smallvec",
  "thread_local",
+ "time",
  "tracing",
  "tracing-core",
  "tracing-log 0.2.0",
@@ -6328,6 +6329,7 @@ dependencies = [
  "async-tungstenite 0.27.0",
  "backtrace",
  "cfg-if 1.0.0",
+ "chrono",
  "clap 4.5.15",
  "color-eyre",
  "config 0.14.0",
@@ -6355,6 +6357,7 @@ dependencies = [
  "signal-hook-async-std",
  "stop-token",
  "sysinfo",
+ "time",
  "tokio",
  "tokio-stream",
  "tokio-util",

Earthfile

@@ -15,10 +15,11 @@ VERSION 0.7
 # Ensure we are using an amd64 platform because some of these targets use cross-platform tooling
 FROM ubuntu:18.04
 
-ENV ZIG_VERSION=0.13.0-dev.46+3648d7df1
+ENV ZIG_VERSION=0.13.0
 ENV CMAKE_VERSION_MINOR=3.30
 ENV CMAKE_VERSION_PATCH=3.30.1
 ENV WASM_BINDGEN_CLI_VERSION=0.2.93
+ENV RUST_VERSION=1.81.0
 ENV RUSTUP_HOME=/usr/local/rustup
 ENV RUSTUP_DIST_SERVER=https://static.rust-lang.org
 ENV CARGO_HOME=/usr/local/cargo
@@ -40,7 +41,7 @@ deps-base:
 # Install Rust
 deps-rust:
     FROM +deps-base
-    RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal
+    RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain=$RUST_VERSION -y -c clippy --no-modify-path --profile minimal
     RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
         rustup --version; \
         cargo --version; \
@@ -60,7 +61,7 @@ deps-rust:
     # Caching tool
     RUN cargo install cargo-chef
     # Install Linux cross-platform tooling
-    RUN curl -O https://ziglang.org/builds/zig-linux-$(arch)-$ZIG_VERSION.tar.xz
+    RUN curl -O https://ziglang.org/download/$ZIG_VERSION/zig-linux-$(arch)-$ZIG_VERSION.tar.xz
     RUN tar -C /usr/local -xJf zig-linux-$(arch)-$ZIG_VERSION.tar.xz
     RUN mv /usr/local/zig-linux-$(arch)-$ZIG_VERSION /usr/local/zig
     RUN cargo install cargo-zigbuild

veilid-cli/Cargo.toml

@@ -9,6 +9,7 @@ authors = ["Veilid Team <contact@veilid.com>"]
 edition = "2021"
 license = "MPL-2.0"
 resolver = "2"
+rust-version = "1.81.0"
 
 [[bin]]
 name = "veilid-cli"

veilid-cli/src/cached_text_view.rs

@@ -62,8 +62,6 @@ pub struct TextContent {
     content: Arc<Mutex<TextContentInner>>,
 }
 
-#[allow(dead_code)]
-
 impl TextContent {
     /// Creates a new text content around the given value.
     ///
@@ -129,6 +127,7 @@ impl TextContent {
     }
 
     /// Remove lines from the end until we have no more than 'count' from the beginning
+    #[expect(dead_code)]
     pub fn resize_front(&self, count: usize) {
         if self.get_content().len() <= count {
             return;
@@ -238,7 +237,6 @@ pub struct CachedTextView {
     width: Option<usize>,
 }
 
-#[allow(dead_code)]
 impl CachedTextView {
     /// Creates a new TextView with the given content.
     pub fn new<S>(content: S, cache_size: usize, max_lines: Option<usize>) -> Self
@@ -281,6 +279,7 @@ impl CachedTextView {
     }
 
     /// Creates a new empty `TextView`.
+    #[expect(dead_code)]
     pub fn empty(cache_size: usize, max_lines: Option<usize>) -> Self {
         CachedTextView::new(ContentType::default(), cache_size, max_lines)
     }
@@ -295,6 +294,7 @@ impl CachedTextView {
     ///
     /// Chainable variant.
     #[must_use]
+    #[expect(dead_code)]
     pub fn style<S: Into<StyleType>>(self, style: S) -> Self {
         self.with(|s| s.set_style(style))
     }
@@ -303,6 +303,7 @@ impl CachedTextView {
     ///
     /// This may be useful if you want horizontal scrolling.
     #[must_use]
+    #[expect(dead_code)]
     pub fn no_wrap(self) -> Self {
         self.with(|s| s.set_content_wrap(false))
     }
@@ -317,6 +318,7 @@ impl CachedTextView {
 
     /// Sets the horizontal alignment for this view.
     #[must_use]
+    #[expect(dead_code)]
     pub fn h_align(mut self, h: HAlign) -> Self {
         self.align.h = h;
 
@@ -325,6 +327,7 @@ impl CachedTextView {
 
     /// Sets the vertical alignment for this view.
     #[must_use]
+    #[expect(dead_code)]
     pub fn v_align(mut self, v: VAlign) -> Self {
         self.align.v = v;
 
@@ -333,6 +336,7 @@ impl CachedTextView {
 
     /// Sets the alignment for this view.
     #[must_use]
+    #[expect(dead_code)]
     pub fn align(mut self, a: Align) -> Self {
         self.align = a;
 
@@ -341,6 +345,7 @@ impl CachedTextView {
 
     /// Center the text horizontally and vertically inside the view.
     #[must_use]
+    #[expect(dead_code)]
     pub fn center(mut self) -> Self {
         self.align = Align::center();
         self
@@ -350,6 +355,7 @@ impl CachedTextView {
     ///
     /// Chainable variant.
     #[must_use]
+    #[expect(dead_code)]
     pub fn content<S>(self, content: S) -> Self
     where
         S: Into<ContentType>,
@@ -392,11 +398,13 @@ impl CachedTextView {
     }
 
     /// Returns the current text in this view.
+    #[cfg_attr(not(test), expect(dead_code))]
     pub fn get_content(&self) -> TextContentRef {
         TextContentInner::get_content(&self.content.content)
     }
 
     /// Returns a shared reference to the content, allowing content mutation.
+    #[expect(dead_code)]
     pub fn get_shared_content(&mut self) -> TextContent {
         // We take &mut here without really needing it,
         // because it sort of "makes sense".

veilid-cli/src/interactive_ui.rs

@@ -103,9 +103,15 @@ impl InteractiveUI {
                 println!("Error: {:?}", e);
                 break;
             }
+
             match readline.readline().timeout_at(done.clone()).await {
                 Ok(Ok(ReadlineEvent::Line(line))) => {
                     let line = line.trim();
+
+                    if !line.is_empty() {
+                        readline.add_history_entry(line.to_string());
+                    }
+
                     if line == "clear" {
                         if let Err(e) = readline.clear() {
                             println!("Error: {:?}", e);
@@ -183,7 +189,6 @@ impl InteractiveUI {
                             self.inner.lock().log_enabled = false;
                         }
                     } else if !line.is_empty() {
-                        readline.add_history_entry(line.to_string());
                         let cmdproc = self.inner.lock().cmdproc.clone();
                         if let Some(cmdproc) = &cmdproc {
                             if let Err(e) = cmdproc.run_command(

veilid-cli/src/main.rs

@@ -36,7 +36,7 @@ struct CmdlineArgs {
     #[arg(long, short = 'p')]
     ipc_path: Option<PathBuf>,
     /// Subnode index to use when connecting
-    #[arg(long, default_value = "0")]
+    #[arg(short('n'), long, default_value = "0")]
     subnode_index: usize,
     /// Address to connect to
     #[arg(long, short = 'a')]

veilid-cli/src/settings.rs

@@ -229,7 +229,7 @@ pub struct Settings {
 }
 
 impl Settings {
-    #[allow(dead_code)]
+    #[cfg_attr(windows, expect(dead_code))]
     fn get_server_default_directory(subpath: &str) -> PathBuf {
         #[cfg(unix)]
         {

veilid-core/Cargo.toml

@@ -10,6 +10,7 @@ edition = "2021"
 build = "build.rs"
 license = "MPL-2.0"
 resolver = "2"
+rust-version = "1.81.0"
 
 [lib]
 crate-type = ["cdylib", "staticlib", "rlib"]

veilid-core/src/crypto/envelope.rs

@@ -1,4 +1,3 @@
-#![allow(dead_code)]
 #![allow(clippy::absurd_extreme_comparisons)]
 use super::*;
 use crate::*;

veilid-core/src/crypto/receipt.rs

@@ -1,4 +1,3 @@
-#![allow(dead_code)]
 #![allow(clippy::absurd_extreme_comparisons)]
 use super::*;
 use crate::*;

veilid-core/src/intf/native/system.rs

@@ -1,8 +1,8 @@
-#![allow(dead_code)]
+use super::*;
 
-use crate::*;
+pub async fn get_outbound_relay_peer(
-
+    _routing_domain: routing_table::RoutingDomain,
-pub async fn get_outbound_relay_peer() -> Option<crate::routing_table::PeerInfo> {
+) -> Option<Arc<routing_table::PeerInfo>> {
     panic!("Native Veilid should never require an outbound relay");
 }
 

veilid-core/src/intf/wasm/system.rs

@@ -1,8 +1,10 @@
-use crate::*;
+use super::*;
 
 //use js_sys::*;
 
-pub async fn get_outbound_relay_peer() -> Option<crate::routing_table::PeerInfo> {
+pub async fn get_outbound_relay_peer(
+    _routing_domain: routing_table::RoutingDomain,
+) -> Option<Arc<routing_table::PeerInfo>> {
     // unimplemented!
     None
 }

veilid-core/src/logging/facilities.rs

@@ -1,4 +1,4 @@
-pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 28] = [
+pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 29] = [
     "mio",
     "h2",
     "hyper",
@@ -27,6 +27,7 @@ pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 28] = [
     "dht",
     "fanout",
     "receipt",
+    "rpc_message",
 ];
 
 pub static FLAME_LOG_FACILITIES_IGNORE_LIST: [&str; 22] = [
@@ -425,3 +426,31 @@ macro_rules! log_crypto {
         trace!(target:"crypto", $fmt, $($arg),+);
     }
 }
+
+#[macro_export]
+macro_rules! log_rpc_message {
+    (error $text:expr) => { error!(
+        target: "rpc_message",
+        "{}",
+        $text,
+    )};
+    (error $fmt:literal, $($arg:expr),+) => {
+        error!(target:"crypto", $fmt, $($arg),+);
+    };
+    (warn $text:expr) => { warn!(
+        target: "rpc_message",
+        "{}",
+        $text,
+    )};
+    (warn $fmt:literal, $($arg:expr),+) => {
+        warn!(target:"crypto", $fmt, $($arg),+);
+    };
+    ($text:expr) => {trace!(
+        target: "rpc_message",
+        "{}",
+        $text,
+    )};
+    ($fmt:literal, $($arg:expr),+) => {
+        trace!(target:"rpc_message", $fmt, $($arg),+);
+    }
+}

veilid-core/src/network_manager/address_filter.rs

@@ -158,7 +158,7 @@ impl AddressFilter {
                 }
             }
             for key in dead_keys {
-                log_net!(debug ">>> FORGIVING: {}", key);
+                warn!("Forgiving: {}", key);
                 inner.punishments_by_ip4.remove(&key);
             }
         }
@@ -174,7 +174,7 @@ impl AddressFilter {
                 }
             }
             for key in dead_keys {
-                log_net!(debug ">>> FORGIVING: {}", key);
+                warn!("Forgiving: {}", key);
                 inner.punishments_by_ip6_prefix.remove(&key);
             }
         }
@@ -190,7 +190,7 @@ impl AddressFilter {
                 }
             }
             for key in dead_keys {
-                log_net!(debug ">>> FORGIVING: {}", key);
+                warn!("Forgiving: {}", key);
                 inner.punishments_by_node_id.remove(&key);
                 // make the entry alive again if it's still here
                 if let Ok(Some(nr)) = self.unlocked_inner.routing_table.lookup_node_ref(key) {
@@ -210,7 +210,7 @@ impl AddressFilter {
                 }
             }
             for key in dead_keys {
-                log_net!(debug ">>> DIALINFO PERMIT: {}", key);
+                log_net!(debug "DialInfo Permit: {}", key);
                 inner.dial_info_failures.remove(&key);
             }
         }
@@ -259,10 +259,10 @@ impl AddressFilter {
 
         let mut inner = self.inner.lock();
         if inner.dial_info_failures.len() >= MAX_DIAL_INFO_FAILURES {
-            log_net!(debug ">>> DIALINFO FAILURE TABLE FULL: {}", dial_info);
+            warn!("DialInfo failure table full: {}", dial_info);
             return;
         }
-        log_net!(debug ">>> DIALINFO FAILURE: {:?}", dial_info);
+        log_net!(debug "DialInfo failure: {:?}", dial_info);
         inner
             .dial_info_failures
             .entry(dial_info)
@@ -279,7 +279,7 @@ impl AddressFilter {
     }
 
     pub fn punish_ip_addr(&self, addr: IpAddr, reason: PunishmentReason) {
-        log_net!(debug ">>> PUNISHED: {} for {:?}", addr, reason);
+        warn!("Punished: {} for {:?}", addr, reason);
         let timestamp = Timestamp::now();
         let punishment = Punishment { reason, timestamp };
 
@@ -326,10 +326,10 @@ impl AddressFilter {
 
         let mut inner = self.inner.lock();
         if inner.punishments_by_node_id.len() >= MAX_PUNISHMENTS_BY_NODE_ID {
-            log_net!(debug ">>> PUNISHMENT TABLE FULL: {}", node_id);
+            warn!("Punishment table full: {}", node_id);
             return;
         }
-        log_net!(debug ">>> PUNISHED: {} for {:?}", node_id, reason);
+        warn!("Punished: {} for {:?}", node_id, reason);
         inner
             .punishments_by_node_id
             .entry(node_id)
@@ -372,7 +372,7 @@ impl AddressFilter {
                 let cnt = inner.conn_count_by_ip4.entry(v4).or_default();
                 assert!(*cnt <= self.unlocked_inner.max_connections_per_ip4);
                 if *cnt == self.unlocked_inner.max_connections_per_ip4 {
-                    warn!("address filter count exceeded: {:?}", v4);
+                    warn!("Address filter count exceeded: {:?}", v4);
                     return Err(AddressFilterError::CountExceeded);
                 }
                 // See if this ip block has connected too frequently
@@ -383,7 +383,7 @@ impl AddressFilter {
                 });
                 assert!(tstamps.len() <= self.unlocked_inner.max_connection_frequency_per_min);
                 if tstamps.len() == self.unlocked_inner.max_connection_frequency_per_min {
-                    warn!("address filter rate exceeded: {:?}", v4);
+                    warn!("Address filter rate exceeded: {:?}", v4);
                     return Err(AddressFilterError::RateExceeded);
                 }
 
@@ -396,14 +396,14 @@ impl AddressFilter {
                 let cnt = inner.conn_count_by_ip6_prefix.entry(v6).or_default();
                 assert!(*cnt <= self.unlocked_inner.max_connections_per_ip6_prefix);
                 if *cnt == self.unlocked_inner.max_connections_per_ip6_prefix {
-                    warn!("address filter count exceeded: {:?}", v6);
+                    warn!("Address filter count exceeded: {:?}", v6);
                     return Err(AddressFilterError::CountExceeded);
                 }
                 // See if this ip block has connected too frequently
                 let tstamps = inner.conn_timestamps_by_ip6_prefix.entry(v6).or_default();
                 assert!(tstamps.len() <= self.unlocked_inner.max_connection_frequency_per_min);
                 if tstamps.len() == self.unlocked_inner.max_connection_frequency_per_min {
-                    warn!("address filter rate exceeded: {:?}", v6);
+                    warn!("Address filter rate exceeded: {:?}", v6);
                     return Err(AddressFilterError::RateExceeded);
                 }
 

veilid-core/src/network_manager/connection_handle.rs

@@ -26,12 +26,12 @@ impl ConnectionHandle {
         }
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn connection_id(&self) -> NetworkConnectionId {
         self.connection_id
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn flow(&self) -> Flow {
         self.flow
     }

veilid-core/src/network_manager/connection_manager.rs

@@ -9,7 +9,6 @@ use stop_token::future::FutureExt;
 
 #[derive(Debug)]
 enum ConnectionManagerEvent {
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
     Accepted(ProtocolNetworkConnection),
     Dead(NetworkConnection),
 }
@@ -45,6 +44,7 @@ struct ConnectionManagerInner {
     sender: flume::Sender<ConnectionManagerEvent>,
     async_processor_jh: Option<MustJoinHandle<()>>,
     stop_source: Option<StopSource>,
+    protected_addresses: HashMap<SocketAddress, NodeRef>,
 }
 
 struct ConnectionManagerArc {
@@ -80,6 +80,7 @@ impl ConnectionManager {
             stop_source: Some(stop_source),
             sender,
             async_processor_jh: Some(async_processor_jh),
+            protected_addresses: HashMap::new(),
         }
     }
     fn new_arc(network_manager: NetworkManager) -> ConnectionManagerArc {
@@ -182,25 +183,61 @@ impl ConnectionManager {
 
     // Internal routine to see if we should keep this connection
     // from being LRU removed. Used on our initiated relay connections.
-    fn should_protect_connection(&self, conn: &NetworkConnection) -> Option<NodeRef> {
+    fn should_protect_connection(
-        let netman = self.network_manager();
+        &self,
-        let routing_table = netman.routing_table();
+        inner: &mut ConnectionManagerInner,
-        let remote_address = conn.flow().remote_address().address();
+        conn: &NetworkConnection,
-        let routing_domain = routing_table.routing_domain_for_address(remote_address)?;
+    ) -> Option<NodeRef> {
-        let relay_node = routing_table.relay_node(routing_domain)?;
+        inner
-        let relay_nr = relay_node.filtered_clone(
+            .protected_addresses
-            NodeRefFilter::new()
+            .get(conn.flow().remote_address())
-                .with_routing_domain(routing_domain)
+            .cloned()
-                .with_address_type(conn.flow().address_type())
+    }
-                .with_protocol_type(conn.flow().protocol_type()),
+
-        );
+    // Update connection protections if things change, like a node becomes a relay
-        let dids = relay_nr.all_filtered_dial_info_details();
+    pub fn update_protections(&self) {
-        for did in dids {
+        let Ok(_guard) = self.arc.startup_lock.enter() else {
-            if did.dial_info.address() == remote_address {
+            return;
-                return Some(relay_nr);
+        };
+
+        let mut lock = self.arc.inner.lock();
+        let Some(inner) = lock.as_mut() else {
+            return;
+        };
+
+        // Get addresses for relays in all routing domains
+        inner.protected_addresses.clear();
+        for routing_domain in RoutingDomainSet::all() {
+            let Some(relay_node) = self
+                .network_manager()
+                .routing_table()
+                .relay_node(routing_domain)
+            else {
+                continue;
+            };
+            for did in relay_node.dial_info_details() {
+                // SocketAddress are distinct per routing domain, so they should not collide
+                // and two nodes should never have the same SocketAddress
+                inner
+                    .protected_addresses
+                    .insert(did.dial_info.socket_address(), relay_node.unfiltered());
             }
         }
-        None
+
+        self.arc
+            .connection_table
+            .with_all_connections_mut(|conn| {
+                if let Some(protect_nr) = conn.protected_node_ref() {
+                    if self.should_protect_connection(inner, conn).is_none() {
+                        log_net!(debug "== Unprotecting connection: {} -> {} for node {}", conn.connection_id(), conn.debug_print(Timestamp::now()), protect_nr);
+                        conn.unprotect();
+                    }
+                } else if let Some(protect_nr) = self.should_protect_connection(inner, conn) {
+                    log_net!(debug "== Protecting existing connection: {} -> {} for node {}", conn.connection_id(), conn.debug_print(Timestamp::now()), protect_nr);
+                    conn.protect(protect_nr);
+                }
+                Option::<()>::None
+        });
     }
 
     // Internal routine to register new connection atomically.
@@ -231,8 +268,8 @@ impl ConnectionManager {
         let handle = conn.get_handle();
 
         // See if this should be a protected connection
-        if let Some(protect_nr) = self.should_protect_connection(&conn) {
+        if let Some(protect_nr) = self.should_protect_connection(inner, &conn) {
-            log_net!(debug "== PROTECTING connection: {} -> {} for node {}", id, conn.debug_print(Timestamp::now()), protect_nr);
+            log_net!(debug "== Protecting new connection: {} -> {} for node {}", id, conn.debug_print(Timestamp::now()), protect_nr);
             conn.protect(protect_nr);
         }
 
@@ -472,7 +509,7 @@ impl ConnectionManager {
 
     // Called by low-level network when any connection-oriented protocol connection appears
     // either from incoming connections.
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+    #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
     pub(super) async fn on_accepted_protocol_network_connection(
         &self,
         protocol_connection: ProtocolNetworkConnection,

veilid-core/src/network_manager/connection_table.rs

@@ -246,6 +246,58 @@ impl ConnectionTable {
         let _ = inner.conn_by_id[protocol_index].get(&id).unwrap();
     }
 
+    #[expect(dead_code)]
+    pub fn with_connection_by_flow<R, F: FnOnce(&NetworkConnection) -> R>(
+        &self,
+        flow: Flow,
+        closure: F,
+    ) -> Option<R> {
+        if flow.protocol_type() == ProtocolType::UDP {
+            return None;
+        }
+
+        let inner = self.inner.lock();
+
+        let id = *inner.id_by_flow.get(&flow)?;
+        let protocol_index = Self::protocol_to_index(flow.protocol_type());
+        let out = inner.conn_by_id[protocol_index].peek(&id).unwrap();
+        Some(closure(out))
+    }
+
+    #[expect(dead_code)]
+    pub fn with_connection_by_flow_mut<R, F: FnOnce(&mut NetworkConnection) -> R>(
+        &self,
+        flow: Flow,
+        closure: F,
+    ) -> Option<R> {
+        if flow.protocol_type() == ProtocolType::UDP {
+            return None;
+        }
+
+        let mut inner = self.inner.lock();
+
+        let id = *inner.id_by_flow.get(&flow)?;
+        let protocol_index = Self::protocol_to_index(flow.protocol_type());
+        let out = inner.conn_by_id[protocol_index].peek_mut(&id).unwrap();
+        Some(closure(out))
+    }
+
+    pub fn with_all_connections_mut<R, F: FnMut(&mut NetworkConnection) -> Option<R>>(
+        &self,
+        mut closure: F,
+    ) -> Option<R> {
+        let mut inner_lock = self.inner.lock();
+        let inner = &mut *inner_lock;
+        for (id, idx) in inner.protocol_index_by_id.iter() {
+            if let Some(conn) = inner.conn_by_id[*idx].peek_mut(id) {
+                if let Some(out) = closure(conn) {
+                    return Some(out);
+                }
+            }
+        }
+        None
+    }
+
     //#[instrument(level = "trace", skip(self), ret)]
     pub fn ref_connection_by_id(
         &self,
@@ -304,7 +356,7 @@ impl ConnectionTable {
     }
 
     //#[instrument(level = "trace", skip(self), ret)]
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn get_connection_ids_by_remote(&self, remote: PeerAddress) -> Vec<NetworkConnectionId> {
         let inner = self.inner.lock();
         inner

veilid-core/src/network_manager/direct_boot.rs

@@ -34,7 +34,7 @@ impl NetworkManager {
 
     // Direct bootstrap request
     #[instrument(level = "trace", target = "net", err, skip(self))]
-    pub async fn boot_request(&self, dial_info: DialInfo) -> EyreResult<Vec<PeerInfo>> {
+    pub async fn boot_request(&self, dial_info: DialInfo) -> EyreResult<Vec<Arc<PeerInfo>>> {
         let timeout_ms = self.with_config(|c| c.network.rpc.timeout_ms);
         // Send boot magic to requested peer address
         let data = BOOT_MAGIC.to_vec();
@@ -51,6 +51,6 @@ impl NetworkManager {
             deserialize_json(std::str::from_utf8(&out_data).wrap_err("bad utf8 in boot peerinfo")?)
                 .wrap_err("failed to deserialize boot peerinfo")?;
 
-        Ok(bootstrap_peerinfo)
+        Ok(bootstrap_peerinfo.into_iter().map(Arc::new).collect())
     }
 }

veilid-core/src/network_manager/mod.rs

@@ -54,8 +54,9 @@ pub const IPADDR_TABLE_SIZE: usize = 1024;
 pub const IPADDR_MAX_INACTIVE_DURATION_US: TimestampDuration =
     TimestampDuration::new(300_000_000u64); // 5 minutes
 pub const NODE_CONTACT_METHOD_CACHE_SIZE: usize = 1024;
-pub const PUBLIC_ADDRESS_CHANGE_DETECTION_COUNT: usize = 5;
+pub const PUBLIC_ADDRESS_CHANGE_CONSISTENCY_DETECTION_COUNT: usize = 3; // Number of consistent results in the cache during outbound-only to trigger detection
-pub const PUBLIC_ADDRESS_CHECK_CACHE_SIZE: usize = 10;
+pub const PUBLIC_ADDRESS_CHANGE_INCONSISTENCY_DETECTION_COUNT: usize = 7; // Number of inconsistent results in the cache during inbound-capable to trigger detection
+pub const PUBLIC_ADDRESS_CHECK_CACHE_SIZE: usize = 10; // Length of consistent/inconsistent result cache for detection
 pub const PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS: u32 = 60;
 pub const PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US: TimestampDuration =
     TimestampDuration::new(300_000_000u64); // 5 minutes
@@ -64,18 +65,6 @@ pub const PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US: TimestampDuration
 pub const ADDRESS_FILTER_TASK_INTERVAL_SECS: u32 = 60;
 pub const BOOT_MAGIC: &[u8; 4] = b"BOOT";
 
-#[derive(Clone, Debug, Default)]
-pub struct ProtocolConfig {
-    pub outbound: ProtocolTypeSet,
-    pub inbound: ProtocolTypeSet,
-    pub family_global: AddressTypeSet,
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
-    pub family_local: AddressTypeSet,
-    pub public_internet_capabilities: Vec<FourCC>,
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
-    pub local_network_capabilities: Vec<FourCC>,
-}
-
 // Things we get when we start up and go away when we shut down
 // Routing table is not in here because we want it to survive a network shutdown/startup restart
 #[derive(Clone)]
@@ -111,20 +100,20 @@ pub(crate) enum NodeContactMethod {
     /// Contact the node directly
     Direct(DialInfo),
     /// Request via signal the node connect back directly (relay, target)
-    SignalReverse(NodeRef, NodeRef),
+    SignalReverse(FilteredNodeRef, FilteredNodeRef),
     /// Request via signal the node negotiate a hole punch (relay, target)
-    SignalHolePunch(NodeRef, NodeRef),
+    SignalHolePunch(FilteredNodeRef, FilteredNodeRef),
     /// Must use an inbound relay to reach the node
-    InboundRelay(NodeRef),
+    InboundRelay(FilteredNodeRef),
     /// Must use outbound relay to reach the node
-    OutboundRelay(NodeRef),
+    OutboundRelay(FilteredNodeRef),
 }
 #[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Hash)]
 struct NodeContactMethodCacheKey {
     node_ids: TypedKeyGroup,
     own_node_info_ts: Timestamp,
     target_node_info_ts: Timestamp,
-    target_node_ref_filter: Option<NodeRefFilter>,
+    target_node_ref_filter: NodeRefFilter,
     target_node_ref_sequencing: Sequencing,
 }
 
@@ -139,7 +128,7 @@ enum SendDataToExistingFlowResult {
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum StartupDisposition {
     Success,
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+    #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
     BindRetry,
 }
 
@@ -148,9 +137,9 @@ struct NetworkManagerInner {
     stats: NetworkManagerStats,
     client_allowlist: LruCache<TypedKey, ClientAllowlistEntry>,
     node_contact_method_cache: LruCache<NodeContactMethodCacheKey, NodeContactMethod>,
-    public_address_check_cache:
+    public_internet_address_check_cache:
         BTreeMap<PublicAddressCheckCacheKey, LruCache<IpAddr, SocketAddress>>,
-    public_address_inconsistencies_table:
+    public_internet_address_inconsistencies_table:
         BTreeMap<PublicAddressCheckCacheKey, HashMap<IpAddr, Timestamp>>,
 }
 
@@ -169,7 +158,7 @@ struct NetworkManagerUnlockedInner {
     update_callback: RwLock<Option<UpdateCallback>>,
     // Background processes
     rolling_transfers_task: TickTask<EyreReport>,
-    public_address_check_task: TickTask<EyreReport>,
+    public_internet_address_check_task: TickTask<EyreReport>,
     address_filter_task: TickTask<EyreReport>,
     // Network Key
     network_key: Option<SharedSecret>,
@@ -189,8 +178,8 @@ impl NetworkManager {
             stats: NetworkManagerStats::default(),
             client_allowlist: LruCache::new_unbounded(),
             node_contact_method_cache: LruCache::new(NODE_CONTACT_METHOD_CACHE_SIZE),
-            public_address_check_cache: BTreeMap::new(),
+            public_internet_address_check_cache: BTreeMap::new(),
-            public_address_inconsistencies_table: BTreeMap::new(),
+            public_internet_address_inconsistencies_table: BTreeMap::new(),
         }
     }
     fn new_unlocked_inner(
@@ -216,7 +205,7 @@ impl NetworkManager {
                 "rolling_transfers_task",
                 ROLLING_TRANSFERS_INTERVAL_SECS,
             ),
-            public_address_check_task: TickTask::new(
+            public_internet_address_check_task: TickTask::new(
                 "public_address_check_task",
                 PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS,
             ),
@@ -525,6 +514,7 @@ impl NetworkManager {
         log_net!(debug "finished network manager shutdown");
     }
 
+    #[expect(dead_code)]
     pub fn update_client_allowlist(&self, client: TypedKey) {
         let mut inner = self.inner.lock();
         match inner.client_allowlist.entry(client) {
@@ -693,7 +683,7 @@ impl NetworkManager {
     pub async fn handle_in_band_receipt<R: AsRef<[u8]>>(
         &self,
         receipt_data: R,
-        inbound_noderef: NodeRef,
+        inbound_noderef: FilteredNodeRef,
     ) -> NetworkResult<()> {
         let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
             return NetworkResult::service_unavailable("network is not started");
@@ -779,11 +769,8 @@ impl NetworkManager {
                 let rpc = self.rpc_processor();
 
                 // Add the peer info to our routing table
-                let mut peer_nr = match routing_table.register_node_with_peer_info(
+                let mut peer_nr = match routing_table.register_node_with_peer_info(peer_info, false)
-                    RoutingDomain::PublicInternet,
+                {
-                    peer_info,
-                    false,
-                ) {
                     Ok(nr) => nr,
                     Err(e) => {
                         return Ok(NetworkResult::invalid_message(format!(
@@ -808,11 +795,8 @@ impl NetworkManager {
                 let rpc = self.rpc_processor();
 
                 // Add the peer info to our routing table
-                let mut peer_nr = match routing_table.register_node_with_peer_info(
+                let mut peer_nr = match routing_table.register_node_with_peer_info(peer_info, false)
-                    RoutingDomain::PublicInternet,
+                {
-                    peer_info,
-                    false,
-                ) {
                     Ok(nr) => nr,
                     Err(e) => {
                         return Ok(NetworkResult::invalid_message(format!(
@@ -826,9 +810,8 @@ impl NetworkManager {
                 let outbound_nrf = routing_table
                     .get_outbound_node_ref_filter(RoutingDomain::PublicInternet)
                     .with_protocol_type(ProtocolType::UDP);
-                peer_nr.set_filter(Some(outbound_nrf));
+                peer_nr.set_filter(outbound_nrf);
-                let Some(hole_punch_dial_info_detail) = peer_nr.first_filtered_dial_info_detail()
+                let Some(hole_punch_dial_info_detail) = peer_nr.first_dial_info_detail() else {
-                else {
                     return Ok(NetworkResult::no_connection_other(format!(
                         "No hole punch capable dialinfo found for node: {}",
                         peer_nr
@@ -836,10 +819,10 @@ impl NetworkManager {
                 };
 
                 // Now that we picked a specific dialinfo, further restrict the noderef to the specific address type
-                let filter = peer_nr.take_filter().unwrap();
+                let filter = peer_nr.take_filter();
                 let filter =
                     filter.with_address_type(hole_punch_dial_info_detail.dial_info.address_type());
-                peer_nr.set_filter(Some(filter));
+                peer_nr.set_filter(filter);
 
                 // Do our half of the hole punch by sending an empty packet
                 // Both sides will do this and then the receipt will get sent over the punched hole
@@ -912,7 +895,7 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all)]
     pub async fn send_envelope<B: AsRef<[u8]>>(
         &self,
-        node_ref: NodeRef,
+        node_ref: FilteredNodeRef,
         destination_node_ref: Option<NodeRef>,
         body: B,
     ) -> EyreResult<NetworkResult<SendDataMethod>> {
@@ -920,7 +903,7 @@ impl NetworkManager {
             return Ok(NetworkResult::no_connection_other("network is not started"));
         };
 
-        let destination_node_ref = destination_node_ref.as_ref().unwrap_or(&node_ref).clone();
+        let destination_node_ref = destination_node_ref.unwrap_or_else(|| node_ref.unfiltered());
         let best_node_id = destination_node_ref.best_node_id();
 
         // Get node's envelope versions and see if we can send to it
@@ -1110,7 +1093,7 @@ impl NetworkManager {
                     .resolve_node(recipient_id, SafetySelection::Unsafe(Sequencing::default()))
                     .await
                 {
-                    Ok(v) => v,
+                    Ok(v) => v.map(|nr| nr.default_filtered()),
                     Err(e) => {
                         log_net!(debug "failed to resolve recipient node for relay, dropping outbound relayed packet: {}" ,e);
                         return Ok(false);
@@ -1125,7 +1108,7 @@ impl NetworkManager {
                 // should be mutually in each others routing tables. The node needing the relay will be
                 // pinging this node regularly to keep itself in the routing table
                 match routing_table.lookup_node_ref(recipient_id) {
-                    Ok(v) => v,
+                    Ok(v) => v.map(|nr| nr.default_filtered()),
                     Err(e) => {
                         log_net!(debug "failed to look up recipient node for relay, dropping outbound relayed packet: {}" ,e);
                         return Ok(false);
@@ -1182,7 +1165,8 @@ impl NetworkManager {
         };
 
         // Cache the envelope information in the routing table
-        let mut source_noderef = match routing_table.register_node_with_existing_connection(
+        let source_noderef = match routing_table.register_node_with_existing_connection(
+            routing_domain,
             envelope.get_sender_typed_id(),
             flow,
             ts,
@@ -1196,9 +1180,6 @@ impl NetworkManager {
         };
         source_noderef.add_envelope_version(envelope.get_version());
 
-        // Enforce routing domain
-        source_noderef.merge_filter(NodeRefFilter::new().with_routing_domain(routing_domain));
-
         // Pass message to RPC system
         if let Err(e) =
             rpc.enqueue_direct_message(envelope, source_noderef, flow, routing_domain, body)

veilid-core/src/network_manager/native/discovery_context.rs

@@ -136,7 +136,7 @@ impl DiscoveryContext {
     // Ask for a public address check from a particular noderef
     // This is done over the normal port using RPC
     #[instrument(level = "trace", skip(self), ret)]
-    async fn request_public_address(&self, node_ref: NodeRef) -> Option<SocketAddress> {
+    async fn request_public_address(&self, node_ref: FilteredNodeRef) -> Option<SocketAddress> {
         let rpc = self.unlocked_inner.routing_table.rpc_processor();
 
         let res = network_result_value_or_log!(match rpc.rpc_call_status(Destination::direct(node_ref.clone())).await {
@@ -217,7 +217,7 @@ impl DiscoveryContext {
         let nodes = self
             .unlocked_inner
             .routing_table
-            .find_fast_public_nodes_filtered(node_count, filters);
+            .find_fast_non_local_nodes_filtered(routing_domain, node_count, filters);
         if nodes.is_empty() {
             log_net!(debug
                 "no external address detection peers of type {:?}:{:?}",
@@ -232,7 +232,7 @@ impl DiscoveryContext {
 
         let get_public_address_func = |node: NodeRef| {
             let this = self.clone();
-            let node = node.filtered_clone(
+            let node = node.custom_filtered(
                 NodeRefFilter::new()
                     .with_routing_domain(routing_domain)
                     .with_dial_info_filter(dial_info_filter),
@@ -246,7 +246,7 @@ impl DiscoveryContext {
                     return Some(ExternalInfo {
                         dial_info,
                         address,
-                        node,
+                        node: node.unfiltered(),
                     });
                 }
                 None
@@ -308,12 +308,7 @@ impl DiscoveryContext {
     ) -> bool {
         let rpc_processor = self.unlocked_inner.routing_table.rpc_processor();
 
-        // asking for node validation doesn't have to use the dial info filter of the dial info we are validating
-        let mut node_ref = node_ref.clone();
-        node_ref.set_filter(None);
-
         // ask the node to send us a dial info validation receipt
-
         match rpc_processor
             .rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect)
             .await

veilid-core/src/network_manager/native/mod.rs

@@ -1,15 +1,17 @@
 mod discovery_context;
 mod igd_manager;
-mod network_class_discovery;
+mod network_state;
 mod network_tcp;
 mod network_udp;
 mod protocol;
 mod start_protocols;
+mod tasks;
 
 use super::*;
 use crate::routing_table::*;
 use connection_manager::*;
 use discovery_context::*;
+use network_state::*;
 use network_tcp::*;
 use protocol::tcp::RawTcpProtocolHandler;
 use protocol::udp::RawUdpProtocolHandler;
@@ -75,26 +77,19 @@ struct NetworkInner {
     /// set if the network needs to be restarted due to a low level configuration change
     /// such as dhcp release or change of address or interfaces being added or removed
     network_needs_restart: bool,
-    /// the calculated protocol configuration for inbound/outbound protocols
+
-    protocol_config: ProtocolConfig,
-    /// set of statically configured protocols with public dialinfo
-    static_public_dialinfo: ProtocolTypeSet,
     /// join handles for all the low level network background tasks
     join_handles: Vec<MustJoinHandle<()>>,
     /// stop source for shutting down the low level network background tasks
     stop_source: Option<StopSource>,
-    /// does our network have ipv4 on any network?
-    enable_ipv4: bool,
-    /// does our network have ipv6 on the global internet?
-    enable_ipv6_global: bool,
-    /// does our network have ipv6 on the local network?
-    enable_ipv6_local: bool,
     /// set if we need to calculate our public dial info again
     needs_public_dial_info_check: bool,
     /// set if we have yet to clear the network during public dial info checking
     network_already_cleared: bool,
     /// the punishment closure to enax
     public_dial_info_check_punishment: Option<Box<dyn FnOnce() + Send + 'static>>,
+    /// Actual bound addresses per protocol
+    bound_address_per_protocol: BTreeMap<ProtocolType, Vec<SocketAddr>>,
     /// mapping of protocol handlers to accept messages from a set of bound socket addresses
     udp_protocol_handlers: BTreeMap<SocketAddr, RawUdpProtocolHandler>,
     /// outbound udp protocol handler for udpv4
@@ -107,8 +102,10 @@ struct NetworkInner {
     listener_states: BTreeMap<SocketAddr, Arc<RwLock<ListenerState>>>,
     /// Preferred local addresses for protocols/address combinations for outgoing connections
     preferred_local_addresses: BTreeMap<(ProtocolType, AddressType), SocketAddr>,
-    /// The list of stable interface addresses we have last seen
+    /// set of statically configured protocols with public dialinfo
-    stable_interface_addresses_at_startup: Vec<IpAddr>,
+    static_public_dial_info: ProtocolTypeSet,
+    /// Network state
+    network_state: Option<NetworkState>,
 }
 
 struct NetworkUnlockedInner {
@@ -125,6 +122,7 @@ struct NetworkUnlockedInner {
     update_network_class_task: TickTask<EyreReport>,
     network_interfaces_task: TickTask<EyreReport>,
     upnp_task: TickTask<EyreReport>,
+    network_task_lock: AsyncMutex<()>,
 
     // Managers
     igd_manager: igd_manager::IGDManager,
@@ -144,20 +142,17 @@ impl Network {
             needs_public_dial_info_check: false,
             network_already_cleared: false,
             public_dial_info_check_punishment: None,
-            protocol_config: Default::default(),
-            static_public_dialinfo: ProtocolTypeSet::empty(),
             join_handles: Vec::new(),
             stop_source: None,
-            enable_ipv4: false,
+            bound_address_per_protocol: BTreeMap::new(),
-            enable_ipv6_global: false,
-            enable_ipv6_local: false,
             udp_protocol_handlers: BTreeMap::new(),
             default_udpv4_protocol_handler: None,
             default_udpv6_protocol_handler: None,
             tls_acceptor: None,
             listener_states: BTreeMap::new(),
             preferred_local_addresses: BTreeMap::new(),
-            stable_interface_addresses_at_startup: Vec::new(),
+            static_public_dial_info: ProtocolTypeSet::new(),
+            network_state: None,
         }
     }
 
@@ -176,6 +171,7 @@ impl Network {
             update_network_class_task: TickTask::new("update_network_class_task", 1),
             network_interfaces_task: TickTask::new("network_interfaces_task", 1),
             upnp_task: TickTask::new("upnp_task", 1),
+            network_task_lock: AsyncMutex::new(()),
             igd_manager: igd_manager::IGDManager::new(config.clone()),
         }
     }
@@ -195,31 +191,7 @@ impl Network {
             )),
         };
 
-        // Set update network class tick task
+        this.setup_tasks();
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .update_network_class_task
-                .set_routine(move |s, l, t| {
-                    Box::pin(this2.clone().update_network_class_task_routine(s, l, t))
-                });
-        }
-        // Set network interfaces tick task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .network_interfaces_task
-                .set_routine(move |s, l, t| {
-                    Box::pin(this2.clone().network_interfaces_task_routine(s, l, t))
-                });
-        }
-        // Set upnp tick task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .upnp_task
-                .set_routine(move |s, l, t| Box::pin(this2.clone().upnp_task_routine(s, l, t)));
-        }
 
         this
     }
@@ -301,11 +273,11 @@ impl Network {
         inner.join_handles.push(jh);
     }
 
-    fn translate_unspecified_address(&self, from: &SocketAddr) -> Vec<SocketAddr> {
+    fn translate_unspecified_address(&self, from: SocketAddr) -> Vec<SocketAddr> {
         if !from.ip().is_unspecified() {
-            vec![*from]
+            vec![from]
         } else {
-            let addrs = self.get_stable_interface_addresses();
+            let addrs = self.last_network_state().stable_interface_addresses;
             addrs
                 .iter()
                 .filter_map(|a| {
@@ -336,46 +308,6 @@ impl Network {
         inner.preferred_local_addresses.get(&key).copied()
     }
 
-    pub(crate) fn is_stable_interface_address(&self, addr: IpAddr) -> bool {
-        let stable_addrs = self.get_stable_interface_addresses();
-        stable_addrs.contains(&addr)
-    }
-
-    pub(crate) fn get_stable_interface_addresses(&self) -> Vec<IpAddr> {
-        let addrs = self.unlocked_inner.interfaces.stable_addresses();
-        let mut addrs: Vec<IpAddr> = addrs
-            .into_iter()
-            .filter(|addr| {
-                let address = Address::from_ip_addr(*addr);
-                address.is_local() || address.is_global()
-            })
-            .collect();
-        addrs.sort();
-        addrs.dedup();
-        addrs
-    }
-
-    // See if our interface addresses have changed, if so redo public dial info if necessary
-    async fn check_interface_addresses(&self) -> EyreResult<bool> {
-        if !self
-            .unlocked_inner
-            .interfaces
-            .refresh()
-            .await
-            .wrap_err("failed to check network interfaces")?
-        {
-            return Ok(false);
-        }
-
-        let mut inner = self.inner.lock();
-        let new_stable_interface_addresses = self.get_stable_interface_addresses();
-        if new_stable_interface_addresses != inner.stable_interface_addresses_at_startup {
-            inner.network_needs_restart = true;
-        }
-
-        Ok(true)
-    }
-
     ////////////////////////////////////////////////////////////
 
     // Record DialInfo failures
@@ -720,216 +652,155 @@ impl Network {
     /////////////////////////////////////////////////////////////////
 
     pub async fn startup_internal(&self) -> EyreResult<StartupDisposition> {
-        // initialize interfaces
+        // Get the initial network state snapshot
-        self.unlocked_inner.interfaces.refresh().await?;
+        // Caution: this -must- happen first because we use unwrap() in last_network_state()
+        let network_state = self.make_network_state().await?;
 
-        // build the set of networks we should consider for the 'LocalNetwork' routing domain
-        let mut local_networks: HashSet<(IpAddr, IpAddr)> = HashSet::new();
-        self.unlocked_inner
-            .interfaces
-            .with_interfaces(|interfaces| {
-                log_net!(debug "interfaces: {:#?}", interfaces);
-
-                for intf in interfaces.values() {
-                    // Skip networks that we should never encounter
-                    if intf.is_loopback() || !intf.is_running() {
-                        continue;
-                    }
-                    // Add network to local networks table
-                    for addr in &intf.addrs {
-                        let netmask = addr.if_addr().netmask();
-                        let network_ip = ipaddr_apply_netmask(addr.if_addr().ip(), netmask);
-                        local_networks.insert((network_ip, netmask));
-                    }
-                }
-            });
-        let local_networks: Vec<(IpAddr, IpAddr)> = local_networks.into_iter().collect();
-        self.unlocked_inner
-            .routing_table
-            .configure_local_network_routing_domain(local_networks);
-
-        // determine if we have ipv4/ipv6 addresses
         {
             let mut inner = self.inner.lock();
 
-            let stable_interface_addresses = self.get_stable_interface_addresses();
+            // Create the shutdown stopper
-
-            inner.enable_ipv4 = false;
-            for addr in stable_interface_addresses.iter().copied() {
-                if addr.is_ipv4() {
-                    log_net!(debug "enable address {:?} as ipv4", addr);
-                    inner.enable_ipv4 = true;
-                } else if addr.is_ipv6() {
-                    let address = Address::from_ip_addr(addr);
-                    if address.is_global() {
-                        log_net!(debug "enable address {:?} as ipv6 global", address);
-                        inner.enable_ipv6_global = true;
-                    } else if address.is_local() {
-                        log_net!(debug "enable address {:?} as ipv6 local", address);
-                        inner.enable_ipv6_local = true;
-                    }
-                }
-            }
-            inner.stable_interface_addresses_at_startup = stable_interface_addresses;
-        }
-
-        // Build our protocol config to share it with other nodes
-        let protocol_config = {
-            let mut inner = self.inner.lock();
-
-            // Create stop source
             inner.stop_source = Some(StopSource::new());
 
-            // get protocol config
+            // Store the first network state snapshot
-            let protocol_config = {
+            inner.network_state = Some(network_state.clone());
-                let c = self.config.get();
+        }
-                let mut inbound = ProtocolTypeSet::new();
-
-                if c.network.protocol.udp.enabled {
-                    inbound.insert(ProtocolType::UDP);
-                }
-                if c.network.protocol.tcp.listen {
-                    inbound.insert(ProtocolType::TCP);
-                }
-                if c.network.protocol.ws.listen {
-                    inbound.insert(ProtocolType::WS);
-                }
-                if c.network.protocol.wss.listen {
-                    inbound.insert(ProtocolType::WSS);
-                }
-
-                let mut outbound = ProtocolTypeSet::new();
-                if c.network.protocol.udp.enabled {
-                    outbound.insert(ProtocolType::UDP);
-                }
-                if c.network.protocol.tcp.connect {
-                    outbound.insert(ProtocolType::TCP);
-                }
-                if c.network.protocol.ws.connect {
-                    outbound.insert(ProtocolType::WS);
-                }
-                if c.network.protocol.wss.connect {
-                    outbound.insert(ProtocolType::WSS);
-                }
-
-                let mut family_global = AddressTypeSet::new();
-                let mut family_local = AddressTypeSet::new();
-                if inner.enable_ipv4 {
-                    family_global.insert(AddressType::IPV4);
-                    family_local.insert(AddressType::IPV4);
-                }
-                if inner.enable_ipv6_global {
-                    family_global.insert(AddressType::IPV6);
-                }
-                if inner.enable_ipv6_local {
-                    family_local.insert(AddressType::IPV6);
-                }
-
-                // set up the routing table's network config
-                // if we have static public dialinfo, upgrade our network class
-                let public_internet_capabilities = {
-                    PUBLIC_INTERNET_CAPABILITIES
-                        .iter()
-                        .copied()
-                        .filter(|cap| !c.capabilities.disable.contains(cap))
-                        .collect::<Vec<Capability>>()
-                };
-                let local_network_capabilities = {
-                    LOCAL_NETWORK_CAPABILITIES
-                        .iter()
-                        .copied()
-                        .filter(|cap| !c.capabilities.disable.contains(cap))
-                        .collect::<Vec<Capability>>()
-                };
-
-                ProtocolConfig {
-                    outbound,
-                    inbound,
-                    family_global,
-                    family_local,
-                    public_internet_capabilities,
-                    local_network_capabilities,
-                }
-            };
-            inner.protocol_config = protocol_config.clone();
-
-            protocol_config
-        };
 
         // Start editing routing table
         let mut editor_public_internet = self
             .unlocked_inner
             .routing_table
-            .edit_routing_domain(RoutingDomain::PublicInternet);
+            .edit_public_internet_routing_domain();
         let mut editor_local_network = self
             .unlocked_inner
             .routing_table
-            .edit_routing_domain(RoutingDomain::LocalNetwork);
+            .edit_local_network_routing_domain();
 
-        // start listeners
+        // Setup network
-        if protocol_config.inbound.contains(ProtocolType::UDP) {
+        editor_local_network.set_local_networks(network_state.local_networks);
-            let res = self
+        editor_local_network.setup_network(
-                .bind_udp_protocol_handlers(&mut editor_public_internet, &mut editor_local_network)
+            network_state.protocol_config.outbound,
-                .await;
+            network_state.protocol_config.inbound,
-            if !matches!(res, Ok(StartupDisposition::Success)) {
+            network_state.protocol_config.family_local,
-                return res;
+            network_state.protocol_config.local_network_capabilities,
-            }
+        );
-        }
-        if protocol_config.inbound.contains(ProtocolType::WS) {
-            let res = self
-                .start_ws_listeners(&mut editor_public_internet, &mut editor_local_network)
-                .await;
-            if !matches!(res, Ok(StartupDisposition::Success)) {
-                return res;
-            }
-        }
-        if protocol_config.inbound.contains(ProtocolType::WSS) {
-            let res = self
-                .start_wss_listeners(&mut editor_public_internet, &mut editor_local_network)
-                .await;
-            if !matches!(res, Ok(StartupDisposition::Success)) {
-                return res;
-            }
-        }
-        if protocol_config.inbound.contains(ProtocolType::TCP) {
-            let res = self
-                .start_tcp_listeners(&mut editor_public_internet, &mut editor_local_network)
-                .await;
-            if !matches!(res, Ok(StartupDisposition::Success)) {
-                return res;
-            }
-        }
 
         editor_public_internet.setup_network(
-            protocol_config.outbound,
+            network_state.protocol_config.outbound,
-            protocol_config.inbound,
+            network_state.protocol_config.inbound,
-            protocol_config.family_global,
+            network_state.protocol_config.family_global,
-            protocol_config.public_internet_capabilities,
+            network_state.protocol_config.public_internet_capabilities,
-        );
-        editor_local_network.setup_network(
-            protocol_config.outbound,
-            protocol_config.inbound,
-            protocol_config.family_local,
-            protocol_config.local_network_capabilities,
         );
+
+        // Start listeners
+        if network_state
+            .protocol_config
+            .inbound
+            .contains(ProtocolType::UDP)
+        {
+            let res = self.bind_udp_protocol_handlers().await;
+            if !matches!(res, Ok(StartupDisposition::Success)) {
+                return res;
+            }
+        }
+        if network_state
+            .protocol_config
+            .inbound
+            .contains(ProtocolType::WS)
+        {
+            let res = self.start_ws_listeners().await;
+            if !matches!(res, Ok(StartupDisposition::Success)) {
+                return res;
+            }
+        }
+        if network_state
+            .protocol_config
+            .inbound
+            .contains(ProtocolType::WSS)
+        {
+            let res = self.start_wss_listeners().await;
+            if !matches!(res, Ok(StartupDisposition::Success)) {
+                return res;
+            }
+        }
+        if network_state
+            .protocol_config
+            .inbound
+            .contains(ProtocolType::TCP)
+        {
+            let res = self.start_tcp_listeners().await;
+            if !matches!(res, Ok(StartupDisposition::Success)) {
+                return res;
+            }
+        }
+
+        // Register all dialinfo
+        self.register_all_dial_info(&mut editor_public_internet, &mut editor_local_network)
+            .await?;
+
+        // Set network class statically if we have static public dialinfo
         let detect_address_changes = {
             let c = self.config.get();
             c.network.detect_address_changes
         };
         if !detect_address_changes {
             let inner = self.inner.lock();
-            if !inner.static_public_dialinfo.is_empty() {
+            if !inner.static_public_dial_info.is_empty() {
                 editor_public_internet.set_network_class(Some(NetworkClass::InboundCapable));
             }
         }
 
-        // commit routing table edits
+        // Set network class statically for local network routing domain until
-        editor_public_internet.commit(true).await;
+        // we can do some reachability analysis eventually
-        editor_local_network.commit(true).await;
+        editor_local_network.set_network_class(Some(NetworkClass::InboundCapable));
+
+        // Commit routing domain edits
+        if editor_public_internet.commit(true).await {
+            editor_public_internet.publish();
+        }
+        if editor_local_network.commit(true).await {
+            editor_local_network.publish();
+        }
 
         Ok(StartupDisposition::Success)
     }
 
+    #[instrument(level = "debug", err, skip_all)]
+    pub(super) async fn register_all_dial_info(
+        &self,
+        editor_public_internet: &mut RoutingDomainEditorPublicInternet,
+        editor_local_network: &mut RoutingDomainEditorLocalNetwork,
+    ) -> EyreResult<()> {
+        let Some(protocol_config) = ({
+            let inner = self.inner.lock();
+            inner
+                .network_state
+                .as_ref()
+                .map(|ns| ns.protocol_config.clone())
+        }) else {
+            bail!("can't register dial info without network state");
+        };
+
+        if protocol_config.inbound.contains(ProtocolType::UDP) {
+            self.register_udp_dial_info(editor_public_internet, editor_local_network)
+                .await?;
+        }
+        if protocol_config.inbound.contains(ProtocolType::WS) {
+            self.register_ws_dial_info(editor_public_internet, editor_local_network)
+                .await?;
+        }
+        if protocol_config.inbound.contains(ProtocolType::WSS) {
+            self.register_wss_dial_info(editor_public_internet, editor_local_network)
+                .await?;
+        }
+        if protocol_config.inbound.contains(ProtocolType::TCP) {
+            self.register_tcp_dial_info(editor_public_internet, editor_local_network)
+                .await?;
+        }
+
+        Ok(())
+    }
+
     #[instrument(level = "debug", err, skip_all)]
     pub async fn startup(&self) -> EyreResult<StartupDisposition> {
         let guard = self.unlocked_inner.startup_lock.startup()?;
@@ -994,19 +865,13 @@ impl Network {
         log_net!(debug "clearing dial info");
 
         routing_table
-            .edit_routing_domain(RoutingDomain::PublicInternet)
+            .edit_public_internet_routing_domain()
-            .clear_dial_info_details(None, None)
+            .shutdown()
-            .set_network_class(None)
-            .clear_relay_node()
-            .commit(true)
             .await;
 
         routing_table
-            .edit_routing_domain(RoutingDomain::LocalNetwork)
+            .edit_local_network_routing_domain()
-            .clear_dial_info_details(None, None)
+            .shutdown()
-            .set_network_class(None)
-            .clear_relay_node()
-            .commit(true)
             .await;
 
         // Reset state including network class
@@ -1051,87 +916,4 @@ impl Network {
     }
 
     //////////////////////////////////////////
-
-    #[instrument(level = "trace", target = "net", skip_all, err)]
-    async fn network_interfaces_task_routine(
-        self,
-        _stop_token: StopToken,
-        _l: u64,
-        _t: u64,
-    ) -> EyreResult<()> {
-        self.check_interface_addresses().await?;
-
-        Ok(())
-    }
-
-    #[instrument(parent = None, level = "trace", target = "net", skip_all, err)]
-    async fn upnp_task_routine(self, _stop_token: StopToken, _l: u64, _t: u64) -> EyreResult<()> {
-        if !self.unlocked_inner.igd_manager.tick().await? {
-            info!("upnp failed, restarting local network");
-            let mut inner = self.inner.lock();
-            inner.network_needs_restart = true;
-        }
-
-        Ok(())
-    }
-
-    #[instrument(level = "trace", target = "net", name = "Network::tick", skip_all, err)]
-    pub(crate) async fn tick(&self) -> EyreResult<()> {
-        let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
-            log_net!(debug "ignoring due to not started up");
-            return Ok(());
-        };
-
-        let (detect_address_changes, upnp) = {
-            let config = self.network_manager().config();
-            let c = config.get();
-            (c.network.detect_address_changes, c.network.upnp)
-        };
-
-        // If we need to figure out our network class, tick the task for it
-        if detect_address_changes {
-            let public_internet_network_class = self
-                .routing_table()
-                .get_network_class(RoutingDomain::PublicInternet)
-                .unwrap_or(NetworkClass::Invalid);
-            let needs_public_dial_info_check = self.needs_public_dial_info_check();
-            if public_internet_network_class == NetworkClass::Invalid
-                || needs_public_dial_info_check
-            {
-                let routing_table = self.routing_table();
-                let rth = routing_table.get_routing_table_health();
-
-                // We want at least two live entries per crypto kind before we start doing this (bootstrap)
-                let mut has_at_least_two = true;
-                for ck in VALID_CRYPTO_KINDS {
-                    if rth
-                        .live_entry_counts
-                        .get(&(RoutingDomain::PublicInternet, ck))
-                        .copied()
-                        .unwrap_or_default()
-                        < 2
-                    {
-                        has_at_least_two = false;
-                        break;
-                    }
-                }
-
-                if has_at_least_two {
-                    self.unlocked_inner.update_network_class_task.tick().await?;
-                }
-            }
-
-            // Check our network interfaces to see if they have changed
-            if !self.needs_restart() {
-                self.unlocked_inner.network_interfaces_task.tick().await?;
-            }
-        }
-
-        // If we need to tick upnp, do it
-        if upnp && !self.needs_restart() {
-            self.unlocked_inner.upnp_task.tick().await?;
-        }
-
-        Ok(())
-    }
 }

veilid-core/src/network_manager/native/network_state.rs

@@ -0,0 +1,189 @@
+use super::*;
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ProtocolConfig {
+    pub outbound: ProtocolTypeSet,
+    pub inbound: ProtocolTypeSet,
+    pub family_global: AddressTypeSet,
+    pub family_local: AddressTypeSet,
+    pub public_internet_capabilities: Vec<FourCC>,
+    pub local_network_capabilities: Vec<FourCC>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) struct NetworkState {
+    /// the calculated protocol configuration for inbound/outbound protocols
+    pub protocol_config: ProtocolConfig,
+    /// does our network have ipv4 on any network?
+    pub enable_ipv4: bool,
+    /// does our network have ipv6 on the global internet?
+    pub enable_ipv6_global: bool,
+    /// does our network have ipv6 on the local network?
+    pub enable_ipv6_local: bool,
+    /// The list of stable interface addresses we have last seen
+    pub stable_interface_addresses: Vec<IpAddr>,
+    /// The local networks (network+mask) most recently seen
+    pub local_networks: Vec<(IpAddr, IpAddr)>,
+}
+
+impl Network {
+    fn make_stable_interface_addresses(&self) -> Vec<IpAddr> {
+        let addrs = self.unlocked_inner.interfaces.stable_addresses();
+        let mut addrs: Vec<IpAddr> = addrs
+            .into_iter()
+            .filter(|addr| {
+                let address = Address::from_ip_addr(*addr);
+                address.is_local() || address.is_global()
+            })
+            .collect();
+        addrs.sort();
+        addrs.dedup();
+        addrs
+    }
+
+    pub(super) fn last_network_state(&self) -> NetworkState {
+        self.inner.lock().network_state.clone().unwrap()
+    }
+
+    pub(super) fn is_stable_interface_address(&self, addr: IpAddr) -> bool {
+        self.inner
+            .lock()
+            .network_state
+            .as_ref()
+            .unwrap()
+            .stable_interface_addresses
+            .contains(&addr)
+    }
+
+    pub(super) async fn make_network_state(&self) -> EyreResult<NetworkState> {
+        // refresh network interfaces
+        self.unlocked_inner
+            .interfaces
+            .refresh()
+            .await
+            .wrap_err("failed to refresh network interfaces")?;
+
+        // build the set of networks we should consider for the 'LocalNetwork' routing domain
+        let mut local_networks: HashSet<(IpAddr, IpAddr)> = HashSet::new();
+
+        self.unlocked_inner
+            .interfaces
+            .with_interfaces(|interfaces| {
+                for intf in interfaces.values() {
+                    // Skip networks that we should never encounter
+                    if intf.is_loopback() || !intf.is_running() {
+                        continue;
+                    }
+                    // Add network to local networks table
+                    for addr in &intf.addrs {
+                        let netmask = addr.if_addr().netmask();
+                        let network_ip = ipaddr_apply_netmask(addr.if_addr().ip(), netmask);
+                        local_networks.insert((network_ip, netmask));
+                    }
+                }
+            });
+        let mut local_networks: Vec<(IpAddr, IpAddr)> = local_networks.into_iter().collect();
+        local_networks.sort();
+
+        // determine if we have ipv4/ipv6 addresses
+        let mut enable_ipv4 = false;
+        let mut enable_ipv6_global = false;
+        let mut enable_ipv6_local = false;
+
+        let stable_interface_addresses = self.make_stable_interface_addresses();
+
+        for addr in stable_interface_addresses.iter().copied() {
+            if addr.is_ipv4() {
+                enable_ipv4 = true;
+            } else if addr.is_ipv6() {
+                let address = Address::from_ip_addr(addr);
+                if address.is_global() {
+                    enable_ipv6_global = true;
+                } else if address.is_local() {
+                    enable_ipv6_local = true;
+                }
+            }
+        }
+
+        // Get protocol config
+        let protocol_config = {
+            let c = self.config.get();
+            let mut inbound = ProtocolTypeSet::new();
+
+            if c.network.protocol.udp.enabled {
+                inbound.insert(ProtocolType::UDP);
+            }
+            if c.network.protocol.tcp.listen {
+                inbound.insert(ProtocolType::TCP);
+            }
+            if c.network.protocol.ws.listen {
+                inbound.insert(ProtocolType::WS);
+            }
+            if c.network.protocol.wss.listen {
+                inbound.insert(ProtocolType::WSS);
+            }
+
+            let mut outbound = ProtocolTypeSet::new();
+            if c.network.protocol.udp.enabled {
+                outbound.insert(ProtocolType::UDP);
+            }
+            if c.network.protocol.tcp.connect {
+                outbound.insert(ProtocolType::TCP);
+            }
+            if c.network.protocol.ws.connect {
+                outbound.insert(ProtocolType::WS);
+            }
+            if c.network.protocol.wss.connect {
+                outbound.insert(ProtocolType::WSS);
+            }
+
+            let mut family_global = AddressTypeSet::new();
+            let mut family_local = AddressTypeSet::new();
+            if enable_ipv4 {
+                family_global.insert(AddressType::IPV4);
+                family_local.insert(AddressType::IPV4);
+            }
+            if enable_ipv6_global {
+                family_global.insert(AddressType::IPV6);
+            }
+            if enable_ipv6_local {
+                family_local.insert(AddressType::IPV6);
+            }
+
+            // set up the routing table's network config
+            // if we have static public dialinfo, upgrade our network class
+            let public_internet_capabilities = {
+                PUBLIC_INTERNET_CAPABILITIES
+                    .iter()
+                    .copied()
+                    .filter(|cap| !c.capabilities.disable.contains(cap))
+                    .collect::<Vec<Capability>>()
+            };
+            let local_network_capabilities = {
+                LOCAL_NETWORK_CAPABILITIES
+                    .iter()
+                    .copied()
+                    .filter(|cap| !c.capabilities.disable.contains(cap))
+                    .collect::<Vec<Capability>>()
+            };
+
+            ProtocolConfig {
+                outbound,
+                inbound,
+                family_global,
+                family_local,
+                public_internet_capabilities,
+                local_network_capabilities,
+            }
+        };
+
+        Ok(NetworkState {
+            protocol_config,
+            enable_ipv4,
+            enable_ipv6_global,
+            enable_ipv6_local,
+            stable_interface_addresses,
+            local_networks,
+        })
+    }
+}

veilid-core/src/network_manager/native/network_tcp.rs

@@ -359,10 +359,9 @@ impl Network {
         &self,
         bind_set: NetworkBindSet,
         is_tls: bool,
+        protocol_type: ProtocolType,
         new_protocol_accept_handler: Box<NewProtocolAcceptHandler>,
-    ) -> EyreResult<Option<Vec<SocketAddress>>> {
+    ) -> EyreResult<bool> {
-        let mut out = Vec::<SocketAddress>::new();
-
         for ip_addr in bind_set.addrs {
             let mut port = bind_set.port;
             loop {
@@ -407,16 +406,24 @@ impl Network {
                     }
 
                     // Return interface dial infos we listen on
-                    let idi_addrs = self.translate_unspecified_address(&addr);
+                    let mut inner = self.inner.lock();
-                    for idi_addr in idi_addrs {
+                    let bapp = inner
-                        out.push(SocketAddress::from_socket_addr(idi_addr));
+                        .bound_address_per_protocol
-                    }
+                        .entry(protocol_type)
+                        .or_default();
+                    bapp.push(addr);
+
+                    Self::set_preferred_local_address(
+                        &mut inner,
+                        PeerAddress::new(SocketAddress::from_socket_addr(addr), protocol_type),
+                    );
+
                     break;
                 }
 
                 if !bind_set.search {
                     log_net!(debug "unable to bind to tcp {}", addr);
-                    return Ok(None);
+                    return Ok(false);
                 }
 
                 if port == 65535u16 {
@@ -431,6 +438,6 @@ impl Network {
             }
         }
 
-        Ok(Some(out))
+        Ok(true)
     }
 }

veilid-core/src/network_manager/native/network_udp.rs

@@ -155,9 +155,7 @@ impl Network {
     pub(super) async fn create_udp_protocol_handlers(
         &self,
         bind_set: NetworkBindSet,
-    ) -> EyreResult<Option<Vec<DialInfo>>> {
+    ) -> EyreResult<bool> {
-        let mut out = Vec::<DialInfo>::new();
-
         for ip_addr in bind_set.addrs {
             let mut port = bind_set.port;
             loop {
@@ -170,17 +168,27 @@ impl Network {
 
                     // Return interface dial infos we listen on
                     if bound {
-                        let idi_addrs = self.translate_unspecified_address(&addr);
+                        let mut inner = self.inner.lock();
-                        for idi_addr in idi_addrs {
+                        let bapp = inner
-                            out.push(DialInfo::udp_from_socketaddr(idi_addr));
+                            .bound_address_per_protocol
-                        }
+                            .entry(ProtocolType::UDP)
+                            .or_default();
+                        bapp.push(addr);
+
+                        Self::set_preferred_local_address(
+                            &mut inner,
+                            PeerAddress::new(
+                                SocketAddress::from_socket_addr(addr),
+                                ProtocolType::UDP,
+                            ),
+                        );
+
                         break;
                     }
                 }
-
                 if !bind_set.search {
                     log_net!(debug "unable to bind to udp {}", addr);
-                    return Ok(None);
+                    return Ok(false);
                 }
 
                 if port == 65535u16 {
@@ -194,7 +202,7 @@ impl Network {
                 }
             }
         }
-        Ok(Some(out))
+        Ok(true)
     }
 
     /////////////////////////////////////////////////////////////////

veilid-core/src/network_manager/native/start_protocols.rs

@@ -123,28 +123,23 @@ impl Network {
     }
 
     // Add local dial info to preferred local address table
-    fn add_preferred_local_address(inner: &mut NetworkInner, pa: PeerAddress) {
+    pub(super) fn set_preferred_local_address(inner: &mut NetworkInner, pa: PeerAddress) {
         let key = (pa.protocol_type(), pa.address_type());
         let sa = pa.socket_addr();
-        let unspec_sa = match sa {
+        // let unspec_sa = match sa {
-            SocketAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, a.port())),
+        //     SocketAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, a.port())),
-            SocketAddr::V6(a) => {
+        //     SocketAddr::V6(a) => {
-                SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, a.port(), 0, 0))
+        //         SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, a.port(), 0, 0))
-            }
+        //     }
-        };
+        // };
-        inner.preferred_local_addresses.insert(key, unspec_sa);
+        inner.preferred_local_addresses.entry(key).or_insert(sa);
     }
 
     /////////////////////////////////////////////////////
 
     #[instrument(level = "trace", skip_all)]
-    pub(super) async fn bind_udp_protocol_handlers(
+    pub(super) async fn bind_udp_protocol_handlers(&self) -> EyreResult<StartupDisposition> {
-        &self,
-        editor_public_internet: &mut RoutingDomainEditor,
-        editor_local_network: &mut RoutingDomainEditor,
-    ) -> EyreResult<StartupDisposition> {
         log_net!("UDP: binding protocol handlers");
-        let routing_table = self.routing_table();
         let (listen_address, public_address, detect_address_changes) = {
             let c = self.config.get();
             (
@@ -172,18 +167,59 @@ impl Network {
             );
         }
 
-        let Some(mut local_dial_info_list) = self.create_udp_protocol_handlers(bind_set).await?
+        if !self.create_udp_protocol_handlers(bind_set).await? {
-        else {
             return Ok(StartupDisposition::BindRetry);
         };
-        local_dial_info_list.sort();
 
-        let mut static_public = false;
+        // Now create tasks for udp listeners
+        self.create_udp_listener_tasks().await?;
 
-        log_net!(
+        {
-            "UDP: protocol handlers bound to {:#?}",
+            let mut inner = self.inner.lock();
-            local_dial_info_list
+            if public_address.is_some() && !detect_address_changes {
-        );
+                inner.static_public_dial_info.insert(ProtocolType::UDP);
+            }
+        }
+
+        Ok(StartupDisposition::Success)
+    }
+
+    #[instrument(level = "trace", skip_all)]
+    pub(super) async fn register_udp_dial_info(
+        &self,
+        editor_public_internet: &mut RoutingDomainEditorPublicInternet,
+        editor_local_network: &mut RoutingDomainEditorLocalNetwork,
+    ) -> EyreResult<()> {
+        log_net!("UDP: registering dial info");
+
+        let (public_address, detect_address_changes) = {
+            let c = self.config.get();
+            (
+                c.network.protocol.udp.public_address.clone(),
+                c.network.detect_address_changes,
+            )
+        };
+
+        let local_dial_info_list = {
+            let mut out = vec![];
+            if let Some(bound_addresses) = {
+                let inner = self.inner.lock();
+                inner
+                    .bound_address_per_protocol
+                    .get(&ProtocolType::UDP)
+                    .cloned()
+            } {
+                for addr in bound_addresses {
+                    let idi_addrs = self.translate_unspecified_address(addr);
+                    for idi_addr in idi_addrs {
+                        out.push(DialInfo::udp_from_socketaddr(idi_addr));
+                    }
+                }
+            }
+            out.sort();
+            out.dedup();
+            out
+        };
 
         // Add static public dialinfo if it's configured
         if let Some(public_address) = public_address.as_ref() {
@@ -197,24 +233,16 @@ impl Network {
                 let pdi = DialInfo::udp_from_socketaddr(pdi_addr);
 
                 // Register the public address
-                editor_public_internet.register_dial_info(pdi.clone(), DialInfoClass::Direct)?;
+                editor_public_internet.add_dial_info(pdi.clone(), DialInfoClass::Direct);
-                static_public = true;
+                editor_public_internet.set_network_class(Some(NetworkClass::InboundCapable));
 
                 // See if this public address is also a local interface address we haven't registered yet
-                let is_interface_address = (|| {
+                if self.is_stable_interface_address(pdi_addr.ip()) {
-                    for ip_addr in self.get_stable_interface_addresses() {
+                    editor_local_network.add_dial_info(
-                        if pdi_addr.ip() == ip_addr {
-                            return true;
-                        }
-                    }
-                    false
-                })();
-
-                if !local_dial_info_list.contains(&pdi) && is_interface_address {
-                    editor_local_network.register_dial_info(
                         DialInfo::udp_from_socketaddr(pdi_addr),
                         DialInfoClass::Direct,
-                    )?;
+                    );
+                    editor_local_network.set_network_class(Some(NetworkClass::InboundCapable));
                 }
             }
         }
@@ -223,48 +251,26 @@ impl Network {
         for di in &local_dial_info_list {
             // If the local interface address is global, then register global dial info
             // if no other public address is specified
-            if !detect_address_changes
+            if !detect_address_changes && public_address.is_none() && di.address().is_global() {
-                && public_address.is_none()
+                editor_public_internet.add_dial_info(di.clone(), DialInfoClass::Direct);
-                && routing_table.ensure_dial_info_is_valid(RoutingDomain::PublicInternet, di)
+                editor_public_internet.set_network_class(Some(NetworkClass::InboundCapable));
-            {
-                editor_public_internet.register_dial_info(di.clone(), DialInfoClass::Direct)?;
-                static_public = true;
             }
 
             // Register interface dial info as well since the address is on the local interface
-            editor_local_network.register_dial_info(di.clone(), DialInfoClass::Direct)?;
+            editor_local_network.add_dial_info(di.clone(), DialInfoClass::Direct);
         }
 
-        {
+        Ok(())
-            let mut inner = self.inner.lock();
-            if static_public {
-                inner.static_public_dialinfo.insert(ProtocolType::UDP);
-            }
-            for ldi in local_dial_info_list {
-                Self::add_preferred_local_address(&mut inner, ldi.peer_address());
-            }
-        }
-
-        // Now create tasks for udp listeners
-        self.create_udp_listener_tasks().await?;
-
-        Ok(StartupDisposition::Success)
     }
 
     #[instrument(level = "trace", skip_all)]
-    pub(super) async fn start_ws_listeners(
+    pub(super) async fn start_ws_listeners(&self) -> EyreResult<StartupDisposition> {
-        &self,
-        editor_public_internet: &mut RoutingDomainEditor,
-        editor_local_network: &mut RoutingDomainEditor,
-    ) -> EyreResult<StartupDisposition> {
         log_net!("WS: binding protocol handlers");
-        let routing_table = self.routing_table();
+        let (listen_address, url, detect_address_changes) = {
-        let (listen_address, url, path, detect_address_changes) = {
             let c = self.config.get();
             (
                 c.network.protocol.ws.listen_address.clone(),
                 c.network.protocol.ws.url.clone(),
-                c.network.protocol.ws.path.clone(),
                 c.network.detect_address_changes,
             )
         };
@@ -285,21 +291,70 @@ impl Network {
                 bind_set.port, bind_set.addrs
             );
         }
-        let Some(socket_addresses) = self
+        if !self
             .start_tcp_listener(
                 bind_set,
                 false,
+                ProtocolType::WS,
                 Box::new(|c, t| Box::new(WebsocketProtocolHandler::new(c, t))),
             )
             .await?
-        else {
+        {
             return Ok(StartupDisposition::BindRetry);
-        };
+        }
-        log_net!("WS: protocol handlers started on {:#?}", socket_addresses);
+
+        {
+            let mut inner = self.inner.lock();
+            if url.is_some() && !detect_address_changes {
+                inner.static_public_dial_info.insert(ProtocolType::WS);
+            }
+        }
+
+        Ok(StartupDisposition::Success)
+    }
+
+    #[instrument(level = "trace", skip_all)]
+    pub(super) async fn register_ws_dial_info(
+        &self,
+        editor_public_internet: &mut RoutingDomainEditorPublicInternet,
+        editor_local_network: &mut RoutingDomainEditorLocalNetwork,
+    ) -> EyreResult<()> {
+        log_net!("WS: registering dial info");
+        let (url, path, detect_address_changes) = {
+            let c = self.config.get();
+            (
+                c.network.protocol.ws.url.clone(),
+                c.network.protocol.ws.path.clone(),
+                c.network.detect_address_changes,
+            )
+        };
 
-        let mut static_public = false;
         let mut registered_addresses: HashSet<IpAddr> = HashSet::new();
 
+        let socket_addresses = {
+            let mut out = vec![];
+            if let Some(bound_addresses) = {
+                let inner = self.inner.lock();
+                inner
+                    .bound_address_per_protocol
+                    .get(&ProtocolType::WS)
+                    .cloned()
+            } {
+                for addr in bound_addresses {
+                    for idi_addr in self
+                        .translate_unspecified_address(addr)
+                        .into_iter()
+                        .map(SocketAddress::from_socket_addr)
+                    {
+                        out.push(idi_addr);
+                    }
+                }
+            }
+            out.sort();
+            out.dedup();
+            out
+        };
+
         // Add static public dialinfo if it's configured
         if let Some(url) = url.as_ref() {
             let mut split_url = SplitUrl::from_str(url).wrap_err("couldn't split url")?;
@@ -318,14 +373,13 @@ impl Network {
                 let pdi = DialInfo::try_ws(SocketAddress::from_socket_addr(gsa), url.clone())
                     .wrap_err("try_ws failed")?;
 
-                editor_public_internet.register_dial_info(pdi.clone(), DialInfoClass::Direct)?;
+                editor_public_internet.add_dial_info(pdi.clone(), DialInfoClass::Direct);
-                static_public = true;
 
                 // See if this public address is also a local interface address
                 if !registered_addresses.contains(&gsa.ip())
                     && self.is_stable_interface_address(gsa.ip())
                 {
-                    editor_local_network.register_dial_info(pdi, DialInfoClass::Direct)?;
+                    editor_local_network.add_dial_info(pdi, DialInfoClass::Direct);
                 }
 
                 registered_addresses.insert(gsa.ip());
@@ -342,40 +396,23 @@ impl Network {
             let local_di =
                 DialInfo::try_ws(*socket_address, local_url).wrap_err("try_ws failed")?;
 
-            if !detect_address_changes
+            if !detect_address_changes && url.is_none() && local_di.address().is_global() {
-                && url.is_none()
-                && routing_table.ensure_dial_info_is_valid(RoutingDomain::PublicInternet, &local_di)
-            {
                 // Register public dial info
-                editor_public_internet
+                editor_public_internet.add_dial_info(local_di.clone(), DialInfoClass::Direct);
-                    .register_dial_info(local_di.clone(), DialInfoClass::Direct)?;
-                static_public = true;
             }
 
             // Register local dial info
-            editor_local_network.register_dial_info(local_di, DialInfoClass::Direct)?;
+            editor_local_network.add_dial_info(local_di, DialInfoClass::Direct);
         }
 
-        let mut inner = self.inner.lock();
+        Ok(())
-        if static_public {
-            inner.static_public_dialinfo.insert(ProtocolType::WS);
-        }
-        for sa in socket_addresses {
-            Self::add_preferred_local_address(&mut inner, PeerAddress::new(sa, ProtocolType::WS));
-        }
-
-        Ok(StartupDisposition::Success)
     }
 
     #[instrument(level = "trace", skip_all)]
-    pub(super) async fn start_wss_listeners(
+    pub(super) async fn start_wss_listeners(&self) -> EyreResult<StartupDisposition> {
-        &self,
-        editor_public_internet: &mut RoutingDomainEditor,
-        editor_local_network: &mut RoutingDomainEditor,
-    ) -> EyreResult<StartupDisposition> {
         log_net!("WSS: binding protocol handlers");
 
-        let (listen_address, url, _detect_address_changes) = {
+        let (listen_address, url, detect_address_changes) = {
             let c = self.config.get();
             (
                 c.network.protocol.wss.listen_address.clone(),
@@ -401,25 +438,49 @@ impl Network {
             );
         }
 
-        let Some(socket_addresses) = self
+        if !self
             .start_tcp_listener(
                 bind_set,
                 true,
+                ProtocolType::WSS,
                 Box::new(|c, t| Box::new(WebsocketProtocolHandler::new(c, t))),
             )
             .await?
-        else {
+        {
             return Ok(StartupDisposition::BindRetry);
-        };
+        }
 
-        log_net!("WSS: protocol handlers started on {:#?}", socket_addresses);
+        {
+            let mut inner = self.inner.lock();
+            if url.is_some() && !detect_address_changes {
+                inner.static_public_dial_info.insert(ProtocolType::WSS);
+            }
+        }
+
+        Ok(StartupDisposition::Success)
+    }
+
+    #[instrument(level = "trace", skip_all)]
+    pub(super) async fn register_wss_dial_info(
+        &self,
+        editor_public_internet: &mut RoutingDomainEditorPublicInternet,
+        editor_local_network: &mut RoutingDomainEditorLocalNetwork,
+    ) -> EyreResult<()> {
+        log_net!("WSS: registering dialinfo");
+
+        let (url, _detect_address_changes) = {
+            let c = self.config.get();
+            (
+                c.network.protocol.wss.url.clone(),
+                c.network.detect_address_changes,
+            )
+        };
 
         // NOTE: No interface dial info for WSS, as there is no way to connect to a local dialinfo via TLS
         // If the hostname is specified, it is the public dialinfo via the URL. If no hostname
         // is specified, then TLS won't validate, so no local dialinfo is possible.
         // This is not the case with unencrypted websockets, which can be specified solely by an IP address
 
-        let mut static_public = false;
         let mut registered_addresses: HashSet<IpAddr> = HashSet::new();
 
         // Add static public dialinfo if it's configured
@@ -440,14 +501,13 @@ impl Network {
                 let pdi = DialInfo::try_wss(SocketAddress::from_socket_addr(gsa), url.clone())
                     .wrap_err("try_wss failed")?;
 
-                editor_public_internet.register_dial_info(pdi.clone(), DialInfoClass::Direct)?;
+                editor_public_internet.add_dial_info(pdi.clone(), DialInfoClass::Direct);
-                static_public = true;
 
                 // See if this public address is also a local interface address
                 if !registered_addresses.contains(&gsa.ip())
                     && self.is_stable_interface_address(gsa.ip())
                 {
-                    editor_local_network.register_dial_info(pdi, DialInfoClass::Direct)?;
+                    editor_local_network.add_dial_info(pdi, DialInfoClass::Direct);
                 }
 
                 registered_addresses.insert(gsa.ip());
@@ -456,26 +516,13 @@ impl Network {
             bail!("WSS URL must be specified due to TLS requirements");
         }
 
-        let mut inner = self.inner.lock();
+        Ok(())
-        if static_public {
-            inner.static_public_dialinfo.insert(ProtocolType::WSS);
-        }
-        for sa in socket_addresses {
-            Self::add_preferred_local_address(&mut inner, PeerAddress::new(sa, ProtocolType::WSS));
-        }
-
-        Ok(StartupDisposition::Success)
     }
 
     #[instrument(level = "trace", skip_all)]
-    pub(super) async fn start_tcp_listeners(
+    pub(super) async fn start_tcp_listeners(&self) -> EyreResult<StartupDisposition> {
-        &self,
-        editor_public_internet: &mut RoutingDomainEditor,
-        editor_local_network: &mut RoutingDomainEditor,
-    ) -> EyreResult<StartupDisposition> {
         log_net!("TCP: binding protocol handlers");
 
-        let routing_table = self.routing_table();
         let (listen_address, public_address, detect_address_changes) = {
             let c = self.config.get();
             (
@@ -501,22 +548,70 @@ impl Network {
                 bind_set.port, bind_set.addrs
             );
         }
-        let Some(socket_addresses) = self
+        if !self
             .start_tcp_listener(
                 bind_set,
                 false,
+                ProtocolType::TCP,
                 Box::new(|c, _| Box::new(RawTcpProtocolHandler::new(c))),
             )
             .await?
-        else {
+        {
             return Ok(StartupDisposition::BindRetry);
+        }
+
+        {
+            let mut inner = self.inner.lock();
+            if public_address.is_some() && !detect_address_changes {
+                inner.static_public_dial_info.insert(ProtocolType::TCP);
+            }
+        }
+
+        Ok(StartupDisposition::Success)
+    }
+
+    #[instrument(level = "trace", skip_all)]
+    pub(super) async fn register_tcp_dial_info(
+        &self,
+        editor_public_internet: &mut RoutingDomainEditorPublicInternet,
+        editor_local_network: &mut RoutingDomainEditorLocalNetwork,
+    ) -> EyreResult<()> {
+        log_net!("TCP: registering dialinfo");
+
+        let (public_address, detect_address_changes) = {
+            let c = self.config.get();
+            (
+                c.network.protocol.tcp.public_address.clone(),
+                c.network.detect_address_changes,
+            )
         };
 
-        log_net!("TCP: protocol handlers started on {:#?}", socket_addresses);
-
-        let mut static_public = false;
         let mut registered_addresses: HashSet<IpAddr> = HashSet::new();
 
+        let socket_addresses = {
+            let mut out = vec![];
+            if let Some(bound_addresses) = {
+                let inner = self.inner.lock();
+                inner
+                    .bound_address_per_protocol
+                    .get(&ProtocolType::TCP)
+                    .cloned()
+            } {
+                for addr in bound_addresses {
+                    for idi_addr in self
+                        .translate_unspecified_address(addr)
+                        .into_iter()
+                        .map(SocketAddress::from_socket_addr)
+                    {
+                        out.push(idi_addr);
+                    }
+                }
+            }
+            out.sort();
+            out.dedup();
+            out
+        };
+
         // Add static public dialinfo if it's configured
         if let Some(public_address) = public_address.as_ref() {
             // Resolve statically configured public dialinfo
@@ -532,12 +627,11 @@ impl Network {
                 }
                 let pdi = DialInfo::tcp_from_socketaddr(pdi_addr);
 
-                editor_public_internet.register_dial_info(pdi.clone(), DialInfoClass::Direct)?;
+                editor_public_internet.add_dial_info(pdi.clone(), DialInfoClass::Direct);
-                static_public = true;
 
                 // See if this public address is also a local interface address
                 if self.is_stable_interface_address(pdi_addr.ip()) {
-                    editor_local_network.register_dial_info(pdi, DialInfoClass::Direct)?;
+                    editor_local_network.add_dial_info(pdi, DialInfoClass::Direct);
                 }
             }
         }
@@ -546,27 +640,14 @@ impl Network {
             let di = DialInfo::tcp(*socket_address);
 
             // Register global dial info if no public address is specified
-            if !detect_address_changes
+            if !detect_address_changes && public_address.is_none() && di.address().is_global() {
-                && public_address.is_none()
+                editor_public_internet.add_dial_info(di.clone(), DialInfoClass::Direct);
-                && routing_table.ensure_dial_info_is_valid(RoutingDomain::PublicInternet, &di)
-            {
-                editor_public_internet.register_dial_info(di.clone(), DialInfoClass::Direct)?;
-                static_public = true;
             }
             // Register interface dial info
-            editor_local_network.register_dial_info(di.clone(), DialInfoClass::Direct)?;
+            editor_local_network.add_dial_info(di.clone(), DialInfoClass::Direct);
             registered_addresses.insert(socket_address.ip_addr());
         }
 
-        let mut inner = self.inner.lock();
+        Ok(())
-
-        if static_public {
-            inner.static_public_dialinfo.insert(ProtocolType::TCP);
-        }
-        for sa in socket_addresses {
-            Self::add_preferred_local_address(&mut inner, PeerAddress::new(sa, ProtocolType::TCP));
-        }
-
-        Ok(StartupDisposition::Success)
     }
 }

veilid-core/src/network_manager/native/tasks/mod.rs

@@ -0,0 +1,110 @@
+mod network_interfaces_task;
+mod update_network_class_task;
+mod upnp_task;
+
+use super::*;
+
+impl Network {
+    pub(crate) fn setup_tasks(&self) {
+        // Set update network class tick task
+        {
+            let this = self.clone();
+            self.unlocked_inner
+                .update_network_class_task
+                .set_routine(move |s, l, t| {
+                    Box::pin(this.clone().update_network_class_task_routine(
+                        s,
+                        Timestamp::new(l),
+                        Timestamp::new(t),
+                    ))
+                });
+        }
+        // Set network interfaces tick task
+        {
+            let this = self.clone();
+            self.unlocked_inner
+                .network_interfaces_task
+                .set_routine(move |s, l, t| {
+                    Box::pin(this.clone().network_interfaces_task_routine(
+                        s,
+                        Timestamp::new(l),
+                        Timestamp::new(t),
+                    ))
+                });
+        }
+        // Set upnp tick task
+        {
+            let this = self.clone();
+            self.unlocked_inner.upnp_task.set_routine(move |s, l, t| {
+                Box::pin(
+                    this.clone()
+                        .upnp_task_routine(s, Timestamp::new(l), Timestamp::new(t)),
+                )
+            });
+        }
+    }
+
+    #[instrument(level = "trace", target = "net", name = "Network::tick", skip_all, err)]
+    pub(crate) async fn tick(&self) -> EyreResult<()> {
+        let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
+            log_net!(debug "ignoring due to not started up");
+            return Ok(());
+        };
+
+        // Ignore this tick if we need to restart
+        if self.needs_restart() {
+            return Ok(());
+        }
+
+        let (detect_address_changes, upnp) = {
+            let config = self.network_manager().config();
+            let c = config.get();
+            (c.network.detect_address_changes, c.network.upnp)
+        };
+
+        // If we need to figure out our network class, tick the task for it
+        if detect_address_changes {
+            // Check our network interfaces to see if they have changed
+            self.unlocked_inner.network_interfaces_task.tick().await?;
+
+            // Check our public dial info to see if it has changed
+            let public_internet_network_class = self
+                .routing_table()
+                .get_network_class(RoutingDomain::PublicInternet)
+                .unwrap_or(NetworkClass::Invalid);
+            let needs_public_dial_info_check = self.needs_public_dial_info_check();
+            if public_internet_network_class == NetworkClass::Invalid
+                || needs_public_dial_info_check
+            {
+                let routing_table = self.routing_table();
+                let rth = routing_table.get_routing_table_health();
+
+                // We want at least two live entries per crypto kind before we start doing this (bootstrap)
+                let mut has_at_least_two = true;
+                for ck in VALID_CRYPTO_KINDS {
+                    if rth
+                        .live_entry_counts
+                        .get(&(RoutingDomain::PublicInternet, ck))
+                        .copied()
+                        .unwrap_or_default()
+                        < 2
+                    {
+                        has_at_least_two = false;
+                        break;
+                    }
+                }
+
+                if has_at_least_two {
+                    self.unlocked_inner.update_network_class_task.tick().await?;
+                }
+            }
+        }
+
+        // If we need to tick upnp, do it
+        if upnp {
+            self.unlocked_inner.upnp_task.tick().await?;
+        }
+
+        Ok(())
+    }
+}

veilid-core/src/network_manager/native/tasks/network_interfaces_task.rs

@@ -0,0 +1,73 @@
+use super::*;
+
+impl Network {
+    #[instrument(level = "trace", target = "net", skip_all, err)]
+    pub(super) async fn network_interfaces_task_routine(
+        self,
+        _stop_token: StopToken,
+        _l: Timestamp,
+        _t: Timestamp,
+    ) -> EyreResult<()> {
+        let _guard = self.unlocked_inner.network_task_lock.lock().await;
+
+        self.update_network_state().await?;
+
+        Ok(())
+    }
+
+    // See if our interface addresses have changed, if so redo public dial info if necessary
+    async fn update_network_state(&self) -> EyreResult<bool> {
+        let mut local_network_changed = false;
+        let mut public_internet_changed = false;
+
+        let last_network_state = self.last_network_state();
+        let new_network_state = match self.make_network_state().await {
+            Ok(v) => v,
+            Err(e) => {
+                log_net!(debug "Skipping network state update: {}", e);
+                return Ok(false);
+            }
+        };
+
+        if new_network_state != last_network_state {
+            // Save new network state
+            {
+                let mut inner = self.inner.lock();
+                inner.network_state = Some(new_network_state.clone());
+            }
+
+            // network state has changed
+            let mut editor_local_network = self
+                .unlocked_inner
+                .routing_table
+                .edit_local_network_routing_domain();
+            editor_local_network.set_local_networks(new_network_state.local_networks);
+            editor_local_network.clear_dial_info_details(None, None);
+
+            let mut editor_public_internet = self
+                .unlocked_inner
+                .routing_table
+                .edit_public_internet_routing_domain();
+
+            // Update protocols
+            self.register_all_dial_info(&mut editor_public_internet, &mut editor_local_network)
+                .await?;
+
+            local_network_changed = editor_local_network.commit(true).await;
+            public_internet_changed = editor_public_internet.commit(true).await;
+
+            // Update local network now
+            if local_network_changed {
+                editor_local_network.publish();
+            }
+        }
+
+        // If any of the new addresses were PublicInternet addresses, re-run public dial info check
+        if public_internet_changed {
+            // inner.network_needs_restart = true;
+            self.set_needs_public_dial_info_check(None);
+        }
+
+        Ok(local_network_changed || public_internet_changed)
+    }
+}

veilid-core/src/network_manager/native/tasks/update_network_class_task.rs

@@ -4,6 +4,28 @@ use futures_util::stream::FuturesUnordered;
 use stop_token::future::FutureExt as StopTokenFutureExt;
 
 impl Network {
+    #[instrument(parent = None, level = "trace", skip(self), err)]
+    pub async fn update_network_class_task_routine(
+        self,
+        stop_token: StopToken,
+        l: Timestamp,
+        t: Timestamp,
+    ) -> EyreResult<()> {
+        let _guard = self.unlocked_inner.network_task_lock.lock().await;
+
+        // Do the public dial info check
+        let out = self.do_public_dial_info_check(stop_token, l, t).await;
+
+        // Done with public dial info check
+        {
+            let mut inner = self.inner.lock();
+            inner.needs_public_dial_info_check = false;
+            inner.public_dial_info_check_punishment = None;
+        }
+
+        out
+    }
+
     #[instrument(level = "trace", skip(self), err)]
     pub async fn update_with_detected_dial_info(&self, ddi: DetectedDialInfo) -> EyreResult<()> {
         let existing_network_class = self
@@ -16,9 +38,7 @@ impl Network {
                 // If we get any symmetric nat dialinfo, this whole network class is outbound only,
                 // and all dial info should be treated as invalid
                 if !matches!(existing_network_class, NetworkClass::OutboundOnly) {
-                    let mut editor = self
+                    let mut editor = self.routing_table().edit_public_internet_routing_domain();
-                        .routing_table()
-                        .edit_routing_domain(RoutingDomain::PublicInternet);
 
                     editor.clear_dial_info_details(None, None);
                     editor.set_network_class(Some(NetworkClass::OutboundOnly));
@@ -67,9 +87,7 @@ impl Network {
                     }
 
                     if clear || add {
-                        let mut editor = self
+                        let mut editor = self.routing_table().edit_public_internet_routing_domain();
-                            .routing_table()
-                            .edit_routing_domain(RoutingDomain::PublicInternet);
 
                         if clear {
                             editor.clear_dial_info_details(
@@ -79,11 +97,7 @@ impl Network {
                         }
 
                         if add {
-                            if let Err(e) =
+                            editor.add_dial_info(did.dial_info.clone(), did.class);
-                                editor.register_dial_info(did.dial_info.clone(), did.class)
-                            {
-                                log_net!(debug "Failed to register detected dialinfo {:?}: {}", did, e);
-                            }
                         }
 
                         editor.set_network_class(Some(NetworkClass::InboundCapable));
@@ -99,13 +113,19 @@ impl Network {
     pub async fn do_public_dial_info_check(
         &self,
         stop_token: StopToken,
-        _l: u64,
+        _l: Timestamp,
-        _t: u64,
+        _t: Timestamp,
     ) -> EyreResult<()> {
         // Figure out if we can optimize TCP/WS checking since they are often on the same port
         let (protocol_config, inbound_protocol_map) = {
             let mut inner = self.inner.lock();
-            let protocol_config = inner.protocol_config.clone();
+            let Some(protocol_config) = inner
+                .network_state
+                .as_ref()
+                .map(|ns| ns.protocol_config.clone())
+            else {
+                bail!("should not be doing public dial info check before we have an initial network state");
+            };
 
             // Allow network to be cleared if external addresses change
             inner.network_already_cleared = false;
@@ -118,7 +138,7 @@ impl Network {
 
                     // Skip things with static public dialinfo
                     // as they don't need to participate in discovery
-                    if inner.static_public_dialinfo.contains(pt) {
+                    if inner.static_public_dial_info.contains(pt) {
                         continue;
                     }
 
@@ -147,9 +167,7 @@ impl Network {
             .collect();
 
         // Set most permissive network config
-        let mut editor = self
+        let mut editor = self.routing_table().edit_public_internet_routing_domain();
-            .routing_table()
-            .edit_routing_domain(RoutingDomain::PublicInternet);
         editor.setup_network(
             protocol_config.outbound,
             protocol_config.inbound,
@@ -171,9 +189,7 @@ impl Network {
                     }
                     inner.network_already_cleared = true;
                 }
-                let mut editor = this
+                let mut editor = this.routing_table().edit_public_internet_routing_domain();
-                    .routing_table()
-                    .edit_routing_domain(RoutingDomain::PublicInternet);
                 editor.clear_dial_info_details(None, None);
                 editor.set_network_class(None);
                 editor.commit(true).await;
@@ -261,7 +277,9 @@ impl Network {
             all_address_types,
             protocol_config.public_internet_capabilities,
         );
-        editor.commit(true).await;
+        if editor.commit(true).await {
+            editor.publish();
+        }
 
         // See if the dial info changed
         let new_public_dial_info: HashSet<DialInfoDetail> = self
@@ -282,25 +300,6 @@ impl Network {
 
         Ok(())
     }
-    #[instrument(parent = None, level = "trace", skip(self), err)]
-    pub async fn update_network_class_task_routine(
-        self,
-        stop_token: StopToken,
-        l: u64,
-        t: u64,
-    ) -> EyreResult<()> {
-        // Do the public dial info check
-        let out = self.do_public_dial_info_check(stop_token, l, t).await;
-
-        // Done with public dial info check
-        {
-            let mut inner = self.inner.lock();
-            inner.needs_public_dial_info_check = false;
-            inner.public_dial_info_check_punishment = None;
-        }
-
-        out
-    }
 
     /// Make a dialinfo from an address and protocol type
     pub fn make_dial_info(&self, addr: SocketAddress, protocol_type: ProtocolType) -> DialInfo {

veilid-core/src/network_manager/native/tasks/upnp_task.rs

@@ -0,0 +1,19 @@
+use super::*;
+
+impl Network {
+    #[instrument(parent = None, level = "trace", target = "net", skip_all, err)]
+    pub(super) async fn upnp_task_routine(
+        self,
+        _stop_token: StopToken,
+        _l: Timestamp,
+        _t: Timestamp,
+    ) -> EyreResult<()> {
+        if !self.unlocked_inner.igd_manager.tick().await? {
+            info!("upnp failed, restarting local network");
+            let mut inner = self.inner.lock();
+            inner.network_needs_restart = true;
+        }
+
+        Ok(())
+    }
+}

veilid-core/src/network_manager/network_connection.rs

@@ -183,7 +183,7 @@ impl NetworkConnection {
         self.flow
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn unique_flow(&self) -> UniqueFlow {
         UniqueFlow {
             flow: self.flow,
@@ -207,6 +207,10 @@ impl NetworkConnection {
         self.protected_nr = Some(protect_nr);
     }
 
+    pub fn unprotect(&mut self) {
+        self.protected_nr = None;
+    }
+
     pub fn add_ref(&mut self) {
         self.ref_count += 1;
     }
@@ -254,13 +258,12 @@ impl NetworkConnection {
         Ok(NetworkResult::Value(out))
     }
 
-    #[allow(dead_code)]
     pub fn stats(&self) -> NetworkConnectionStats {
         let stats = self.stats.lock();
         stats.clone()
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn established_time(&self) -> Timestamp {
         self.established_time
     }

veilid-core/src/network_manager/receipt_manager.rs

@@ -7,12 +7,16 @@ use routing_table::*;
 use stop_token::future::FutureExt;
 
 #[derive(Clone, Debug)]
-#[allow(dead_code)]
 pub(crate) enum ReceiptEvent {
     ReturnedOutOfBand,
-    ReturnedInBand { inbound_noderef: NodeRef },
+    ReturnedInBand {
+        inbound_noderef: FilteredNodeRef,
+    },
     ReturnedSafety,
-    ReturnedPrivate { private_route: PublicKey },
+    ReturnedPrivate {
+        #[expect(dead_code)]
+        private_route: PublicKey,
+    },
     Expired,
     Cancelled,
 }
@@ -20,7 +24,7 @@ pub(crate) enum ReceiptEvent {
 #[derive(Clone, Debug)]
 pub(super) enum ReceiptReturned {
     OutOfBand,
-    InBand { inbound_noderef: NodeRef },
+    InBand { inbound_noderef: FilteredNodeRef },
     Safety,
     Private { private_route: PublicKey },
 }
@@ -53,7 +57,6 @@ where
 type ReceiptCallbackType = Box<dyn ReceiptCallback>;
 type ReceiptSingleShotType = SingleShotEventual<ReceiptEvent>;
 
-#[allow(dead_code)]
 enum ReceiptRecordCallbackType {
     Normal(ReceiptCallbackType),
     SingleShot(Option<ReceiptSingleShotType>),
@@ -92,7 +95,7 @@ impl fmt::Debug for ReceiptRecord {
 }
 
 impl ReceiptRecord {
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn new(
         receipt: Receipt,
         expiration_ts: Timestamp,
@@ -352,7 +355,6 @@ impl ReceiptManager {
         log_net!(debug "finished receipt manager shutdown");
     }
 
-    #[allow(dead_code)]
     #[instrument(level = "trace", target = "receipt", skip_all)]
     pub fn record_receipt(
         &self,
@@ -418,7 +420,7 @@ impl ReceiptManager {
         inner.next_oldest_ts = new_next_oldest_ts;
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub async fn cancel_receipt(&self, nonce: &Nonce) -> EyreResult<()> {
         event!(target: "receipt", Level::DEBUG, "== Cancel Receipt {}", nonce.encode());
 

veilid-core/src/network_manager/send_data.rs

@@ -15,7 +15,7 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     pub(crate) async fn send_data(
         &self,
-        destination_node_ref: NodeRef,
+        destination_node_ref: FilteredNodeRef,
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<SendDataMethod>> {
         // First try to send data to the last flow we've seen this peer on
@@ -81,7 +81,7 @@ impl NetworkManager {
     pub(crate) fn try_possibly_relayed_contact_method(
         &self,
         possibly_relayed_contact_method: NodeContactMethod,
-        destination_node_ref: NodeRef,
+        destination_node_ref: FilteredNodeRef,
         data: Vec<u8>,
     ) -> SendPinBoxFuture<EyreResult<NetworkResult<SendDataMethod>>> {
         let this = self.clone();
@@ -179,7 +179,7 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     async fn send_data_ncm_existing(
         &self,
-        target_node_ref: NodeRef,
+        target_node_ref: FilteredNodeRef,
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<SendDataMethod>> {
         // First try to send data to the last connection we've seen this peer on
@@ -213,7 +213,7 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     async fn send_data_ncm_unreachable(
         &self,
-        target_node_ref: NodeRef,
+        target_node_ref: FilteredNodeRef,
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<SendDataMethod>> {
         // Try to send data to the last socket we've seen this peer on
@@ -248,8 +248,8 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     async fn send_data_ncm_signal_reverse(
         &self,
-        relay_nr: NodeRef,
+        relay_nr: FilteredNodeRef,
-        target_node_ref: NodeRef,
+        target_node_ref: FilteredNodeRef,
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<SendDataMethod>> {
         // First try to send data to the last socket we've seen this peer on
@@ -291,8 +291,8 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     async fn send_data_ncm_signal_hole_punch(
         &self,
-        relay_nr: NodeRef,
+        relay_nr: FilteredNodeRef,
-        target_node_ref: NodeRef,
+        target_node_ref: FilteredNodeRef,
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<SendDataMethod>> {
         // First try to send data to the last socket we've seen this peer on
@@ -334,7 +334,7 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     async fn send_data_ncm_direct(
         &self,
-        node_ref: NodeRef,
+        node_ref: FilteredNodeRef,
         dial_info: DialInfo,
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<SendDataMethod>> {
@@ -362,7 +362,7 @@ impl NetworkManager {
                 }
                 SendDataToExistingFlowResult::NotSent(d) => {
                     // Connection couldn't send, kill it
-                    node_ref.clear_last_connection(flow);
+                    node_ref.clear_last_flow(flow);
                     d
                 }
             }
@@ -393,7 +393,7 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     pub(crate) fn get_node_contact_method(
         &self,
-        target_node_ref: NodeRef,
+        target_node_ref: FilteredNodeRef,
     ) -> EyreResult<NodeContactMethod> {
         let routing_table = self.routing_table();
 
@@ -415,31 +415,31 @@ impl NetworkManager {
             }
         };
 
-        // Get cache key
-        let ncm_key = NodeContactMethodCacheKey {
-            node_ids: target_node_ref.node_ids(),
-            own_node_info_ts: routing_table.get_own_node_info_ts(routing_domain),
-            target_node_info_ts: target_node_ref.node_info_ts(routing_domain),
-            target_node_ref_filter: target_node_ref.filter_ref().cloned(),
-            target_node_ref_sequencing: target_node_ref.sequencing(),
-        };
-        if let Some(ncm) = self.inner.lock().node_contact_method_cache.get(&ncm_key) {
-            return Ok(ncm.clone());
-        }
-
         // Node A is our own node
         // Use whatever node info we've calculated so far
-        let peer_a = routing_table.get_own_peer_info(routing_domain);
+        let peer_a = routing_table.get_current_peer_info(routing_domain);
 
         // Node B is the target node
         let peer_b = match target_node_ref.make_peer_info(routing_domain) {
-            Some(ni) => ni,
+            Some(pi) => Arc::new(pi),
             None => {
                 log_net!("no node info for node {:?}", target_node_ref);
                 return Ok(NodeContactMethod::Unreachable);
             }
         };
 
+        // Get cache key
+        let ncm_key = NodeContactMethodCacheKey {
+            node_ids: target_node_ref.node_ids(),
+            own_node_info_ts: peer_a.signed_node_info().timestamp(),
+            target_node_info_ts: peer_b.signed_node_info().timestamp(),
+            target_node_ref_filter: target_node_ref.filter(),
+            target_node_ref_sequencing: target_node_ref.sequencing(),
+        };
+        if let Some(ncm) = self.inner.lock().node_contact_method_cache.get(&ncm_key) {
+            return Ok(ncm.clone());
+        }
+
         // Dial info filter comes from the target node ref but must be filtered by this node's outbound capabilities
         let dial_info_filter = target_node_ref.dial_info_filter().filtered(
             &DialInfoFilter::all()
@@ -463,7 +463,7 @@ impl NetworkManager {
         for did in peer_b
             .signed_node_info()
             .node_info()
-            .all_filtered_dial_info_details(DialInfoDetail::NO_SORT, |_| true)
+            .filtered_dial_info_details(DialInfoDetail::NO_SORT, |_| true)
         {
             if let Some(ts) = address_filter.get_dial_info_failed_ts(&did.dial_info) {
                 dial_info_failures_map.insert(did.dial_info, ts);
@@ -488,8 +488,8 @@ impl NetworkManager {
         // Get the best contact method with these parameters from the routing domain
         let cm = routing_table.get_contact_method(
             routing_domain,
-            &peer_a,
+            peer_a.clone(),
-            &peer_b,
+            peer_b.clone(),
             dial_info_filter,
             sequencing,
             dif_sort,
@@ -585,8 +585,8 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     async fn do_reverse_connect(
         &self,
-        relay_nr: NodeRef,
+        relay_nr: FilteredNodeRef,
-        target_nr: NodeRef,
+        target_nr: FilteredNodeRef,
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<UniqueFlow>> {
         // Detect if network is stopping so we can break out of this
@@ -611,22 +611,24 @@ impl NetworkManager {
             ));
         };
 
-        // Ensure we have a valid network class so our peer info is useful
+        // Get our published peer info
-        if !self.routing_table().has_valid_network_class(routing_domain) {
+        let Some(published_peer_info) =
+            self.routing_table().get_published_peer_info(routing_domain)
+        else {
             return Ok(NetworkResult::no_connection_other(
                 "Network class not yet valid for reverse connect",
             ));
         };
 
-        // Get our peer info
-        let peer_info = self.routing_table().get_own_peer_info(routing_domain);
-
         // Issue the signal
         let rpc = self.rpc_processor();
         network_result_try!(rpc
             .rpc_call_signal(
-                Destination::relay(relay_nr.clone(), target_nr.clone()),
+                Destination::relay(relay_nr.clone(), target_nr.unfiltered()),
-                SignalInfo::ReverseConnect { receipt, peer_info },
+                SignalInfo::ReverseConnect {
+                    receipt,
+                    peer_info: published_peer_info
+                },
             )
             .await
             .wrap_err("failed to send signal")?);
@@ -694,8 +696,8 @@ impl NetworkManager {
     #[instrument(level = "trace", target = "net", skip_all, err)]
     async fn do_hole_punch(
         &self,
-        relay_nr: NodeRef,
+        relay_nr: FilteredNodeRef,
-        target_nr: NodeRef,
+        target_nr: FilteredNodeRef,
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<UniqueFlow>> {
         // Detect if network is stopping so we can break out of this
@@ -703,12 +705,12 @@ impl NetworkManager {
             return Ok(NetworkResult::service_unavailable("network is stopping"));
         };
 
-        // Ensure we are filtered down to UDP (the only hole punch protocol supported today)
+        // Ensure target is filtered down to UDP (the only hole punch protocol supported today)
-        assert!(target_nr
+        // Relay can be any protocol because the signal rpc contains the dialinfo to connect over
-            .filter_ref()
+        assert_eq!(
-            .map(|nrf| nrf.dial_info_filter.protocol_type_set
+            target_nr.dial_info_filter().protocol_type_set,
-                == ProtocolTypeSet::only(ProtocolType::UDP))
+            ProtocolType::UDP
-            .unwrap_or_default());
+        );
 
         // Build a return receipt for the signal
         let receipt_timeout = TimestampDuration::new_ms(
@@ -727,19 +729,18 @@ impl NetworkManager {
             ));
         };
 
-        // Ensure we have a valid network class so our peer info is useful
+        // Get our published peer info
-        if !self.routing_table().has_valid_network_class(routing_domain) {
+        let Some(published_peer_info) =
+            self.routing_table().get_published_peer_info(routing_domain)
+        else {
             return Ok(NetworkResult::no_connection_other(
                 "Network class not yet valid for hole punch",
             ));
         };
 
-        // Get our peer info
-        let peer_info = self.routing_table().get_own_peer_info(routing_domain);
-
         // Get the udp direct dialinfo for the hole punch
         let hole_punch_did = target_nr
-            .first_filtered_dial_info_detail()
+            .first_dial_info_detail()
             .ok_or_else(|| eyre!("No hole punch capable dialinfo found for node"))?;
 
         // Do our half of the hole punch by sending an empty packet
@@ -756,8 +757,11 @@ impl NetworkManager {
         let rpc = self.rpc_processor();
         network_result_try!(rpc
             .rpc_call_signal(
-                Destination::relay(relay_nr, target_nr.clone()),
+                Destination::relay(relay_nr, target_nr.unfiltered()),
-                SignalInfo::HolePunch { receipt, peer_info },
+                SignalInfo::HolePunch {
+                    receipt,
+                    peer_info: published_peer_info
+                },
             )
             .await
             .wrap_err("failed to send signal")?);

veilid-core/src/network_manager/stats.rs

@@ -69,7 +69,7 @@ impl NetworkManager {
             .add_down(bytes);
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn get_stats(&self) -> NetworkManagerStats {
         let inner = self.inner.lock();
         inner.stats.clone()

veilid-core/src/network_manager/tasks/local_network_address_check.rs

@@ -0,0 +1,15 @@
+use super::*;
+
+impl NetworkManager {
+    // Determine if a local IP address has changed
+    // this means we should restart the low level network and and recreate all of our dial info
+    // Wait until we have received confirmation from N different peers
+    pub fn report_local_network_socket_address(
+        &self,
+        _socket_address: SocketAddress,
+        _flow: Flow,
+        _reporting_peer: NodeRef,
+    ) {
+        // XXX: Nothing here yet.
+    }
+}

veilid-core/src/network_manager/tasks/mod.rs

@@ -1,4 +1,5 @@
-pub mod public_address_check;
+pub mod local_network_address_check;
+pub mod public_internet_address_check;
 pub mod rolling_transfers;
 
 use super::*;
@@ -19,13 +20,13 @@ impl NetworkManager {
                 });
         }
 
-        // Set public address check task
+        // Set public internet address check task
         {
             let this = self.clone();
             self.unlocked_inner
-                .public_address_check_task
+                .public_internet_address_check_task
                 .set_routine(move |s, l, t| {
-                    Box::pin(this.clone().public_address_check_task_routine(
+                    Box::pin(this.clone().public_internet_address_check_task_routine(
                         s,
                         Timestamp::new(l),
                         Timestamp::new(t),

veilid-core/src/network_manager/tasks/public_internet_address_check.rs

@@ -3,7 +3,7 @@ use super::*;
 impl NetworkManager {
     // Clean up the public address check tables, removing entries that have timed out
     #[instrument(parent = None, level = "trace", skip_all, err)]
-    pub(crate) async fn public_address_check_task_routine(
+    pub(crate) async fn public_internet_address_check_task_routine(
         self,
         _stop_token: StopToken,
         _last_ts: Timestamp,
@@ -11,32 +11,17 @@ impl NetworkManager {
     ) -> EyreResult<()> {
         // go through public_address_inconsistencies_table and time out things that have expired
         let mut inner = self.inner.lock();
-        for pait_v in inner.public_address_inconsistencies_table.values_mut() {
+        for pait_v in inner
-            let mut expired = Vec::new();
+            .public_internet_address_inconsistencies_table
-            for (addr, exp_ts) in pait_v.iter() {
+            .values_mut()
-                if *exp_ts <= cur_ts {
+        {
-                    expired.push(*addr);
+            pait_v.retain(|_addr, exp_ts| {
-                }
+                // Keep it if it's in the future
-            }
+                *exp_ts > cur_ts
-            for exp in expired {
+            });
-                pait_v.remove(&exp);
-            }
         }
         Ok(())
     }
-
-    // Determine if a local IP address has changed
-    // this means we should restart the low level network and and recreate all of our dial info
-    // Wait until we have received confirmation from N different peers
-    pub fn report_local_network_socket_address(
-        &self,
-        _socket_address: SocketAddress,
-        _flow: Flow,
-        _reporting_peer: NodeRef,
-    ) {
-        // XXX: Nothing here yet.
-    }
-
     // Determine if a global IP address has changed
     // this means we should recreate our public dial info if it is not static and rediscover it
     // Wait until we have received confirmation from N different peers
@@ -46,7 +31,7 @@ impl NetworkManager {
         flow: Flow,                    // the flow used
         reporting_peer: NodeRef,       // the peer's noderef reporting the socket address
     ) {
-        log_network_result!("report_global_socket_address\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer);
+        log_network_result!("report_public_internet_socket_address:\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer);
 
         // Ignore these reports if we are currently detecting public dial info
         let net = self.net();
@@ -69,17 +54,22 @@ impl NetworkManager {
             return;
         }
 
+        // Get our current published peer info
+        let routing_table = self.routing_table();
+        let Some(published_peer_info) =
+            routing_table.get_published_peer_info(RoutingDomain::PublicInternet)
+        else {
+            return;
+        };
+
         // If we are a webapp we should skip this completely
         // because we will never get inbound dialinfo directly on our public ip address
         // If we have an invalid network class, this is not necessary yet
-        let routing_table = self.routing_table();
+        let public_internet_network_class = published_peer_info
-        let public_internet_network_class = routing_table
+            .signed_node_info()
-            .get_network_class(RoutingDomain::PublicInternet)
+            .node_info()
-            .unwrap_or(NetworkClass::Invalid);
+            .network_class();
-        if matches!(
+        if matches!(public_internet_network_class, NetworkClass::WebApp) {
-            public_internet_network_class,
-            NetworkClass::Invalid | NetworkClass::WebApp
-        ) {
             return;
         }
 
@@ -119,7 +109,7 @@ impl NetworkManager {
         let addr_proto_type_key =
             PublicAddressCheckCacheKey(flow.protocol_type(), flow.address_type());
         if inner
-            .public_address_inconsistencies_table
+            .public_internet_address_inconsistencies_table
             .get(&addr_proto_type_key)
             .map(|pait| pait.contains_key(&reporting_ipblock))
             .unwrap_or(false)
@@ -130,17 +120,17 @@ impl NetworkManager {
         // Insert this new public address into the lru cache for the address check
         // if we've seen this address before, it brings it to the front
         let pacc = inner
-            .public_address_check_cache
+            .public_internet_address_check_cache
             .entry(addr_proto_type_key)
             .or_insert_with(|| LruCache::new(PUBLIC_ADDRESS_CHECK_CACHE_SIZE));
         pacc.insert(reporting_ipblock, socket_address);
 
         // Determine if our external address has likely changed
-        let mut bad_public_address_detection_punishment: Option<
+        let mut bad_public_internet_address_detection_punishment: Option<
             Box<dyn FnOnce() + Send + 'static>,
         > = None;
 
-        let needs_public_address_detection = if matches!(
+        let needs_public_internet_address_detection = if matches!(
             public_internet_network_class,
             NetworkClass::InboundCapable
         ) {
@@ -148,11 +138,12 @@ impl NetworkManager {
             let dial_info_filter = flow.make_dial_info_filter();
 
             // Get current external ip/port from registered global dialinfo
-            let current_addresses: BTreeSet<SocketAddress> = routing_table
+            let current_addresses: BTreeSet<SocketAddress> = published_peer_info
-                .all_filtered_dial_info_details(
+                .signed_node_info()
-                    RoutingDomain::PublicInternet.into(),
+                .node_info()
-                    &dial_info_filter,
+                .filtered_dial_info_details(DialInfoDetail::NO_SORT, |did| {
-                )
+                    did.matches_filter(&dial_info_filter)
+                })
                 .iter()
                 .map(|did| {
                     // Strip port from direct and mapped addresses
@@ -180,7 +171,7 @@ impl NetworkManager {
                 if !current_addresses.contains(a)
                     && !current_addresses.contains(&a.with_port(0))
                     && !inner
-                        .public_address_inconsistencies_table
+                        .public_internet_address_inconsistencies_table
                         .get(&addr_proto_type_key)
                         .map(|pait| pait.contains_key(reporting_ip_block))
                         .unwrap_or(false)
@@ -195,39 +186,42 @@ impl NetworkManager {
             // If we have enough inconsistencies to consider changing our public dial info,
             // add them to our denylist (throttling) and go ahead and check for new
             // public dialinfo
-            let inconsistent = if inconsistencies.len() >= PUBLIC_ADDRESS_CHANGE_DETECTION_COUNT {
+            let inconsistent =
-                let exp_ts = Timestamp::now() + PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US;
+                if inconsistencies.len() >= PUBLIC_ADDRESS_CHANGE_INCONSISTENCY_DETECTION_COUNT {
-                let pait = inner
+                    let exp_ts = Timestamp::now() + PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US;
-                    .public_address_inconsistencies_table
-                    .entry(addr_proto_type_key)
-                    .or_default();
-                for i in &inconsistencies {
-                    pait.insert(*i, exp_ts);
-                }
-
-                // Run this routine if the inconsistent nodes turn out to be lying
-                let this = self.clone();
-                bad_public_address_detection_punishment = Some(Box::new(move || {
-                    let mut inner = this.inner.lock();
                     let pait = inner
-                        .public_address_inconsistencies_table
+                        .public_internet_address_inconsistencies_table
                         .entry(addr_proto_type_key)
                         .or_default();
-                    let exp_ts =
+                    for i in &inconsistencies {
-                        Timestamp::now() + PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US;
+                        pait.insert(*i, exp_ts);
-                    for i in inconsistencies {
-                        pait.insert(i, exp_ts);
                     }
-                }));
 
-                true
+                    // Run this routine if the inconsistent nodes turn out to be lying
-            } else {
+                    let this = self.clone();
-                false
+                    bad_public_internet_address_detection_punishment = Some(Box::new(move || {
-            };
+                        // xxx does this even work??
+
+                        let mut inner = this.inner.lock();
+                        let pait = inner
+                            .public_internet_address_inconsistencies_table
+                            .entry(addr_proto_type_key)
+                            .or_default();
+                        let exp_ts =
+                            Timestamp::now() + PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US;
+                        for i in inconsistencies {
+                            pait.insert(i, exp_ts);
+                        }
+                    }));
+
+                    true
+                } else {
+                    false
+                };
 
             // // debug code
             // if inconsistent {
-            //     log_net!("public_address_check_cache: {:#?}\ncurrent_addresses: {:#?}\ninconsistencies: {}", inner
+            //     log_net!("report_public_internet_socket_address: {:#?}\ncurrent_addresses: {:#?}\ninconsistencies: {}", inner
             //                 .public_address_check_cache, current_addresses, inconsistencies);
             // }
 
@@ -246,7 +240,7 @@ impl NetworkManager {
                 if let Some(current_address) = current_address {
                     if current_address == *a {
                         consistencies += 1;
-                        if consistencies >= PUBLIC_ADDRESS_CHANGE_DETECTION_COUNT {
+                        if consistencies >= PUBLIC_ADDRESS_CHANGE_CONSISTENCY_DETECTION_COUNT {
                             consistent = true;
                             break;
                         }
@@ -264,26 +258,28 @@ impl NetworkManager {
             unreachable!();
         };
 
-        if needs_public_address_detection {
+        if needs_public_internet_address_detection {
             if detect_address_changes {
                 // Reset the address check cache now so we can start detecting fresh
-                info!("Public address has changed, detecting public dial info");
+                info!("PublicInternet address has changed, detecting public dial info");
-                log_net!(debug "report_global_socket_address\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer);
+                log_net!(debug "report_public_internet_socket_address:\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer);
                 log_net!(debug
-                    "public_address_check_cache: {:#?}",
+                    "public_internet_address_check_cache: {:#?}",
-                    inner.public_address_check_cache
+                    inner.public_internet_address_check_cache
                 );
 
-                inner.public_address_check_cache.clear();
+                inner.public_internet_address_check_cache.clear();
 
                 // Re-detect the public dialinfo
-                net.set_needs_public_dial_info_check(bad_public_address_detection_punishment);
+                net.set_needs_public_dial_info_check(
+                    bad_public_internet_address_detection_punishment,
+                );
             } else {
-                warn!("Public address may have changed. Restarting the server may be required.");
+                warn!("PublicInternet address may have changed. Restarting the server may be required.");
-                warn!("report_global_socket_address\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer);
+                warn!("report_public_internet_socket_address:\nsocket_address: {:#?}\nflow: {:#?}\nreporting_peer: {:#?}", socket_address, flow, reporting_peer);
                 warn!(
-                    "public_address_check_cache: {:#?}",
+                    "public_internet_address_check_cache: {:#?}",
-                    inner.public_address_check_cache
+                    inner.public_internet_address_check_cache
                 );
             }
         }

veilid-core/src/network_manager/types/address.rs

@@ -20,7 +20,7 @@ impl Address {
             SocketAddr::V6(v6) => Address::IPV6(*v6.ip()),
         }
     }
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+    #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
     pub fn from_ip_addr(addr: IpAddr) -> Address {
         match addr {
             IpAddr::V4(v4) => Address::IPV4(v4),

veilid-core/src/network_manager/types/dial_info/mod.rs

@@ -234,7 +234,7 @@ impl DialInfo {
             Self::WSS(di) => di.socket_address.address(),
         }
     }
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn set_address(&mut self, address: Address) {
         match self {
             Self::UDP(di) => di.socket_address.set_address(address),
@@ -259,7 +259,7 @@ impl DialInfo {
             Self::WSS(di) => di.socket_address.ip_addr(),
         }
     }
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+    #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
     pub fn port(&self) -> u16 {
         match self {
             Self::UDP(di) => di.socket_address.port(),
@@ -268,7 +268,7 @@ impl DialInfo {
             Self::WSS(di) => di.socket_address.port(),
         }
     }
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+    #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
     pub fn set_port(&mut self, port: u16) {
         match self {
             Self::UDP(di) => di.socket_address.set_port(port),
@@ -277,7 +277,6 @@ impl DialInfo {
             Self::WSS(di) => di.socket_address.set_port(port),
         }
     }
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
     pub fn to_socket_addr(&self) -> SocketAddr {
         match self {
             Self::UDP(di) => di.socket_address.socket_addr(),
@@ -457,7 +456,7 @@ impl DialInfo {
             }
         }
     }
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub async fn to_url(&self) -> String {
         match self {
             DialInfo::UDP(di) => intf::ptr_lookup(di.socket_address.ip_addr())

veilid-core/src/network_manager/types/dial_info_filter.rs

@@ -46,7 +46,7 @@ impl DialInfoFilter {
     pub fn is_dead(&self) -> bool {
         self.protocol_type_set.is_empty() || self.address_type_set.is_empty()
     }
-    pub fn with_sequencing(self, sequencing: Sequencing) -> (bool, DialInfoFilter) {
+    pub fn apply_sequencing(self, sequencing: Sequencing) -> (bool, DialInfoFilter) {
         // Get first filtered dialinfo
         match sequencing {
             Sequencing::NoPreference => (false, self),

veilid-core/src/network_manager/types/punishment.rs

@@ -10,7 +10,7 @@ pub enum PunishmentReason {
     // Node-level punishments
     FailedToDecodeOperation,
     WrongSenderPeerInfo,
-    FailedToVerifySenderPeerInfo,
+    // FailedToVerifySenderPeerInfo,
     FailedToRegisterSenderPeerInfo,
     // Route-level punishments
     // FailedToDecodeRoutedMessage,

veilid-core/src/network_manager/types/signal_info.rs

@@ -1,21 +1,21 @@
 use super::*;
 
 /// Parameter for Signal operation
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug)]
 pub enum SignalInfo {
     /// UDP Hole Punch Request
     HolePunch {
         /// /// Receipt to be returned after the hole punch
         receipt: Vec<u8>,
         /// Sender's peer info
-        peer_info: PeerInfo,
+        peer_info: Arc<PeerInfo>,
     },
     /// Reverse Connection Request
     ReverseConnect {
         /// Receipt to be returned by the reverse connection
         receipt: Vec<u8>,
         /// Sender's peer info
-        peer_info: PeerInfo,
+        peer_info: Arc<PeerInfo>,
     },
     // XXX: WebRTC
 }

veilid-core/src/network_manager/types/socket_address.rs

@@ -30,7 +30,6 @@ impl SocketAddress {
     pub fn port(&self) -> u16 {
         self.port
     }
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
     pub fn set_port(&mut self, port: u16) {
         self.port = port
     }

veilid-core/src/network_manager/wasm/mod.rs

@@ -51,6 +51,14 @@ pub const MAX_CAPABILITIES: usize = 64;
 
 /////////////////////////////////////////////////////////////////
 
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ProtocolConfig {
+    pub outbound: ProtocolTypeSet,
+    pub inbound: ProtocolTypeSet,
+    pub family_global: AddressTypeSet,
+    pub public_internet_capabilities: Vec<FourCC>,
+}
+
 struct NetworkInner {
     network_needs_restart: bool,
     protocol_config: ProtocolConfig,
@@ -371,7 +379,6 @@ impl Network {
             };
 
             let family_global = supported_address_types;
-            let family_local = supported_address_types;
 
             let public_internet_capabilities = {
                 PUBLIC_INTERNET_CAPABILITIES
@@ -385,8 +392,6 @@ impl Network {
                 outbound,
                 inbound,
                 family_global,
-                family_local,
-                local_network_capabilities: vec![],
                 public_internet_capabilities,
             }
         };
@@ -396,7 +401,7 @@ impl Network {
         let mut editor_public_internet = self
             .unlocked_inner
             .routing_table
-            .edit_routing_domain(RoutingDomain::PublicInternet);
+            .edit_public_internet_routing_domain();
 
         // set up the routing table's network config
         // if we have static public dialinfo, upgrade our network class
@@ -409,8 +414,10 @@ impl Network {
         );
         editor_public_internet.set_network_class(Some(NetworkClass::WebApp));
 
-        // commit routing table edits
+        // commit routing domain edits
-        editor_public_internet.commit(true).await;
+        if editor_public_internet.commit(true).await {
+            editor_public_internet.publish();
+        }
 
         Ok(StartupDisposition::Success)
     }
@@ -459,14 +466,9 @@ impl Network {
 
         // Reset state
         let routing_table = self.routing_table();
-
-        // Drop all dial info
         routing_table
-            .edit_routing_domain(RoutingDomain::PublicInternet)
+            .edit_public_internet_routing_domain()
-            .clear_dial_info_details(None, None)
+            .shutdown()
-            .set_network_class(None)
-            .clear_relay_node()
-            .commit(true)
             .await;
 
         // Cancels all async background tasks by dropping join handles

veilid-core/src/network_manager/wasm/protocol/mod.rs

@@ -6,7 +6,6 @@ use std::io;
 
 #[derive(Debug)]
 pub(in crate::network_manager) enum ProtocolNetworkConnection {
-    #[allow(dead_code)]
     //Dummy(DummyNetworkConnection),
     Ws(ws::WebsocketNetworkConnection),
     //WebRTC(wrtc::WebRTCNetworkConnection),

veilid-core/src/routing_table/bucket_entry.rs

@@ -243,7 +243,6 @@ impl BucketEntryInner {
     }
 
     // Less is faster
-    #[allow(dead_code)]
     pub fn cmp_fastest(e1: &Self, e2: &Self) -> std::cmp::Ordering {
         // Lower latency to the front
         if let Some(e1_latency) = &e1.peer_stats.latency {
@@ -295,28 +294,18 @@ impl BucketEntryInner {
         }
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn sort_fastest_reliable_fn(
         cur_ts: Timestamp,
     ) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
         move |e1, e2| Self::cmp_fastest_reliable(cur_ts, e1, e2)
     }
 
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
-    pub fn clear_signed_node_info(&mut self, routing_domain: RoutingDomain) {
-        // Get the correct signed_node_info for the chosen routing domain
-        let opt_current_sni = match routing_domain {
-            RoutingDomain::LocalNetwork => &mut self.local_network.signed_node_info,
-            RoutingDomain::PublicInternet => &mut self.public_internet.signed_node_info,
-        };
-        *opt_current_sni = None;
-    }
-
     pub fn update_signed_node_info(
         &mut self,
         routing_domain: RoutingDomain,
         signed_node_info: SignedNodeInfo,
-    ) {
+    ) -> bool {
         // Get the correct signed_node_info for the chosen routing domain
         let opt_current_sni = match routing_domain {
             RoutingDomain::LocalNetwork => &mut self.local_network.signed_node_info,
@@ -341,11 +330,11 @@ impl BucketEntryInner {
                         self.updated_since_last_network_change = true;
                         self.make_not_dead(Timestamp::now());
                     }
-                    return;
+                    return false;
                 }
 
                 // See if anything has changed in this update beside the timestamp
-                if signed_node_info.node_info() != current_sni.node_info() {
+                if !signed_node_info.equivalent(current_sni) {
                     node_info_changed = true;
                 }
             }
@@ -369,6 +358,8 @@ impl BucketEntryInner {
         if node_info_changed {
             self.clear_last_flows_except_latest();
         }
+
+        node_info_changed
     }
 
     pub fn has_node_info(&self, routing_domain_set: RoutingDomainSet) -> bool {
@@ -425,7 +416,7 @@ impl BucketEntryInner {
         let node_ids = self.node_ids();
         opt_current_sni
             .as_ref()
-            .map(|s| PeerInfo::new(node_ids, *s.clone()))
+            .map(|s| PeerInfo::new(routing_domain, node_ids, *s.clone()))
     }
 
     pub fn best_routing_domain(
@@ -588,7 +579,7 @@ impl BucketEntryInner {
         self.envelope_support = envelope_support;
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn envelope_support(&self) -> Vec<u8> {
         self.envelope_support.clone()
     }

veilid-core/src/routing_table/debug.rs

@@ -55,6 +55,14 @@ impl RoutingTable {
         out
     }
 
+    pub(crate) fn debug_info_nodeid(&self) -> String {
+        let mut out = String::new();
+        for nid in self.unlocked_inner.node_ids().iter() {
+            out += &format!("{}\n", nid);
+        }
+        out
+    }
+
     pub(crate) fn debug_info_nodeinfo(&self) -> String {
         let mut out = String::new();
         let inner = self.inner.read();
@@ -94,13 +102,25 @@ impl RoutingTable {
         out
     }
 
-    pub(crate) fn debug_info_peerinfo(&self, routing_domain: RoutingDomain) -> String {
+    pub(crate) fn debug_info_peerinfo(
+        &self,
+        routing_domain: RoutingDomain,
+        published: bool,
+    ) -> String {
         let mut out = String::new();
-        out += &format!(
+        if published {
-            "{:?} PeerInfo:\n  {:#?}\n",
+            out += &format!(
-            routing_domain,
+                "{:?} Published PeerInfo:\n  {:#?}\n",
-            self.get_own_peer_info(routing_domain)
+                routing_domain,
-        );
+                self.get_published_peer_info(routing_domain)
+            );
+        } else {
+            out += &format!(
+                "{:?} Current PeerInfo:\n  {:#?}\n",
+                routing_domain,
+                self.get_current_peer_info(routing_domain)
+            );
+        }
         out
     }
 
@@ -113,7 +133,7 @@ impl RoutingTable {
                 PunishmentReason::InvalidFraming => "PFRAME",
                 PunishmentReason::FailedToDecodeOperation => "PDECOP",
                 PunishmentReason::WrongSenderPeerInfo => "PSPBAD",
-                PunishmentReason::FailedToVerifySenderPeerInfo => "PSPVER",
+                // PunishmentReason::FailedToVerifySenderPeerInfo => "PSPVER",
                 PunishmentReason::FailedToRegisterSenderPeerInfo => "PSPREG",
                 //
             },

veilid-core/src/routing_table/find_peers.rs

@@ -7,37 +7,33 @@ impl RoutingTable {
     #[instrument(level = "trace", target = "rtab", skip_all)]
     pub fn find_preferred_closest_peers(
         &self,
+        routing_domain: RoutingDomain,
         key: TypedKey,
         capabilities: &[Capability],
-    ) -> NetworkResult<Vec<PeerInfo>> {
+    ) -> NetworkResult<Vec<Arc<PeerInfo>>> {
-        if !self.has_valid_network_class(RoutingDomain::PublicInternet) {
-            // Our own node info is not yet available, drop this request.
-            return NetworkResult::service_unavailable(
-                "Not finding closest peers because our network class is still invalid",
-            );
-        }
         if Crypto::validate_crypto_kind(key.kind).is_err() {
             return NetworkResult::invalid_message("invalid crypto kind");
         }
 
+        let Some(published_peer_info) = self.get_published_peer_info(routing_domain) else {
+            return NetworkResult::service_unavailable(
+                "Not finding closest peers because our network class is still invalid",
+            );
+        };
+
         // find N nodes closest to the target node in our routing table
-        let own_peer_info = self.get_own_peer_info(RoutingDomain::PublicInternet);
         let filter = Box::new(
             |rti: &RoutingTableInner, opt_entry: Option<Arc<BucketEntry>>| {
-                // Ensure only things that are valid/signed in the PublicInternet domain are returned
+                // Ensure only things that are valid/signed in the chosen routing domain are returned
-                if !rti.filter_has_valid_signed_node_info(
+                if !rti.filter_has_valid_signed_node_info(routing_domain, true, opt_entry.clone()) {
-                    RoutingDomain::PublicInternet,
-                    true,
-                    opt_entry.clone(),
-                ) {
                     return false;
                 }
                 // Ensure capabilities are met
                 match opt_entry {
                     Some(entry) => entry.with(rti, |_rti, e| {
-                        e.has_all_capabilities(RoutingDomain::PublicInternet, capabilities)
+                        e.has_all_capabilities(routing_domain, capabilities)
                     }),
-                    None => own_peer_info
+                    None => published_peer_info
                         .signed_node_info()
                         .node_info()
                         .has_all_capabilities(capabilities),
@@ -57,7 +53,7 @@ impl RoutingTable {
             filters,
             // transform
             |rti, entry| {
-                rti.transform_to_peer_info(RoutingDomain::PublicInternet, &own_peer_info, entry)
+                rti.transform_to_peer_info(routing_domain, published_peer_info.clone(), entry)
             },
         ) {
             Ok(v) => v,
@@ -76,9 +72,10 @@ impl RoutingTable {
     #[instrument(level = "trace", target = "rtab", skip_all)]
     pub fn find_preferred_peers_closer_to_key(
         &self,
+        routing_domain: RoutingDomain,
         key: TypedKey,
         required_capabilities: Vec<Capability>,
-    ) -> NetworkResult<Vec<PeerInfo>> {
+    ) -> NetworkResult<Vec<Arc<PeerInfo>>> {
         // add node information for the requesting node to our routing table
         let crypto_kind = key.kind;
         let own_node_id = self.node_id(crypto_kind);
@@ -99,14 +96,12 @@ impl RoutingTable {
                 };
                 // Ensure only things that have a minimum set of capabilities are returned
                 entry.with(rti, |rti, e| {
-                    if !e
+                    if !e.has_all_capabilities(routing_domain, &required_capabilities) {
-                        .has_all_capabilities(RoutingDomain::PublicInternet, &required_capabilities)
-                    {
                         return false;
                     }
                     // Ensure only things that are valid/signed in the PublicInternet domain are returned
                     if !rti.filter_has_valid_signed_node_info(
-                        RoutingDomain::PublicInternet,
+                        routing_domain,
                         true,
                         Some(entry.clone()),
                     ) {
@@ -139,7 +134,7 @@ impl RoutingTable {
             // transform
             |rti, entry| {
                 entry.unwrap().with(rti, |_rti, e| {
-                    e.make_peer_info(RoutingDomain::PublicInternet).unwrap()
+                    Arc::new(e.make_peer_info(routing_domain).unwrap())
                 })
             },
         ) {
@@ -174,7 +169,7 @@ impl RoutingTable {
         vcrypto: CryptoSystemVersion,
         key_far: TypedKey,
         key_near: TypedKey,
-        peers: &[PeerInfo],
+        peers: &[Arc<PeerInfo>],
     ) -> EyreResult<bool> {
         let kind = vcrypto.kind();
 

veilid-core/src/routing_table/mod.rs

@@ -3,11 +3,8 @@ mod bucket_entry;
 mod debug;
 mod find_peers;
 mod node_ref;
-mod node_ref_filter;
 mod privacy;
 mod route_spec_store;
-mod routing_domain_editor;
-mod routing_domains;
 mod routing_table_inner;
 mod stats_accounting;
 mod tasks;
@@ -26,11 +23,8 @@ use hashlink::LruCache;
 
 pub(crate) use bucket_entry::*;
 pub(crate) use node_ref::*;
-pub(crate) use node_ref_filter::*;
 pub(crate) use privacy::*;
 pub(crate) use route_spec_store::*;
-pub(crate) use routing_domain_editor::*;
-pub(crate) use routing_domains::*;
 pub(crate) use routing_table_inner::*;
 pub(crate) use stats_accounting::*;
 
@@ -57,9 +51,6 @@ const ROUTING_TABLE: &str = "routing_table";
 const SERIALIZED_BUCKET_MAP: &[u8] = b"serialized_bucket_map";
 const CACHE_VALIDITY_KEY: &[u8] = b"cache_validity_key";
 
-// Critical sections
-const LOCK_TAG_TICK: &str = "TICK";
-
 type LowLevelProtocolPorts = BTreeSet<(LowLevelProtocolType, AddressType, u16)>;
 type ProtocolToPortMapping = BTreeMap<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>;
 #[derive(Clone, Debug)]
@@ -499,15 +490,6 @@ impl RoutingTable {
         Ok(())
     }
 
-    /// Set up the local network routing domain with our local routing table configuration
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
-    pub fn configure_local_network_routing_domain(&self, local_networks: Vec<(IpAddr, IpAddr)>) {
-        log_net!(debug "configure_local_network_routing_domain: {:#?}", local_networks);
-        self.inner
-            .write()
-            .configure_local_network_routing_domain(local_networks);
-    }
-
     /////////////////////////////////////
     /// Locked operations
 
@@ -519,7 +501,7 @@ impl RoutingTable {
         self.inner.read().route_spec_store.as_ref().unwrap().clone()
     }
 
-    pub fn relay_node(&self, domain: RoutingDomain) -> Option<NodeRef> {
+    pub fn relay_node(&self, domain: RoutingDomain) -> Option<FilteredNodeRef> {
         self.inner.read().relay_node(domain)
     }
 
@@ -541,13 +523,6 @@ impl RoutingTable {
             .all_filtered_dial_info_details(routing_domain_set, filter)
     }
 
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
-    pub fn ensure_dial_info_is_valid(&self, domain: RoutingDomain, dial_info: &DialInfo) -> bool {
-        self.inner
-            .read()
-            .ensure_dial_info_is_valid(domain, dial_info)
-    }
-
     pub fn signed_node_info_is_valid_in_routing_domain(
         &self,
         routing_domain: RoutingDomain,
@@ -562,8 +537,8 @@ impl RoutingTable {
     pub fn get_contact_method(
         &self,
         routing_domain: RoutingDomain,
-        peer_a: &PeerInfo,
+        peer_a: Arc<PeerInfo>,
-        peer_b: &PeerInfo,
+        peer_b: Arc<PeerInfo>,
         dial_info_filter: DialInfoFilter,
         sequencing: Sequencing,
         dif_sort: Option<Arc<DialInfoDetailSort>>,
@@ -578,14 +553,24 @@ impl RoutingTable {
         )
     }
 
-    #[instrument(level = "debug", skip(self))]
+    /// Edit the PublicInternet RoutingDomain
-    pub fn edit_routing_domain(&self, domain: RoutingDomain) -> RoutingDomainEditor {
+    pub fn edit_public_internet_routing_domain(&self) -> RoutingDomainEditorPublicInternet {
-        RoutingDomainEditor::new(self.clone(), domain)
+        RoutingDomainEditorPublicInternet::new(self.clone())
     }
 
-    /// Return a copy of our node's peerinfo
+    /// Edit the LocalNetwork RoutingDomain
-    pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo {
+    pub fn edit_local_network_routing_domain(&self) -> RoutingDomainEditorLocalNetwork {
-        self.inner.read().get_own_peer_info(routing_domain)
+        RoutingDomainEditorLocalNetwork::new(self.clone())
+    }
+
+    /// Return a copy of our node's peerinfo (may not yet be published)
+    pub fn get_published_peer_info(&self, routing_domain: RoutingDomain) -> Option<Arc<PeerInfo>> {
+        self.inner.read().get_published_peer_info(routing_domain)
+    }
+
+    /// Return a copy of our node's peerinfo (may not yet be published)
+    pub fn get_current_peer_info(&self, routing_domain: RoutingDomain) -> Arc<PeerInfo> {
+        self.inner.read().get_current_peer_info(routing_domain)
     }
 
     /// If we have a valid network class in this routing domain, then our 'NodeInfo' is valid
@@ -594,12 +579,8 @@ impl RoutingTable {
         self.inner.read().has_valid_network_class(routing_domain)
     }
 
-    /// Return our current node info timestamp
-    pub fn get_own_node_info_ts(&self, routing_domain: RoutingDomain) -> Timestamp {
-        self.inner.read().get_own_node_info_ts(routing_domain)
-    }
-
     /// Return the domain's currently registered network class
+    #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
     pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option<NetworkClass> {
         self.inner.read().get_network_class(routing_domain)
     }
@@ -633,7 +614,7 @@ impl RoutingTable {
         &self,
         routing_domain: RoutingDomain,
         cur_ts: Timestamp,
-    ) -> Vec<NodeRef> {
+    ) -> Vec<FilteredNodeRef> {
         self.inner
             .read()
             .get_nodes_needing_ping(self.clone(), routing_domain, cur_ts)
@@ -671,7 +652,7 @@ impl RoutingTable {
         node_id: TypedKey,
         routing_domain_set: RoutingDomainSet,
         dial_info_filter: DialInfoFilter,
-    ) -> EyreResult<Option<NodeRef>> {
+    ) -> EyreResult<Option<FilteredNodeRef>> {
         self.inner.read().lookup_and_filter_noderef(
             self.clone(),
             node_id,
@@ -686,16 +667,12 @@ impl RoutingTable {
     #[instrument(level = "trace", skip_all, err)]
     pub fn register_node_with_peer_info(
         &self,
-        routing_domain: RoutingDomain,
+        peer_info: Arc<PeerInfo>,
-        peer_info: PeerInfo,
         allow_invalid: bool,
-    ) -> EyreResult<NodeRef> {
+    ) -> EyreResult<FilteredNodeRef> {
-        self.inner.write().register_node_with_peer_info(
+        self.inner
-            self.clone(),
+            .write()
-            routing_domain,
+            .register_node_with_peer_info(self.clone(), peer_info, allow_invalid)
-            peer_info,
-            allow_invalid,
-        )
     }
 
     /// Shortcut function to add a node to our routing table if it doesn't exist
@@ -703,12 +680,14 @@ impl RoutingTable {
     #[instrument(level = "trace", skip_all, err)]
     pub fn register_node_with_existing_connection(
         &self,
+        routing_domain: RoutingDomain,
         node_id: TypedKey,
         flow: Flow,
         timestamp: Timestamp,
-    ) -> EyreResult<NodeRef> {
+    ) -> EyreResult<FilteredNodeRef> {
         self.inner.write().register_node_with_existing_connection(
             self.clone(),
+            routing_domain,
             node_id,
             flow,
             timestamp,
@@ -810,7 +789,7 @@ impl RoutingTable {
     }
 
     /// Makes a filter that finds nodes with a matching inbound dialinfo
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+    #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
     pub fn make_inbound_dial_info_entry_filter<'a>(
         routing_domain: RoutingDomain,
         dial_info_filter: DialInfoFilter,
@@ -864,14 +843,18 @@ impl RoutingTable {
         })
     }
 
-    pub fn find_fast_public_nodes_filtered(
+    pub fn find_fast_non_local_nodes_filtered(
         &self,
+        routing_domain: RoutingDomain,
         node_count: usize,
         filters: VecDeque<RoutingTableEntryFilter>,
     ) -> Vec<NodeRef> {
-        self.inner
+        self.inner.read().find_fast_non_local_nodes_filtered(
-            .read()
+            self.clone(),
-            .find_fast_public_nodes_filtered(self.clone(), node_count, filters)
+            routing_domain,
+            node_count,
+            filters,
+        )
     }
 
     /// Retrieve up to N of each type of protocol capable nodes for a single crypto kind
@@ -953,7 +936,7 @@ impl RoutingTable {
             protocol_types_len * 2 * max_per_type,
             filters,
             |_rti, entry: Option<Arc<BucketEntry>>| {
-                NodeRef::new(self.clone(), entry.unwrap().clone(), None)
+                NodeRef::new(self.clone(), entry.unwrap().clone())
             },
         )
     }
@@ -1018,28 +1001,22 @@ impl RoutingTable {
             .sort_and_clean_closest_noderefs(node_id, closest_nodes)
     }
 
-    #[instrument(level = "trace", skip(self, peers))]
+    #[instrument(level = "trace", skip(self, peer_info_list))]
-    pub fn register_find_node_answer(
+    pub fn register_nodes_with_peer_info_list(
         &self,
-        crypto_kind: CryptoKind,
+        peer_info_list: Vec<Arc<PeerInfo>>,
-        peers: Vec<PeerInfo>,
     ) -> Vec<NodeRef> {
         // Register nodes we'd found
-        let mut out = Vec::<NodeRef>::with_capacity(peers.len());
+        let mut out = Vec::<NodeRef>::with_capacity(peer_info_list.len());
-        for p in peers {
+        for p in peer_info_list {
-            // Ensure we're getting back nodes we asked for
-            if !p.node_ids().kinds().contains(&crypto_kind) {
-                continue;
-            }
-
             // Don't register our own node
             if self.matches_own_node_id(p.node_ids()) {
                 continue;
             }
 
             // Register the node if it's new
-            match self.register_node_with_peer_info(RoutingDomain::PublicInternet, p, false) {
+            match self.register_node_with_peer_info(p, false) {
-                Ok(nr) => out.push(nr),
+                Ok(nr) => out.push(nr.unfiltered()),
                 Err(e) => {
                     log_rtab!(debug "failed to register node with peer info from find node answer: {}", e);
                 }
@@ -1051,7 +1028,7 @@ impl RoutingTable {
     /// Finds nodes near a particular node id
     /// Ensures all returned nodes have a set of capabilities enabled
     #[instrument(level = "trace", skip(self), err)]
-    pub async fn find_node(
+    pub async fn find_nodes_close_to_node_id(
         &self,
         node_ref: NodeRef,
         node_id: TypedKey,
@@ -1062,33 +1039,38 @@ impl RoutingTable {
         let res = network_result_try!(
             rpc_processor
                 .clone()
-                .rpc_call_find_node(Destination::direct(node_ref), node_id, capabilities)
+                .rpc_call_find_node(
+                    Destination::direct(node_ref.default_filtered()),
+                    node_id,
+                    capabilities
+                )
                 .await?
         );
 
         // register nodes we'd found
         Ok(NetworkResult::value(
-            self.register_find_node_answer(node_id.kind, res.answer),
+            self.register_nodes_with_peer_info_list(res.answer),
         ))
     }
 
     /// Ask a remote node to list the nodes it has around the current node
     /// Ensures all returned nodes have a set of capabilities enabled
     #[instrument(level = "trace", skip(self), err)]
-    pub async fn find_self(
+    pub async fn find_nodes_close_to_self(
         &self,
         crypto_kind: CryptoKind,
         node_ref: NodeRef,
         capabilities: Vec<Capability>,
     ) -> EyreResult<NetworkResult<Vec<NodeRef>>> {
         let self_node_id = self.node_id(crypto_kind);
-        self.find_node(node_ref, self_node_id, capabilities).await
+        self.find_nodes_close_to_node_id(node_ref, self_node_id, capabilities)
+            .await
     }
 
     /// Ask a remote node to list the nodes it has around itself
     /// Ensures all returned nodes have a set of capabilities enabled
     #[instrument(level = "trace", skip(self), err)]
-    pub async fn find_target(
+    pub async fn find_nodes_close_to_node_ref(
         &self,
         crypto_kind: CryptoKind,
         node_ref: NodeRef,
@@ -1097,7 +1079,8 @@ impl RoutingTable {
         let Some(target_node_id) = node_ref.node_ids().get(crypto_kind) else {
             bail!("no target node ids for this crypto kind");
         };
-        self.find_node(node_ref, target_node_id, capabilities).await
+        self.find_nodes_close_to_node_id(node_ref, target_node_id, capabilities)
+            .await
     }
 
     /// Ask node to 'find node' on own node so we can get some more nodes near ourselves
@@ -1111,7 +1094,7 @@ impl RoutingTable {
         capabilities: Vec<Capability>,
     ) {
         // Ask node for nodes closest to our own node
-        let closest_nodes = network_result_value_or_log!(match self.find_self(crypto_kind, node_ref.clone(), capabilities.clone()).await {
+        let closest_nodes = network_result_value_or_log!(match self.find_nodes_close_to_self(crypto_kind, node_ref.clone(), capabilities.clone()).await {
             Err(e) => {
                 log_rtab!(error
                     "find_self failed for {:?}: {:?}",
@@ -1127,7 +1110,7 @@ impl RoutingTable {
         // Ask each node near us to find us as well
         if wide {
             for closest_nr in closest_nodes {
-                network_result_value_or_log!(match self.find_self(crypto_kind, closest_nr.clone(), capabilities.clone()).await {
+                network_result_value_or_log!(match self.find_nodes_close_to_self(crypto_kind, closest_nr.clone(), capabilities.clone()).await {
                     Err(e) => {
                         log_rtab!(error
                             "find_self failed for {:?}: {:?}",

veilid-core/src/routing_table/node_ref.rs

@@ -1,623 +0,0 @@
-use super::*;
-use crate::crypto::*;
-use alloc::fmt;
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-pub(crate) struct NodeRefBaseCommon {
-    routing_table: RoutingTable,
-    entry: Arc<BucketEntry>,
-    filter: Option<NodeRefFilter>,
-    sequencing: Sequencing,
-    #[cfg(feature = "tracking")]
-    track_id: usize,
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-pub(crate) trait NodeRefBase: Sized {
-    // Common field access
-    fn common(&self) -> &NodeRefBaseCommon;
-    fn common_mut(&mut self) -> &mut NodeRefBaseCommon;
-
-    // Comparators
-    fn same_entry<T: NodeRefBase>(&self, other: &T) -> bool {
-        Arc::ptr_eq(&self.common().entry, &other.common().entry)
-    }
-    fn same_bucket_entry(&self, entry: &Arc<BucketEntry>) -> bool {
-        Arc::ptr_eq(&self.common().entry, entry)
-    }
-
-    // Implementation-specific operators
-    fn operate<T, F>(&self, f: F) -> T
-    where
-        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T;
-    fn operate_mut<T, F>(&self, f: F) -> T
-    where
-        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T;
-
-    // Filtering
-    fn filter_ref(&self) -> Option<&NodeRefFilter> {
-        self.common().filter.as_ref()
-    }
-
-    fn take_filter(&mut self) -> Option<NodeRefFilter> {
-        self.common_mut().filter.take()
-    }
-
-    fn set_filter(&mut self, filter: Option<NodeRefFilter>) {
-        self.common_mut().filter = filter
-    }
-
-    fn set_sequencing(&mut self, sequencing: Sequencing) {
-        self.common_mut().sequencing = sequencing;
-    }
-    fn sequencing(&self) -> Sequencing {
-        self.common().sequencing
-    }
-
-    fn merge_filter(&mut self, filter: NodeRefFilter) {
-        let common_mut = self.common_mut();
-        if let Some(self_filter) = common_mut.filter.take() {
-            common_mut.filter = Some(self_filter.filtered(&filter));
-        } else {
-            common_mut.filter = Some(filter);
-        }
-    }
-
-    // fn is_filter_dead(&self) -> bool {
-    //     if let Some(filter) = &self.common().filter {
-    //         filter.is_dead()
-    //     } else {
-    //         false
-    //     }
-    // }
-
-    fn routing_domain_set(&self) -> RoutingDomainSet {
-        self.common()
-            .filter
-            .as_ref()
-            .map(|f| f.routing_domain_set)
-            .unwrap_or(RoutingDomainSet::all())
-    }
-
-    fn dial_info_filter(&self) -> DialInfoFilter {
-        self.common()
-            .filter
-            .as_ref()
-            .map(|f| f.dial_info_filter)
-            .unwrap_or(DialInfoFilter::all())
-    }
-
-    fn best_routing_domain(&self) -> Option<RoutingDomain> {
-        self.operate(|rti, e| {
-            e.best_routing_domain(
-                rti,
-                self.common()
-                    .filter
-                    .as_ref()
-                    .map(|f| f.routing_domain_set)
-                    .unwrap_or(RoutingDomainSet::all()),
-            )
-        })
-    }
-
-    // Accessors
-    fn routing_table(&self) -> RoutingTable {
-        self.common().routing_table.clone()
-    }
-    fn node_ids(&self) -> TypedKeyGroup {
-        self.operate(|_rti, e| e.node_ids())
-    }
-    fn best_node_id(&self) -> TypedKey {
-        self.operate(|_rti, e| e.best_node_id())
-    }
-    fn update_node_status(&self, routing_domain: RoutingDomain, node_status: NodeStatus) {
-        self.operate_mut(|_rti, e| {
-            e.update_node_status(routing_domain, node_status);
-        });
-    }
-    // fn envelope_support(&self) -> Vec<u8> {
-    //     self.operate(|_rti, e| e.envelope_support())
-    // }
-    fn add_envelope_version(&self, envelope_version: u8) {
-        self.operate_mut(|_rti, e| e.add_envelope_version(envelope_version))
-    }
-    // fn set_envelope_support(&self, envelope_support: Vec<u8>) {
-    //     self.operate_mut(|_rti, e| e.set_envelope_support(envelope_support))
-    // }
-    fn best_envelope_version(&self) -> Option<u8> {
-        self.operate(|_rti, e| e.best_envelope_version())
-    }
-    fn state_reason(&self, cur_ts: Timestamp) -> BucketEntryStateReason {
-        self.operate(|_rti, e| e.state_reason(cur_ts))
-    }
-    fn state(&self, cur_ts: Timestamp) -> BucketEntryState {
-        self.operate(|_rti, e| e.state(cur_ts))
-    }
-    fn peer_stats(&self) -> PeerStats {
-        self.operate(|_rti, e| e.peer_stats().clone())
-    }
-
-    // Per-RoutingDomain accessors
-    fn make_peer_info(&self, routing_domain: RoutingDomain) -> Option<PeerInfo> {
-        self.operate(|_rti, e| e.make_peer_info(routing_domain))
-    }
-    fn node_info(&self, routing_domain: RoutingDomain) -> Option<NodeInfo> {
-        self.operate(|_rti, e| e.node_info(routing_domain).cloned())
-    }
-    fn signed_node_info_has_valid_signature(&self, routing_domain: RoutingDomain) -> bool {
-        self.operate(|_rti, e| {
-            e.signed_node_info(routing_domain)
-                .map(|sni| sni.has_any_signature())
-                .unwrap_or(false)
-        })
-    }
-    fn node_info_ts(&self, routing_domain: RoutingDomain) -> Timestamp {
-        self.operate(|_rti, e| {
-            e.signed_node_info(routing_domain)
-                .map(|sni| sni.timestamp())
-                .unwrap_or(0u64.into())
-        })
-    }
-    fn has_seen_our_node_info_ts(
-        &self,
-        routing_domain: RoutingDomain,
-        our_node_info_ts: Timestamp,
-    ) -> bool {
-        self.operate(|_rti, e| e.has_seen_our_node_info_ts(routing_domain, our_node_info_ts))
-    }
-    fn set_seen_our_node_info_ts(&self, routing_domain: RoutingDomain, seen_ts: Timestamp) {
-        self.operate_mut(|_rti, e| e.set_seen_our_node_info_ts(routing_domain, seen_ts));
-    }
-    // fn network_class(&self, routing_domain: RoutingDomain) -> Option<NetworkClass> {
-    //     self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.network_class()))
-    // }
-    // fn outbound_protocols(&self, routing_domain: RoutingDomain) -> Option<ProtocolTypeSet> {
-    //     self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.outbound_protocols()))
-    // }
-    // fn address_types(&self, routing_domain: RoutingDomain) -> Option<AddressTypeSet> {
-    //     self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.address_types()))
-    // }
-    // fn node_info_outbound_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter {
-    //     let mut dif = DialInfoFilter::all();
-    //     if let Some(outbound_protocols) = self.outbound_protocols(routing_domain) {
-    //         dif = dif.with_protocol_type_set(outbound_protocols);
-    //     }
-    //     if let Some(address_types) = self.address_types(routing_domain) {
-    //         dif = dif.with_address_type_set(address_types);
-    //     }
-    //     dif
-    // }
-    fn relay(&self, routing_domain: RoutingDomain) -> EyreResult<Option<NodeRef>> {
-        self.operate_mut(|rti, e| {
-            let Some(sni) = e.signed_node_info(routing_domain) else {
-                return Ok(None);
-            };
-            let Some(rpi) = sni.relay_peer_info() else {
-                return Ok(None);
-            };
-            // If relay is ourselves, then return None, because we can't relay through ourselves
-            // and to contact this node we should have had an existing inbound connection
-            if rti.unlocked_inner.matches_own_node_id(rpi.node_ids()) {
-                bail!("Can't relay though ourselves");
-            }
-
-            // Register relay node and return noderef
-            let nr =
-                rti.register_node_with_peer_info(self.routing_table(), routing_domain, rpi, false)?;
-            Ok(Some(nr))
-        })
-    }
-
-    // Filtered accessors
-    fn first_filtered_dial_info_detail(&self) -> Option<DialInfoDetail> {
-        let routing_domain_set = self.routing_domain_set();
-        let dial_info_filter = self.dial_info_filter();
-        let sequencing = self.common().sequencing;
-        let (ordered, dial_info_filter) = dial_info_filter.with_sequencing(sequencing);
-
-        let sort = if ordered {
-            Some(DialInfoDetail::ordered_sequencing_sort)
-        } else {
-            None
-        };
-
-        if dial_info_filter.is_dead() {
-            return None;
-        }
-
-        let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter);
-
-        self.operate(|_rt, e| {
-            for routing_domain in routing_domain_set {
-                if let Some(ni) = e.node_info(routing_domain) {
-                    if let Some(did) = ni.first_filtered_dial_info_detail(sort, filter) {
-                        return Some(did);
-                    }
-                }
-            }
-            None
-        })
-    }
-
-    fn all_filtered_dial_info_details(&self) -> Vec<DialInfoDetail> {
-        let routing_domain_set = self.routing_domain_set();
-        let dial_info_filter = self.dial_info_filter();
-
-        let (sort, dial_info_filter) = match self.common().sequencing {
-            Sequencing::NoPreference => (None, dial_info_filter),
-            Sequencing::PreferOrdered => (
-                Some(DialInfoDetail::ordered_sequencing_sort),
-                dial_info_filter,
-            ),
-            Sequencing::EnsureOrdered => (
-                Some(DialInfoDetail::ordered_sequencing_sort),
-                dial_info_filter.filtered(
-                    &DialInfoFilter::all().with_protocol_type_set(ProtocolType::all_ordered_set()),
-                ),
-            ),
-        };
-
-        let mut out = Vec::new();
-        self.operate(|_rt, e| {
-            for routing_domain in routing_domain_set {
-                if let Some(ni) = e.node_info(routing_domain) {
-                    let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter);
-                    if let Some(did) = ni.first_filtered_dial_info_detail(sort, filter) {
-                        out.push(did);
-                    }
-                }
-            }
-        });
-        out.remove_duplicates();
-        out
-    }
-
-    /// Get the most recent 'last connection' to this node
-    /// Filtered first and then sorted by ordering preference and then by most recent
-    fn last_flow(&self) -> Option<Flow> {
-        self.operate(|rti, e| {
-            // apply sequencing to filter and get sort
-            let sequencing = self.common().sequencing;
-            let filter = self.common().filter.unwrap_or_default();
-            let (ordered, filter) = filter.with_sequencing(sequencing);
-            let mut last_connections = e.last_flows(rti, true, filter);
-
-            if ordered {
-                last_connections.sort_by(|a, b| {
-                    ProtocolType::ordered_sequencing_sort(a.0.protocol_type(), b.0.protocol_type())
-                });
-            }
-
-            last_connections.first().map(|x| x.0)
-        })
-    }
-
-    fn clear_last_connections(&self) {
-        self.operate_mut(|_rti, e| e.clear_last_flows())
-    }
-
-    fn set_last_flow(&self, flow: Flow, ts: Timestamp) {
-        self.operate_mut(|rti, e| {
-            e.set_last_flow(flow, ts);
-            rti.touch_recent_peer(e.best_node_id(), flow);
-        })
-    }
-
-    fn clear_last_connection(&self, flow: Flow) {
-        self.operate_mut(|_rti, e| {
-            e.remove_last_flow(flow);
-        })
-    }
-
-    fn has_any_dial_info(&self) -> bool {
-        self.operate(|_rti, e| {
-            for rtd in RoutingDomain::all() {
-                if let Some(sni) = e.signed_node_info(rtd) {
-                    if sni.has_any_dial_info() {
-                        return true;
-                    }
-                }
-            }
-            false
-        })
-    }
-
-    fn report_protected_connection_dropped(&self) {
-        self.stats_failed_to_send(Timestamp::now(), false);
-    }
-
-    fn report_failed_route_test(&self) {
-        self.stats_failed_to_send(Timestamp::now(), false);
-    }
-
-    fn stats_question_sent(&self, ts: Timestamp, bytes: ByteCount, expects_answer: bool) {
-        self.operate_mut(|rti, e| {
-            rti.transfer_stats_accounting().add_up(bytes);
-            e.question_sent(ts, bytes, expects_answer);
-        })
-    }
-    fn stats_question_rcvd(&self, ts: Timestamp, bytes: ByteCount) {
-        self.operate_mut(|rti, e| {
-            rti.transfer_stats_accounting().add_down(bytes);
-            e.question_rcvd(ts, bytes);
-        })
-    }
-    fn stats_answer_sent(&self, bytes: ByteCount) {
-        self.operate_mut(|rti, e| {
-            rti.transfer_stats_accounting().add_up(bytes);
-            e.answer_sent(bytes);
-        })
-    }
-    fn stats_answer_rcvd(&self, send_ts: Timestamp, recv_ts: Timestamp, bytes: ByteCount) {
-        self.operate_mut(|rti, e| {
-            rti.transfer_stats_accounting().add_down(bytes);
-            rti.latency_stats_accounting()
-                .record_latency(recv_ts.saturating_sub(send_ts));
-            e.answer_rcvd(send_ts, recv_ts, bytes);
-        })
-    }
-    fn stats_question_lost(&self) {
-        self.operate_mut(|_rti, e| {
-            e.question_lost();
-        })
-    }
-    fn stats_failed_to_send(&self, ts: Timestamp, expects_answer: bool) {
-        self.operate_mut(|_rti, e| {
-            e.failed_to_send(ts, expects_answer);
-        })
-    }
-}
-
-////////////////////////////////////////////////////////////////////////////////////
-
-/// Reference to a routing table entry
-/// Keeps entry in the routing table until all references are gone
-pub(crate) struct NodeRef {
-    common: NodeRefBaseCommon,
-}
-
-impl NodeRef {
-    pub fn new(
-        routing_table: RoutingTable,
-        entry: Arc<BucketEntry>,
-        filter: Option<NodeRefFilter>,
-    ) -> Self {
-        entry.ref_count.fetch_add(1u32, Ordering::AcqRel);
-
-        Self {
-            common: NodeRefBaseCommon {
-                routing_table,
-                entry,
-                filter,
-                sequencing: Sequencing::NoPreference,
-                #[cfg(feature = "tracking")]
-                track_id: entry.track(),
-            },
-        }
-    }
-
-    pub fn filtered_clone(&self, filter: NodeRefFilter) -> Self {
-        let mut out = self.clone();
-        out.merge_filter(filter);
-        out
-    }
-
-    pub fn locked<'a>(&self, rti: &'a RoutingTableInner) -> NodeRefLocked<'a> {
-        NodeRefLocked::new(rti, self.clone())
-    }
-    pub fn locked_mut<'a>(&self, rti: &'a mut RoutingTableInner) -> NodeRefLockedMut<'a> {
-        NodeRefLockedMut::new(rti, self.clone())
-    }
-}
-
-impl NodeRefBase for NodeRef {
-    fn common(&self) -> &NodeRefBaseCommon {
-        &self.common
-    }
-
-    fn common_mut(&mut self) -> &mut NodeRefBaseCommon {
-        &mut self.common
-    }
-
-    fn operate<T, F>(&self, f: F) -> T
-    where
-        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T,
-    {
-        let inner = &*self.common.routing_table.inner.read();
-        self.common.entry.with(inner, f)
-    }
-
-    fn operate_mut<T, F>(&self, f: F) -> T
-    where
-        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T,
-    {
-        let inner = &mut *self.common.routing_table.inner.write();
-        self.common.entry.with_mut(inner, f)
-    }
-}
-
-impl Clone for NodeRef {
-    fn clone(&self) -> Self {
-        self.common
-            .entry
-            .ref_count
-            .fetch_add(1u32, Ordering::AcqRel);
-
-        Self {
-            common: NodeRefBaseCommon {
-                routing_table: self.common.routing_table.clone(),
-                entry: self.common.entry.clone(),
-                filter: self.common.filter,
-                sequencing: self.common.sequencing,
-                #[cfg(feature = "tracking")]
-                track_id: self.common.entry.write().track(),
-            },
-        }
-    }
-}
-
-impl fmt::Display for NodeRef {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.common.entry.with_inner(|e| e.best_node_id()))
-    }
-}
-
-impl fmt::Debug for NodeRef {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("NodeRef")
-            .field("node_ids", &self.common.entry.with_inner(|e| e.node_ids()))
-            .field("filter", &self.common.filter)
-            .field("sequencing", &self.common.sequencing)
-            .finish()
-    }
-}
-
-impl Drop for NodeRef {
-    fn drop(&mut self) {
-        #[cfg(feature = "tracking")]
-        self.common.entry.write().untrack(self.track_id);
-
-        // drop the noderef and queue a bucket kick if it was the last one
-        let new_ref_count = self
-            .common
-            .entry
-            .ref_count
-            .fetch_sub(1u32, Ordering::AcqRel)
-            - 1;
-        if new_ref_count == 0 {
-            // get node ids with inner unlocked because nothing could be referencing this entry now
-            // and we don't know when it will get dropped, possibly inside a lock
-            let node_ids = self.common().entry.with_inner(|e| e.node_ids());
-            self.common.routing_table.queue_bucket_kicks(node_ids);
-        }
-    }
-}
-
-////////////////////////////////////////////////////////////////////////////////////
-
-/// Locked reference to a routing table entry
-/// For internal use inside the RoutingTable module where you have
-/// already locked a RoutingTableInner
-/// Keeps entry in the routing table until all references are gone
-pub(crate) struct NodeRefLocked<'a> {
-    inner: Mutex<&'a RoutingTableInner>,
-    nr: NodeRef,
-}
-
-impl<'a> NodeRefLocked<'a> {
-    pub fn new(inner: &'a RoutingTableInner, nr: NodeRef) -> Self {
-        Self {
-            inner: Mutex::new(inner),
-            nr,
-        }
-    }
-
-    pub fn unlocked(&self) -> NodeRef {
-        self.nr.clone()
-    }
-}
-
-impl<'a> NodeRefBase for NodeRefLocked<'a> {
-    fn common(&self) -> &NodeRefBaseCommon {
-        &self.nr.common
-    }
-
-    fn common_mut(&mut self) -> &mut NodeRefBaseCommon {
-        &mut self.nr.common
-    }
-
-    fn operate<T, F>(&self, f: F) -> T
-    where
-        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T,
-    {
-        let inner = &*self.inner.lock();
-        self.nr.common.entry.with(inner, f)
-    }
-
-    fn operate_mut<T, F>(&self, _f: F) -> T
-    where
-        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T,
-    {
-        panic!("need to locked_mut() for this operation")
-    }
-}
-
-impl<'a> fmt::Display for NodeRefLocked<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.nr)
-    }
-}
-
-impl<'a> fmt::Debug for NodeRefLocked<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("NodeRefLocked")
-            .field("nr", &self.nr)
-            .finish()
-    }
-}
-
-////////////////////////////////////////////////////////////////////////////////////
-
-/// Mutable locked reference to a routing table entry
-/// For internal use inside the RoutingTable module where you have
-/// already locked a RoutingTableInner
-/// Keeps entry in the routing table until all references are gone
-pub(crate) struct NodeRefLockedMut<'a> {
-    inner: Mutex<&'a mut RoutingTableInner>,
-    nr: NodeRef,
-}
-
-impl<'a> NodeRefLockedMut<'a> {
-    pub fn new(inner: &'a mut RoutingTableInner, nr: NodeRef) -> Self {
-        Self {
-            inner: Mutex::new(inner),
-            nr,
-        }
-    }
-
-    // pub fn unlocked(&self) -> NodeRef {
-    //     self.nr.clone()
-    // }
-}
-
-impl<'a> NodeRefBase for NodeRefLockedMut<'a> {
-    fn common(&self) -> &NodeRefBaseCommon {
-        &self.nr.common
-    }
-
-    fn common_mut(&mut self) -> &mut NodeRefBaseCommon {
-        &mut self.nr.common
-    }
-
-    fn operate<T, F>(&self, f: F) -> T
-    where
-        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T,
-    {
-        let inner = &*self.inner.lock();
-        self.nr.common.entry.with(inner, f)
-    }
-
-    fn operate_mut<T, F>(&self, f: F) -> T
-    where
-        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T,
-    {
-        let inner = &mut *self.inner.lock();
-        self.nr.common.entry.with_mut(inner, f)
-    }
-}
-
-impl<'a> fmt::Display for NodeRefLockedMut<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.nr)
-    }
-}
-
-impl<'a> fmt::Debug for NodeRefLockedMut<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("NodeRefLockedMut")
-            .field("nr", &self.nr)
-            .finish()
-    }
-}

veilid-core/src/routing_table/node_ref/filtered_node_ref.rs

@@ -0,0 +1,168 @@
+use super::*;
+
+pub struct FilteredNodeRef {
+    routing_table: RoutingTable,
+    entry: Arc<BucketEntry>,
+    filter: NodeRefFilter,
+    sequencing: Sequencing,
+    #[cfg(feature = "tracking")]
+    track_id: usize,
+}
+
+impl FilteredNodeRef {
+    pub fn new(
+        routing_table: RoutingTable,
+        entry: Arc<BucketEntry>,
+        filter: NodeRefFilter,
+        sequencing: Sequencing,
+    ) -> Self {
+        entry.ref_count.fetch_add(1u32, Ordering::AcqRel);
+
+        Self {
+            routing_table,
+            entry,
+            filter,
+            sequencing,
+            #[cfg(feature = "tracking")]
+            track_id: entry.track(),
+        }
+    }
+
+    pub fn unfiltered(&self) -> NodeRef {
+        NodeRef::new(self.routing_table.clone(), self.entry.clone())
+    }
+
+    pub fn filtered_clone(&self, filter: NodeRefFilter) -> FilteredNodeRef {
+        let mut out = self.clone();
+        out.merge_filter(filter);
+        out
+    }
+
+    pub fn sequencing_clone(&self, sequencing: Sequencing) -> FilteredNodeRef {
+        FilteredNodeRef::new(
+            self.routing_table.clone(),
+            self.entry.clone(),
+            self.filter(),
+            sequencing,
+        )
+    }
+
+    pub fn locked<'a>(&self, rti: &'a RoutingTableInner) -> LockedFilteredNodeRef<'a> {
+        LockedFilteredNodeRef::new(rti, self.clone())
+    }
+
+    #[expect(dead_code)]
+    pub fn locked_mut<'a>(&self, rti: &'a mut RoutingTableInner) -> LockedMutFilteredNodeRef<'a> {
+        LockedMutFilteredNodeRef::new(rti, self.clone())
+    }
+
+    pub fn set_filter(&mut self, filter: NodeRefFilter) {
+        self.filter = filter
+    }
+
+    pub fn merge_filter(&mut self, filter: NodeRefFilter) {
+        self.filter = self.filter.filtered(&filter);
+    }
+
+    pub fn set_sequencing(&mut self, sequencing: Sequencing) {
+        self.sequencing = sequencing;
+    }
+}
+
+impl NodeRefAccessorsTrait for FilteredNodeRef {
+    fn routing_table(&self) -> RoutingTable {
+        self.routing_table.clone()
+    }
+    fn entry(&self) -> Arc<BucketEntry> {
+        self.entry.clone()
+    }
+
+    fn sequencing(&self) -> Sequencing {
+        self.sequencing
+    }
+
+    fn routing_domain_set(&self) -> RoutingDomainSet {
+        self.filter.routing_domain_set
+    }
+
+    fn filter(&self) -> NodeRefFilter {
+        self.filter
+    }
+
+    fn take_filter(&mut self) -> NodeRefFilter {
+        let f = self.filter;
+        self.filter = NodeRefFilter::new();
+        f
+    }
+
+    fn dial_info_filter(&self) -> DialInfoFilter {
+        self.filter.dial_info_filter
+    }
+}
+
+impl NodeRefOperateTrait for FilteredNodeRef {
+    fn operate<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T,
+    {
+        let inner = &*self.routing_table.inner.read();
+        self.entry.with(inner, f)
+    }
+
+    fn operate_mut<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T,
+    {
+        let inner = &mut *self.routing_table.inner.write();
+        self.entry.with_mut(inner, f)
+    }
+}
+
+impl NodeRefCommonTrait for FilteredNodeRef {}
+
+impl Clone for FilteredNodeRef {
+    fn clone(&self) -> Self {
+        self.entry.ref_count.fetch_add(1u32, Ordering::AcqRel);
+
+        Self {
+            routing_table: self.routing_table.clone(),
+            entry: self.entry.clone(),
+            filter: self.filter,
+            sequencing: self.sequencing,
+            #[cfg(feature = "tracking")]
+            track_id: self.entry.write().track(),
+        }
+    }
+}
+
+impl fmt::Display for FilteredNodeRef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.entry.with_inner(|e| e.best_node_id()))
+    }
+}
+
+impl fmt::Debug for FilteredNodeRef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("FilteredNodeRef")
+            .field("node_ids", &self.entry.with_inner(|e| e.node_ids()))
+            .field("filter", &self.filter)
+            .field("sequencing", &self.sequencing)
+            .finish()
+    }
+}
+
+impl Drop for FilteredNodeRef {
+    fn drop(&mut self) {
+        #[cfg(feature = "tracking")]
+        self.entry.write().untrack(self.track_id);
+
+        // drop the noderef and queue a bucket kick if it was the last one
+        let new_ref_count = self.entry.ref_count.fetch_sub(1u32, Ordering::AcqRel) - 1;
+        if new_ref_count == 0 {
+            // get node ids with inner unlocked because nothing could be referencing this entry now
+            // and we don't know when it will get dropped, possibly inside a lock
+            let node_ids = self.entry.with_inner(|e| e.node_ids());
+            self.routing_table.queue_bucket_kicks(node_ids);
+        }
+    }
+}

veilid-core/src/routing_table/node_ref/mod.rs

@@ -0,0 +1,184 @@
+mod filtered_node_ref;
+mod node_ref_filter;
+mod node_ref_lock;
+mod node_ref_lock_mut;
+mod traits;
+
+use super::*;
+
+pub use filtered_node_ref::*;
+pub use node_ref_filter::*;
+pub use node_ref_lock::*;
+pub use node_ref_lock_mut::*;
+pub use traits::*;
+
+///////////////////////////////////////////////////////////////////////////
+// Default NodeRef
+
+pub struct NodeRef {
+    routing_table: RoutingTable,
+    entry: Arc<BucketEntry>,
+    #[cfg(feature = "tracking")]
+    track_id: usize,
+}
+
+impl NodeRef {
+    pub fn new(routing_table: RoutingTable, entry: Arc<BucketEntry>) -> Self {
+        entry.ref_count.fetch_add(1u32, Ordering::AcqRel);
+
+        Self {
+            routing_table,
+            entry,
+            #[cfg(feature = "tracking")]
+            track_id: entry.track(),
+        }
+    }
+
+    pub fn default_filtered(&self) -> FilteredNodeRef {
+        FilteredNodeRef::new(
+            self.routing_table.clone(),
+            self.entry.clone(),
+            NodeRefFilter::new(),
+            Sequencing::default(),
+        )
+    }
+
+    pub fn sequencing_filtered(&self, sequencing: Sequencing) -> FilteredNodeRef {
+        FilteredNodeRef::new(
+            self.routing_table.clone(),
+            self.entry.clone(),
+            NodeRefFilter::new(),
+            sequencing,
+        )
+    }
+
+    pub fn routing_domain_filtered<R: Into<RoutingDomainSet>>(
+        &self,
+        routing_domain_set: R,
+    ) -> FilteredNodeRef {
+        FilteredNodeRef::new(
+            self.routing_table.clone(),
+            self.entry.clone(),
+            NodeRefFilter::new().with_routing_domain_set(routing_domain_set.into()),
+            Sequencing::default(),
+        )
+    }
+
+    pub fn custom_filtered(&self, filter: NodeRefFilter) -> FilteredNodeRef {
+        FilteredNodeRef::new(
+            self.routing_table.clone(),
+            self.entry.clone(),
+            filter,
+            Sequencing::default(),
+        )
+    }
+
+    #[expect(dead_code)]
+    pub fn dial_info_filtered(&self, filter: DialInfoFilter) -> FilteredNodeRef {
+        FilteredNodeRef::new(
+            self.routing_table.clone(),
+            self.entry.clone(),
+            NodeRefFilter::new().with_dial_info_filter(filter),
+            Sequencing::default(),
+        )
+    }
+
+    pub fn locked<'a>(&self, rti: &'a RoutingTableInner) -> LockedNodeRef<'a> {
+        LockedNodeRef::new(rti, self.clone())
+    }
+    pub fn locked_mut<'a>(&self, rti: &'a mut RoutingTableInner) -> LockedMutNodeRef<'a> {
+        LockedMutNodeRef::new(rti, self.clone())
+    }
+}
+
+impl NodeRefAccessorsTrait for NodeRef {
+    fn routing_table(&self) -> RoutingTable {
+        self.routing_table.clone()
+    }
+    fn entry(&self) -> Arc<BucketEntry> {
+        self.entry.clone()
+    }
+
+    fn sequencing(&self) -> Sequencing {
+        Sequencing::NoPreference
+    }
+
+    fn routing_domain_set(&self) -> RoutingDomainSet {
+        RoutingDomainSet::all()
+    }
+
+    fn filter(&self) -> NodeRefFilter {
+        NodeRefFilter::new()
+    }
+
+    fn take_filter(&mut self) -> NodeRefFilter {
+        NodeRefFilter::new()
+    }
+
+    fn dial_info_filter(&self) -> DialInfoFilter {
+        DialInfoFilter::all()
+    }
+}
+
+impl NodeRefOperateTrait for NodeRef {
+    fn operate<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T,
+    {
+        let inner = &*self.routing_table.inner.read();
+        self.entry.with(inner, f)
+    }
+
+    fn operate_mut<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T,
+    {
+        let inner = &mut *self.routing_table.inner.write();
+        self.entry.with_mut(inner, f)
+    }
+}
+
+impl NodeRefCommonTrait for NodeRef {}
+
+impl Clone for NodeRef {
+    fn clone(&self) -> Self {
+        self.entry.ref_count.fetch_add(1u32, Ordering::AcqRel);
+
+        Self {
+            routing_table: self.routing_table.clone(),
+            entry: self.entry.clone(),
+            #[cfg(feature = "tracking")]
+            track_id: self.entry.write().track(),
+        }
+    }
+}
+
+impl fmt::Display for NodeRef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.entry.with_inner(|e| e.best_node_id()))
+    }
+}
+
+impl fmt::Debug for NodeRef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("NodeRef")
+            .field("node_ids", &self.entry.with_inner(|e| e.node_ids()))
+            .finish()
+    }
+}
+
+impl Drop for NodeRef {
+    fn drop(&mut self) {
+        #[cfg(feature = "tracking")]
+        self.entry.write().untrack(self.track_id);
+
+        // drop the noderef and queue a bucket kick if it was the last one
+        let new_ref_count = self.entry.ref_count.fetch_sub(1u32, Ordering::AcqRel) - 1;
+        if new_ref_count == 0 {
+            // get node ids with inner unlocked because nothing could be referencing this entry now
+            // and we don't know when it will get dropped, possibly inside a lock
+            let node_ids = self.entry.with_inner(|e| e.node_ids());
+            self.routing_table.queue_bucket_kicks(node_ids);
+        }
+    }
+}

veilid-core/src/routing_table/node_ref/node_ref_filter.rs

@@ -35,7 +35,7 @@ impl NodeRefFilter {
         self.dial_info_filter = self.dial_info_filter.with_protocol_type(protocol_type);
         self
     }
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn with_protocol_type_set(mut self, protocol_set: ProtocolTypeSet) -> Self {
         self.dial_info_filter = self.dial_info_filter.with_protocol_type_set(protocol_set);
         self
@@ -44,7 +44,7 @@ impl NodeRefFilter {
         self.dial_info_filter = self.dial_info_filter.with_address_type(address_type);
         self
     }
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn with_address_type_set(mut self, address_set: AddressTypeSet) -> Self {
         self.dial_info_filter = self.dial_info_filter.with_address_type_set(address_set);
         self
@@ -56,12 +56,12 @@ impl NodeRefFilter {
             .filtered(&other_filter.dial_info_filter);
         self
     }
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn is_dead(&self) -> bool {
         self.dial_info_filter.is_dead() || self.routing_domain_set.is_empty()
     }
-    pub fn with_sequencing(mut self, sequencing: Sequencing) -> (bool, Self) {
+    pub fn apply_sequencing(mut self, sequencing: Sequencing) -> (bool, Self) {
-        let (ordered, dif) = self.dial_info_filter.with_sequencing(sequencing);
+        let (ordered, dif) = self.dial_info_filter.apply_sequencing(sequencing);
         self.dial_info_filter = dif;
         (ordered, self)
     }

veilid-core/src/routing_table/node_ref/node_ref_lock.rs

@@ -0,0 +1,102 @@
+use super::*;
+
+pub type LockedNodeRef<'a> = NodeRefLock<'a, NodeRef>;
+pub type LockedFilteredNodeRef<'a> = NodeRefLock<'a, FilteredNodeRef>;
+
+/// Locked reference to a routing table entry
+/// For internal use inside the RoutingTable module where you have
+/// already locked a RoutingTableInner
+/// Keeps entry in the routing table until all references are gone
+pub struct NodeRefLock<
+    'a,
+    N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone,
+> {
+    inner: Mutex<&'a RoutingTableInner>,
+    nr: N,
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    NodeRefLock<'a, N>
+{
+    pub fn new(inner: &'a RoutingTableInner, nr: N) -> Self {
+        Self {
+            inner: Mutex::new(inner),
+            nr,
+        }
+    }
+
+    pub fn unlocked(&self) -> N {
+        self.nr.clone()
+    }
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    NodeRefAccessorsTrait for NodeRefLock<'a, N>
+{
+    fn routing_table(&self) -> RoutingTable {
+        self.nr.routing_table()
+    }
+    fn entry(&self) -> Arc<BucketEntry> {
+        self.nr.entry()
+    }
+
+    fn sequencing(&self) -> Sequencing {
+        self.nr.sequencing()
+    }
+
+    fn routing_domain_set(&self) -> RoutingDomainSet {
+        self.nr.routing_domain_set()
+    }
+
+    fn filter(&self) -> NodeRefFilter {
+        self.nr.filter()
+    }
+
+    fn take_filter(&mut self) -> NodeRefFilter {
+        self.nr.take_filter()
+    }
+
+    fn dial_info_filter(&self) -> DialInfoFilter {
+        self.nr.dial_info_filter()
+    }
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    NodeRefOperateTrait for NodeRefLock<'a, N>
+{
+    fn operate<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T,
+    {
+        let inner = &*self.inner.lock();
+        self.nr.entry().with(inner, f)
+    }
+
+    fn operate_mut<T, F>(&self, _f: F) -> T
+    where
+        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T,
+    {
+        panic!("need to locked_mut() for this operation")
+    }
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    NodeRefCommonTrait for NodeRefLock<'a, N>
+{
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    fmt::Display for NodeRefLock<'a, N>
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.nr)
+    }
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    fmt::Debug for NodeRefLock<'a, N>
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("NodeRefLock").field("nr", &self.nr).finish()
+    }
+}

veilid-core/src/routing_table/node_ref/node_ref_lock_mut.rs

@@ -0,0 +1,106 @@
+use super::*;
+
+pub type LockedMutNodeRef<'a> = NodeRefLockMut<'a, NodeRef>;
+pub type LockedMutFilteredNodeRef<'a> = NodeRefLockMut<'a, FilteredNodeRef>;
+
+/// Mutable locked reference to a routing table entry
+/// For internal use inside the RoutingTable module where you have
+/// already locked a RoutingTableInner
+/// Keeps entry in the routing table until all references are gone
+pub struct NodeRefLockMut<
+    'a,
+    N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone,
+> {
+    inner: Mutex<&'a mut RoutingTableInner>,
+    nr: N,
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    NodeRefLockMut<'a, N>
+{
+    pub fn new(inner: &'a mut RoutingTableInner, nr: N) -> Self {
+        Self {
+            inner: Mutex::new(inner),
+            nr,
+        }
+    }
+
+    #[expect(dead_code)]
+    pub fn unlocked(&self) -> N {
+        self.nr.clone()
+    }
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    NodeRefAccessorsTrait for NodeRefLockMut<'a, N>
+{
+    fn routing_table(&self) -> RoutingTable {
+        self.nr.routing_table()
+    }
+    fn entry(&self) -> Arc<BucketEntry> {
+        self.nr.entry()
+    }
+
+    fn sequencing(&self) -> Sequencing {
+        self.nr.sequencing()
+    }
+
+    fn routing_domain_set(&self) -> RoutingDomainSet {
+        self.nr.routing_domain_set()
+    }
+
+    fn filter(&self) -> NodeRefFilter {
+        self.nr.filter()
+    }
+
+    fn take_filter(&mut self) -> NodeRefFilter {
+        self.nr.take_filter()
+    }
+
+    fn dial_info_filter(&self) -> DialInfoFilter {
+        self.nr.dial_info_filter()
+    }
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    NodeRefOperateTrait for NodeRefLockMut<'a, N>
+{
+    fn operate<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T,
+    {
+        let inner = &*self.inner.lock();
+        self.nr.entry().with(inner, f)
+    }
+
+    fn operate_mut<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T,
+    {
+        let inner = &mut *self.inner.lock();
+        self.nr.entry().with_mut(inner, f)
+    }
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    NodeRefCommonTrait for NodeRefLockMut<'a, N>
+{
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    fmt::Display for NodeRefLockMut<'a, N>
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.nr)
+    }
+}
+
+impl<'a, N: NodeRefAccessorsTrait + NodeRefOperateTrait + fmt::Debug + fmt::Display + Clone>
+    fmt::Debug for NodeRefLockMut<'a, N>
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("NodeRefLockMut")
+            .field("nr", &self.nr)
+            .finish()
+    }
+}

veilid-core/src/routing_table/node_ref/traits.rs

@@ -0,0 +1,300 @@
+use super::*;
+
+// Field accessors
+pub trait NodeRefAccessorsTrait {
+    fn routing_table(&self) -> RoutingTable;
+    fn entry(&self) -> Arc<BucketEntry>;
+    fn sequencing(&self) -> Sequencing;
+    fn routing_domain_set(&self) -> RoutingDomainSet;
+    fn filter(&self) -> NodeRefFilter;
+    fn take_filter(&mut self) -> NodeRefFilter;
+    fn dial_info_filter(&self) -> DialInfoFilter;
+    // fn node_info_outbound_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter;
+    // fn is_filter_dead(&self) -> bool;
+}
+
+// Operate on entry
+pub trait NodeRefOperateTrait {
+    fn operate<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T;
+    fn operate_mut<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T;
+}
+
+// Common Operations
+pub trait NodeRefCommonTrait: NodeRefAccessorsTrait + NodeRefOperateTrait {
+    fn same_entry<T: NodeRefAccessorsTrait>(&self, other: &T) -> bool {
+        Arc::ptr_eq(&self.entry(), &other.entry())
+    }
+
+    fn same_bucket_entry(&self, entry: &Arc<BucketEntry>) -> bool {
+        Arc::ptr_eq(&self.entry(), entry)
+    }
+
+    fn node_ids(&self) -> TypedKeyGroup {
+        self.operate(|_rti, e| e.node_ids())
+    }
+    fn best_node_id(&self) -> TypedKey {
+        self.operate(|_rti, e| e.best_node_id())
+    }
+
+    fn update_node_status(&self, routing_domain: RoutingDomain, node_status: NodeStatus) {
+        self.operate_mut(|_rti, e| {
+            e.update_node_status(routing_domain, node_status);
+        });
+    }
+    fn best_routing_domain(&self) -> Option<RoutingDomain> {
+        self.operate(|rti, e| e.best_routing_domain(rti, self.routing_domain_set()))
+    }
+
+    // fn envelope_support(&self) -> Vec<u8> {
+    //     self.operate(|_rti, e| e.envelope_support())
+    // }
+    fn add_envelope_version(&self, envelope_version: u8) {
+        self.operate_mut(|_rti, e| e.add_envelope_version(envelope_version))
+    }
+    // fn set_envelope_support(&self, envelope_support: Vec<u8>) {
+    //     self.operate_mut(|_rti, e| e.set_envelope_support(envelope_support))
+    // }
+    fn best_envelope_version(&self) -> Option<u8> {
+        self.operate(|_rti, e| e.best_envelope_version())
+    }
+    fn state_reason(&self, cur_ts: Timestamp) -> BucketEntryStateReason {
+        self.operate(|_rti, e| e.state_reason(cur_ts))
+    }
+    fn state(&self, cur_ts: Timestamp) -> BucketEntryState {
+        self.operate(|_rti, e| e.state(cur_ts))
+    }
+    fn peer_stats(&self) -> PeerStats {
+        self.operate(|_rti, e| e.peer_stats().clone())
+    }
+
+    fn make_peer_info(&self, routing_domain: RoutingDomain) -> Option<PeerInfo> {
+        self.operate(|_rti, e| e.make_peer_info(routing_domain))
+    }
+    fn node_info(&self, routing_domain: RoutingDomain) -> Option<NodeInfo> {
+        self.operate(|_rti, e| e.node_info(routing_domain).cloned())
+    }
+    fn signed_node_info_has_valid_signature(&self, routing_domain: RoutingDomain) -> bool {
+        self.operate(|_rti, e| {
+            e.signed_node_info(routing_domain)
+                .map(|sni| sni.has_any_signature())
+                .unwrap_or(false)
+        })
+    }
+    fn node_info_ts(&self, routing_domain: RoutingDomain) -> Timestamp {
+        self.operate(|_rti, e| {
+            e.signed_node_info(routing_domain)
+                .map(|sni| sni.timestamp())
+                .unwrap_or(0u64.into())
+        })
+    }
+    fn has_seen_our_node_info_ts(
+        &self,
+        routing_domain: RoutingDomain,
+        our_node_info_ts: Timestamp,
+    ) -> bool {
+        self.operate(|_rti, e| e.has_seen_our_node_info_ts(routing_domain, our_node_info_ts))
+    }
+    fn set_seen_our_node_info_ts(&self, routing_domain: RoutingDomain, seen_ts: Timestamp) {
+        self.operate_mut(|_rti, e| e.set_seen_our_node_info_ts(routing_domain, seen_ts));
+    }
+    // fn network_class(&self, routing_domain: RoutingDomain) -> Option<NetworkClass> {
+    //     self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.network_class()))
+    // }
+    // fn outbound_protocols(&self, routing_domain: RoutingDomain) -> Option<ProtocolTypeSet> {
+    //     self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.outbound_protocols()))
+    // }
+    // fn address_types(&self, routing_domain: RoutingDomain) -> Option<AddressTypeSet> {
+    //     self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.address_types()))
+    // }
+
+    fn relay(&self, routing_domain: RoutingDomain) -> EyreResult<Option<FilteredNodeRef>> {
+        self.operate_mut(|rti, e| {
+            let Some(sni) = e.signed_node_info(routing_domain) else {
+                return Ok(None);
+            };
+            let Some(rpi) = sni.relay_peer_info(routing_domain) else {
+                return Ok(None);
+            };
+            // If relay is ourselves, then return None, because we can't relay through ourselves
+            // and to contact this node we should have had an existing inbound connection
+            if rti.unlocked_inner.matches_own_node_id(rpi.node_ids()) {
+                bail!("Can't relay though ourselves");
+            }
+
+            // Register relay node and return noderef
+            let nr = rti.register_node_with_peer_info(self.routing_table(), rpi, false)?;
+            Ok(Some(nr))
+        })
+    }
+    // DialInfo
+    fn first_dial_info_detail(&self) -> Option<DialInfoDetail> {
+        let routing_domain_set = self.routing_domain_set();
+        let dial_info_filter = self.dial_info_filter();
+        let sequencing = self.sequencing();
+        let (ordered, dial_info_filter) = dial_info_filter.apply_sequencing(sequencing);
+        let sort = ordered.then_some(DialInfoDetail::ordered_sequencing_sort);
+
+        if dial_info_filter.is_dead() {
+            return None;
+        }
+
+        let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter);
+
+        self.operate(|_rt, e| {
+            for routing_domain in routing_domain_set {
+                if let Some(ni) = e.node_info(routing_domain) {
+                    if let Some(did) = ni.first_filtered_dial_info_detail(sort, filter) {
+                        return Some(did);
+                    }
+                }
+            }
+            None
+        })
+    }
+
+    fn dial_info_details(&self) -> Vec<DialInfoDetail> {
+        let routing_domain_set = self.routing_domain_set();
+        let dial_info_filter = self.dial_info_filter();
+        let sequencing = self.sequencing();
+        let (ordered, dial_info_filter) = dial_info_filter.apply_sequencing(sequencing);
+        let sort = ordered.then_some(DialInfoDetail::ordered_sequencing_sort);
+
+        let mut out = Vec::new();
+
+        if dial_info_filter.is_dead() {
+            return out;
+        }
+
+        let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter);
+
+        self.operate(|_rt, e| {
+            for routing_domain in routing_domain_set {
+                if let Some(ni) = e.node_info(routing_domain) {
+                    let mut dids = ni.filtered_dial_info_details(sort, filter);
+                    out.append(&mut dids);
+                }
+            }
+        });
+        out.remove_duplicates();
+        out
+    }
+
+    /// Get the most recent 'last connection' to this node
+    /// Filtered first and then sorted by ordering preference and then by most recent
+    fn last_flow(&self) -> Option<Flow> {
+        self.operate(|rti, e| {
+            // apply sequencing to filter and get sort
+            let sequencing = self.sequencing();
+            let filter = self.filter();
+            let (ordered, filter) = filter.apply_sequencing(sequencing);
+            let mut last_flows = e.last_flows(rti, true, filter);
+
+            if ordered {
+                last_flows.sort_by(|a, b| {
+                    ProtocolType::ordered_sequencing_sort(a.0.protocol_type(), b.0.protocol_type())
+                });
+            }
+
+            last_flows.first().map(|x| x.0)
+        })
+    }
+
+    /// Get all the 'last connection' flows for this node
+    #[expect(dead_code)]
+    fn last_flows(&self) -> Vec<Flow> {
+        self.operate(|rti, e| {
+            // apply sequencing to filter and get sort
+            let sequencing = self.sequencing();
+            let filter = self.filter();
+            let (ordered, filter) = filter.apply_sequencing(sequencing);
+            let mut last_flows = e.last_flows(rti, true, filter);
+
+            if ordered {
+                last_flows.sort_by(|a, b| {
+                    ProtocolType::ordered_sequencing_sort(a.0.protocol_type(), b.0.protocol_type())
+                });
+            }
+
+            last_flows.into_iter().map(|x| x.0).collect()
+        })
+    }
+
+    fn clear_last_flows(&self) {
+        self.operate_mut(|_rti, e| e.clear_last_flows())
+    }
+
+    fn set_last_flow(&self, flow: Flow, ts: Timestamp) {
+        self.operate_mut(|rti, e| {
+            e.set_last_flow(flow, ts);
+            rti.touch_recent_peer(e.best_node_id(), flow);
+        })
+    }
+
+    fn clear_last_flow(&self, flow: Flow) {
+        self.operate_mut(|_rti, e| {
+            e.remove_last_flow(flow);
+        })
+    }
+
+    fn has_any_dial_info(&self) -> bool {
+        self.operate(|_rti, e| {
+            for rtd in RoutingDomain::all() {
+                if let Some(sni) = e.signed_node_info(rtd) {
+                    if sni.has_any_dial_info() {
+                        return true;
+                    }
+                }
+            }
+            false
+        })
+    }
+
+    fn report_protected_connection_dropped(&self) {
+        self.stats_failed_to_send(Timestamp::now(), false);
+    }
+
+    fn report_failed_route_test(&self) {
+        self.stats_failed_to_send(Timestamp::now(), false);
+    }
+
+    fn stats_question_sent(&self, ts: Timestamp, bytes: ByteCount, expects_answer: bool) {
+        self.operate_mut(|rti, e| {
+            rti.transfer_stats_accounting().add_up(bytes);
+            e.question_sent(ts, bytes, expects_answer);
+        })
+    }
+    fn stats_question_rcvd(&self, ts: Timestamp, bytes: ByteCount) {
+        self.operate_mut(|rti, e| {
+            rti.transfer_stats_accounting().add_down(bytes);
+            e.question_rcvd(ts, bytes);
+        })
+    }
+    fn stats_answer_sent(&self, bytes: ByteCount) {
+        self.operate_mut(|rti, e| {
+            rti.transfer_stats_accounting().add_up(bytes);
+            e.answer_sent(bytes);
+        })
+    }
+    fn stats_answer_rcvd(&self, send_ts: Timestamp, recv_ts: Timestamp, bytes: ByteCount) {
+        self.operate_mut(|rti, e| {
+            rti.transfer_stats_accounting().add_down(bytes);
+            rti.latency_stats_accounting()
+                .record_latency(recv_ts.saturating_sub(send_ts));
+            e.answer_rcvd(send_ts, recv_ts, bytes);
+        })
+    }
+    fn stats_question_lost(&self) {
+        self.operate_mut(|_rti, e| {
+            e.question_lost();
+        })
+    }
+    fn stats_failed_to_send(&self, ts: Timestamp, expects_answer: bool) {
+        self.operate_mut(|_rti, e| {
+            e.failed_to_send(ts, expects_answer);
+        })
+    }
+}

veilid-core/src/routing_table/privacy.rs

@@ -4,7 +4,7 @@ use super::*;
 // Compiled Privacy Objects
 
 /// An encrypted private/safety route hop
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub(crate) struct RouteHopData {
     /// The nonce used in the encryption ENC(Xn,DH(PKn,SKapr))
     pub nonce: Nonce,
@@ -12,13 +12,22 @@ pub(crate) struct RouteHopData {
     pub blob: Vec<u8>,
 }
 
+impl fmt::Debug for RouteHopData {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("RouteHopData")
+            .field("nonce", &self.nonce)
+            .field("blob", &format!("len={}", self.blob.len()))
+            .finish()
+    }
+}
+
 /// How to find a route node
 #[derive(Clone, Debug)]
 pub(crate) enum RouteNode {
     /// Route node is optimized, no contact method information as this node id has been seen before
     NodeId(PublicKey),
     /// Route node with full contact method information to ensure the peer is reachable
-    PeerInfo(Box<PeerInfo>),
+    PeerInfo(Arc<PeerInfo>),
 }
 
 impl RouteNode {
@@ -47,12 +56,8 @@ impl RouteNode {
             }
             RouteNode::PeerInfo(pi) => {
                 //
-                match routing_table.register_node_with_peer_info(
+                match routing_table.register_node_with_peer_info(pi.clone(), false) {
-                    RoutingDomain::PublicInternet,
+                    Ok(nr) => Some(nr.unfiltered()),
-                    *pi.clone(),
-                    false,
-                ) {
-                    Ok(nr) => Some(nr),
                     Err(e) => {
                         log_rtab!(debug "failed to register route node: {}", e);
                         None

veilid-core/src/routing_table/route_spec_store/mod.rs

@@ -238,16 +238,14 @@ impl RouteSpecStore {
             );
         }
 
-        // Ensure we have a valid network class so our peer info is useful
+        // Get our peer info
-        if !rti.has_valid_network_class(RoutingDomain::PublicInternet) {
+        let Some(published_peer_info) = rti.get_published_peer_info(RoutingDomain::PublicInternet)
+        else {
             apibail_try_again!(
                 "unable to allocate route until we have a valid PublicInternet network class"
             );
         };
 
-        // Get our peer info
-        let our_peer_info = rti.get_own_peer_info(RoutingDomain::PublicInternet);
-
         // Get relay node if we have one
         let opt_own_relay_nr = rti
             .relay_node(RoutingDomain::PublicInternet)
@@ -297,7 +295,7 @@ impl RouteSpecStore {
                     // or our relay is on their ipblock, or their relay is on our relays same ipblock
 
                     // our node vs their node
-                    if our_peer_info
+                    if published_peer_info
                         .signed_node_info()
                         .node_info()
                         .node_is_on_same_ipblock(sni.node_info(), ip6_prefix_size)
@@ -306,20 +304,22 @@ impl RouteSpecStore {
                     }
                     if let Some(rni) = sni.relay_info() {
                         // our node vs their relay
-                        if our_peer_info
+                        if published_peer_info
                             .signed_node_info()
                             .node_info()
                             .node_is_on_same_ipblock(rni, ip6_prefix_size)
                         {
                             return false;
                         }
-                        if let Some(our_rni) = our_peer_info.signed_node_info().relay_info() {
+                        if let Some(our_rni) = published_peer_info.signed_node_info().relay_info() {
                             // our relay vs their relay
                             if our_rni.node_is_on_same_ipblock(rni, ip6_prefix_size) {
                                 return false;
                             }
                         }
-                    } else if let Some(our_rni) = our_peer_info.signed_node_info().relay_info() {
+                    } else if let Some(our_rni) =
+                        published_peer_info.signed_node_info().relay_info()
+                    {
                         // our relay vs their node
                         if our_rni.node_is_on_same_ipblock(sni.node_info(), ip6_prefix_size) {
                             return false;
@@ -419,7 +419,7 @@ impl RouteSpecStore {
 
         let routing_table = self.unlocked_inner.routing_table.clone();
         let transform = |_rti: &RoutingTableInner, entry: Option<Arc<BucketEntry>>| -> NodeRef {
-            NodeRef::new(routing_table.clone(), entry.unwrap(), None)
+            NodeRef::new(routing_table.clone(), entry.unwrap())
         };
 
         // Pull the whole routing table in sorted order
@@ -432,12 +432,14 @@ impl RouteSpecStore {
         }
 
         // Get peer info for everything
-        let nodes_pi: Vec<PeerInfo> = nodes
+        let nodes_pi: Vec<Arc<PeerInfo>> = nodes
             .iter()
             .map(|nr| {
-                nr.locked(rti)
+                Arc::new(
-                    .make_peer_info(RoutingDomain::PublicInternet)
+                    nr.locked(rti)
-                    .unwrap()
+                        .make_peer_info(RoutingDomain::PublicInternet)
+                        .unwrap(),
+                )
             })
             .collect();
 
@@ -487,14 +489,14 @@ impl RouteSpecStore {
             // Ensure this route is viable by checking that each node can contact the next one
             let mut can_do_sequenced = true;
             if directions.contains(Direction::Outbound) {
-                let mut previous_node = &our_peer_info;
+                let mut previous_node = published_peer_info.clone();
                 let mut reachable = true;
                 for n in permutation {
-                    let current_node = nodes_pi.get(*n).unwrap();
+                    let current_node = nodes_pi.get(*n).cloned().unwrap();
                     let cm = rti.get_contact_method(
                         RoutingDomain::PublicInternet,
-                        previous_node,
+                        previous_node.clone(),
-                        current_node,
+                        current_node.clone(),
                         DialInfoFilter::all(),
                         sequencing,
                         None,
@@ -508,8 +510,8 @@ impl RouteSpecStore {
                     if can_do_sequenced {
                         let cm = rti.get_contact_method(
                             RoutingDomain::PublicInternet,
-                            previous_node,
+                            previous_node.clone(),
-                            current_node,
+                            current_node.clone(),
                             DialInfoFilter::all(),
                             Sequencing::EnsureOrdered,
                             None,
@@ -526,14 +528,14 @@ impl RouteSpecStore {
                 }
             }
             if directions.contains(Direction::Inbound) {
-                let mut next_node = &our_peer_info;
+                let mut next_node = published_peer_info.clone();
                 let mut reachable = true;
                 for n in permutation.iter().rev() {
-                    let current_node = nodes_pi.get(*n).unwrap();
+                    let current_node = nodes_pi.get(*n).cloned().unwrap();
                     let cm = rti.get_contact_method(
                         RoutingDomain::PublicInternet,
-                        next_node,
+                        next_node.clone(),
-                        current_node,
+                        current_node.clone(),
                         DialInfoFilter::all(),
                         sequencing,
                         None,
@@ -547,8 +549,8 @@ impl RouteSpecStore {
                     if can_do_sequenced {
                         let cm = rti.get_contact_method(
                             RoutingDomain::PublicInternet,
-                            next_node,
+                            next_node.clone(),
-                            current_node,
+                            current_node.clone(),
                             DialInfoFilter::all(),
                             Sequencing::EnsureOrdered,
                             None,
@@ -710,18 +712,18 @@ impl RouteSpecStore {
     }
 
     #[instrument(level = "trace", target = "route", skip(self), ret, err)]
-    async fn test_allocated_route(&self, private_route_id: RouteId) -> VeilidAPIResult<bool> {
+    async fn test_allocated_route(
+        &self,
+        private_route_id: RouteId,
+    ) -> VeilidAPIResult<Option<bool>> {
         // Make loopback route to test with
         let (dest, hops) = {
             // Get the best allocated route for this id
             let (key, hops) = {
                 let inner = &mut *self.inner.lock();
                 let Some(rssd) = inner.content.get_detail(&private_route_id) else {
-                    apibail_invalid_argument!(
+                    // Route id is already dead
-                        "route id not allocated",
+                    return Ok(Some(false));
-                        "private_route_id",
-                        private_route_id
-                    );
                 };
                 let Some(key) = rssd.get_best_route_set_key() else {
                     apibail_internal!("no best key to test allocated route");
@@ -733,7 +735,21 @@ impl RouteSpecStore {
             };
 
             // Get the private route to send to
-            let private_route = self.assemble_private_route(&key, None)?;
+            let private_route = match self.assemble_private_route(&key, None) {
+                Ok(v) => v,
+                Err(VeilidAPIError::InvalidTarget { message: _ }) => {
+                    // Route missing means its dead
+                    return Ok(Some(false));
+                }
+                Err(VeilidAPIError::TryAgain { message: _ }) => {
+                    // Try again means we didn't test because we couldnt assemble
+                    return Ok(None);
+                }
+                Err(e) => {
+                    return Err(e);
+                }
+            };
+
             // Always test routes with safety routes that are more likely to succeed
             let stability = Stability::Reliable;
             // Routes should test with the most likely to succeed sequencing they are capable of
@@ -769,15 +785,15 @@ impl RouteSpecStore {
                 for hop in hops {
                     hop.report_failed_route_test();
                 }
-                return Ok(false);
+                return Ok(Some(false));
             }
         };
 
-        Ok(true)
+        Ok(Some(true))
     }
 
     #[instrument(level = "trace", target = "route", skip(self), ret, err)]
-    async fn test_remote_route(&self, private_route_id: RouteId) -> VeilidAPIResult<bool> {
+    async fn test_remote_route(&self, private_route_id: RouteId) -> VeilidAPIResult<Option<bool>> {
         // Make private route test
         let dest = {
             // Get the route to test
@@ -812,11 +828,11 @@ impl RouteSpecStore {
             NetworkResult::Value(v) => v,
             _ => {
                 // Did not error, but did not come back, just return false
-                return Ok(false);
+                return Ok(Some(false));
             }
         };
 
-        Ok(true)
+        Ok(Some(true))
     }
 
     /// Release an allocated route that is no longer in use
@@ -848,7 +864,7 @@ impl RouteSpecStore {
 
     /// Test an allocated route for continuity
     #[instrument(level = "trace", target = "route", skip(self), ret, err)]
-    pub async fn test_route(&self, id: RouteId) -> VeilidAPIResult<bool> {
+    pub async fn test_route(&self, id: RouteId) -> VeilidAPIResult<Option<bool>> {
         let is_remote = self.is_route_id_remote(&id);
         if is_remote {
             self.test_remote_route(id).await
@@ -1040,23 +1056,18 @@ impl RouteSpecStore {
                         .lookup_node_ref(routing_table.clone(), TypedKey::new(crypto_kind, id))
                         .map_err(VeilidAPIError::internal)?,
                     RouteNode::PeerInfo(pi) => Some(
-                        rti.register_node_with_peer_info(
+                        rti.register_node_with_peer_info(routing_table.clone(), pi, false)
-                            routing_table.clone(),
+                            .map_err(VeilidAPIError::internal)?
-                            RoutingDomain::PublicInternet,
+                            .unfiltered(),
-                            *pi,
-                            false,
-                        )
-                        .map_err(VeilidAPIError::internal)?,
                     ),
                 };
-                if opt_first_hop.is_none() {
+                let Some(first_hop) = opt_first_hop else {
                     // Can't reach this private route any more
                     apibail_generic!("can't reach private route any more");
-                }
+                };
-                let mut first_hop = opt_first_hop.unwrap();
 
                 // Set sequencing requirement
-                first_hop.set_sequencing(sequencing);
+                let mut first_hop = first_hop.sequencing_filtered(sequencing);
 
                 // Enforce the routing domain
                 first_hop.merge_filter(
@@ -1115,10 +1126,10 @@ impl RouteSpecStore {
             || safety_rssd.get_stats().last_received_ts.is_some();
 
         // Get the first hop noderef of the safety route
-        let mut first_hop = safety_rssd.hop_node_ref(0).unwrap();
+        let first_hop = safety_rssd.hop_node_ref(0).unwrap();
 
         // Ensure sequencing requirement is set on first hop
-        first_hop.set_sequencing(safety_spec.sequencing);
+        let mut first_hop = first_hop.sequencing_filtered(safety_spec.sequencing);
 
         // Enforce the routing domain
         first_hop
@@ -1201,7 +1212,7 @@ impl RouteSpecStore {
                             if pi.is_none() {
                                 apibail_internal!("peer info should exist for route but doesn't");
                             }
-                            RouteNode::PeerInfo(Box::new(pi.unwrap()))
+                            RouteNode::PeerInfo(Arc::new(pi.unwrap()))
                         },
                         next_hop: Some(route_hop_data),
                     };
@@ -1386,11 +1397,10 @@ impl RouteSpecStore {
         };
 
         // Ensure our network class is valid before attempting to assemble any routes
-        if !rti.has_valid_network_class(RoutingDomain::PublicInternet) {
+        let Some(published_peer_info) = rti.get_published_peer_info(RoutingDomain::PublicInternet)
-            apibail_try_again!(
+        else {
-                "unable to assemble route until we have a valid PublicInternet network class"
+            apibail_try_again!("unable to assemble route until we have published peerinfo");
-            );
+        };
-        }
 
         // Make innermost route hop to our own node
         let mut route_hop = RouteHop {
@@ -1404,8 +1414,7 @@ impl RouteSpecStore {
                 };
                 RouteNode::NodeId(node_id.value)
             } else {
-                let pi = rti.get_own_peer_info(RoutingDomain::PublicInternet);
+                RouteNode::PeerInfo(published_peer_info)
-                RouteNode::PeerInfo(Box::new(pi))
             },
             next_hop: None,
         };
@@ -1449,7 +1458,7 @@ impl RouteSpecStore {
                     if pi.is_none() {
                         apibail_internal!("peer info should exist for route but doesn't");
                     }
-                    RouteNode::PeerInfo(Box::new(pi.unwrap()))
+                    RouteNode::PeerInfo(Arc::new(pi.unwrap()))
                 },
                 next_hop: Some(route_hop_data),
             }
@@ -1474,7 +1483,8 @@ impl RouteSpecStore {
     ) -> VeilidAPIResult<PrivateRoute> {
         let inner = &*self.inner.lock();
         let Some(rsid) = inner.content.get_id_by_key(key) else {
-            apibail_invalid_argument!("route key does not exist", "key", key);
+            // Route doesn't exist
+            apibail_invalid_target!("route id does not exist");
         };
         let Some(rssd) = inner.content.get_detail(&rsid) else {
             apibail_internal!("route id does not exist");
@@ -1504,7 +1514,7 @@ impl RouteSpecStore {
     ) -> VeilidAPIResult<Vec<PrivateRoute>> {
         let inner = &*self.inner.lock();
         let Some(rssd) = inner.content.get_detail(id) else {
-            apibail_invalid_argument!("route id does not exist", "id", id);
+            apibail_invalid_target!("route id does not exist");
         };
 
         // See if we can optimize this compilation yet
@@ -1529,10 +1539,7 @@ impl RouteSpecStore {
         let cur_ts = Timestamp::now();
 
         // decode the pr blob
-        let private_routes = RouteSpecStore::blob_to_private_routes(
+        let private_routes = self.blob_to_private_routes(blob)?;
-            self.unlocked_inner.routing_table.crypto(),
-            blob,
-        )?;
 
         // make the route id
         let id = self.generate_remote_route_id(&private_routes)?;
@@ -1620,7 +1627,11 @@ impl RouteSpecStore {
 
     /// Check to see if this remote (not ours) private route has seen our current node info yet
     /// This happens when you communicate with a private route without a safety route
-    pub fn has_remote_private_route_seen_our_node_info(&self, key: &PublicKey) -> bool {
+    pub fn has_remote_private_route_seen_our_node_info(
+        &self,
+        key: &PublicKey,
+        published_peer_info: &PeerInfo,
+    ) -> bool {
         let inner = &*self.inner.lock();
 
         // Check for local route. If this is not a remote private route,
@@ -1633,18 +1644,15 @@ impl RouteSpecStore {
         if let Some(rrid) = inner.cache.get_remote_private_route_id_by_key(key) {
             let cur_ts = Timestamp::now();
             if let Some(rpri) = inner.cache.peek_remote_private_route(cur_ts, &rrid) {
-                let our_node_info_ts = self
+                return rpri
-                    .unlocked_inner
+                    .has_seen_our_node_info_ts(published_peer_info.signed_node_info().timestamp());
-                    .routing_table
-                    .get_own_node_info_ts(RoutingDomain::PublicInternet);
-                return rpri.has_seen_our_node_info_ts(our_node_info_ts);
             }
         }
 
         false
     }
 
-    /// Mark a remote private route as having seen our current node info
+    /// Mark a remote private route as having seen our current published node info
     /// PRIVACY:
     /// We do not accept node info timestamps from remote private routes because this would
     /// enable a deanonymization attack, whereby a node could be 'pinged' with a doctored node_info with a
@@ -1655,10 +1663,14 @@ impl RouteSpecStore {
         key: &PublicKey,
         cur_ts: Timestamp,
     ) -> VeilidAPIResult<()> {
-        let our_node_info_ts = self
+        let Some(our_node_info_ts) = self
             .unlocked_inner
             .routing_table
-            .get_own_node_info_ts(RoutingDomain::PublicInternet);
+            .get_published_peer_info(RoutingDomain::PublicInternet)
+            .map(|pi| pi.signed_node_info().timestamp())
+        else {
+            apibail_internal!("peer info is not yet published");
+        };
 
         let inner = &mut *self.inner.lock();
 
@@ -1676,7 +1688,7 @@ impl RouteSpecStore {
             }
         }
 
-        apibail_invalid_argument!("private route is missing from store", "key", key);
+        apibail_invalid_target!("private route is missing from store");
     }
 
     /// Get the route statistics for any route we know about, local or remote
@@ -1732,7 +1744,7 @@ impl RouteSpecStore {
     pub fn mark_route_published(&self, id: &RouteId, published: bool) -> VeilidAPIResult<()> {
         let inner = &mut *self.inner.lock();
         let Some(rssd) = inner.content.get_detail_mut(id) else {
-            apibail_invalid_argument!("route does not exist", "id", id);
+            apibail_invalid_target!("route does not exist");
         };
         rssd.set_published(published);
         Ok(())
@@ -1776,10 +1788,10 @@ impl RouteSpecStore {
     }
 
     /// Convert binary blob to private route vector
-    pub fn blob_to_private_routes(
+    pub fn blob_to_private_routes(&self, blob: Vec<u8>) -> VeilidAPIResult<Vec<PrivateRoute>> {
-        crypto: Crypto,
+        // Get crypto
-        blob: Vec<u8>,
+        let crypto = self.unlocked_inner.routing_table.crypto();
-    ) -> VeilidAPIResult<Vec<PrivateRoute>> {
+
         // Deserialize count
         if blob.is_empty() {
             apibail_invalid_argument!(
@@ -1795,6 +1807,9 @@ impl RouteSpecStore {
         }
 
         // Deserialize stream of private routes
+        let decode_context = RPCDecodeContext {
+            routing_domain: RoutingDomain::PublicInternet,
+        };
         let mut pr_slice = &blob[1..];
         let mut out = Vec::with_capacity(pr_count);
         for _ in 0..pr_count {
@@ -1807,7 +1822,7 @@ impl RouteSpecStore {
             let pr_reader = reader
                 .get_root::<veilid_capnp::private_route::Reader>()
                 .map_err(VeilidAPIError::internal)?;
-            let private_route = decode_private_route(&pr_reader).map_err(|e| {
+            let private_route = decode_private_route(&decode_context, &pr_reader).map_err(|e| {
                 VeilidAPIError::invalid_argument("failed to decode private route", "e", e)
             })?;
             private_route.validate(crypto.clone()).map_err(|e| {

veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs

@@ -15,7 +15,8 @@ pub(crate) struct CompiledRoute {
     /// The secret used to encrypt the message payload
     pub secret: SecretKey,
     /// The node ref to the first hop in the compiled route
-    pub first_hop: NodeRef,
+    /// filtered to the safetyselection it was compiled with
+    pub first_hop: FilteredNodeRef,
 }
 
 /// Ephemeral data used to help the RouteSpecStore operate efficiently

veilid-core/src/routing_table/route_spec_store/route_stats.rs

@@ -94,7 +94,7 @@ impl RouteStats {
     }
 
     /// Get the transfer stats
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn transfer_stats(&self) -> &TransferStatsDownUp {
         &self.transfer_stats_down_up
     }

veilid-core/src/routing_table/routing_domain_editor.rs

@@ -1,268 +0,0 @@
-use super::*;
-
-#[derive(Debug)]
-enum RoutingDomainChange {
-    ClearDialInfoDetails {
-        address_type: Option<AddressType>,
-        protocol_type: Option<ProtocolType>,
-    },
-    ClearRelayNode,
-    SetRelayNode {
-        relay_node: NodeRef,
-    },
-    SetRelayNodeKeepalive {
-        ts: Option<Timestamp>,
-    },
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
-    AddDialInfoDetail {
-        dial_info_detail: DialInfoDetail,
-    },
-    SetupNetwork {
-        outbound_protocols: ProtocolTypeSet,
-        inbound_protocols: ProtocolTypeSet,
-        address_types: AddressTypeSet,
-        capabilities: Vec<Capability>,
-    },
-    SetNetworkClass {
-        network_class: Option<NetworkClass>,
-    },
-}
-
-pub(crate) struct RoutingDomainEditor {
-    routing_table: RoutingTable,
-    routing_domain: RoutingDomain,
-    changes: Vec<RoutingDomainChange>,
-}
-
-impl RoutingDomainEditor {
-    pub(super) fn new(routing_table: RoutingTable, routing_domain: RoutingDomain) -> Self {
-        Self {
-            routing_table,
-            routing_domain,
-            changes: Vec::new(),
-        }
-    }
-
-    #[instrument(level = "debug", skip(self))]
-    pub fn clear_dial_info_details(
-        &mut self,
-        address_type: Option<AddressType>,
-        protocol_type: Option<ProtocolType>,
-    ) -> &mut Self {
-        self.changes
-            .push(RoutingDomainChange::ClearDialInfoDetails {
-                address_type,
-                protocol_type,
-            });
-        self
-    }
-    #[instrument(level = "debug", skip(self))]
-    pub fn clear_relay_node(&mut self) -> &mut Self {
-        self.changes.push(RoutingDomainChange::ClearRelayNode);
-        self
-    }
-    #[instrument(level = "debug", skip(self))]
-    pub fn set_relay_node(&mut self, relay_node: NodeRef) -> &mut Self {
-        self.changes
-            .push(RoutingDomainChange::SetRelayNode { relay_node });
-        self
-    }
-    #[instrument(level = "debug", skip(self))]
-    pub fn set_relay_node_keepalive(&mut self, ts: Option<Timestamp>) -> &mut Self {
-        self.changes
-            .push(RoutingDomainChange::SetRelayNodeKeepalive { ts });
-        self
-    }
-    #[instrument(level = "debug", skip(self))]
-    pub fn register_dial_info(
-        &mut self,
-        dial_info: DialInfo,
-        class: DialInfoClass,
-    ) -> EyreResult<&mut Self> {
-        if !self
-            .routing_table
-            .ensure_dial_info_is_valid(self.routing_domain, &dial_info)
-        {
-            return Err(eyre!(
-                "dial info '{}' is not valid in routing domain '{:?}'",
-                dial_info,
-                self.routing_domain
-            ));
-        }
-
-        self.changes.push(RoutingDomainChange::AddDialInfoDetail {
-            dial_info_detail: DialInfoDetail {
-                dial_info: dial_info.clone(),
-                class,
-            },
-        });
-
-        Ok(self)
-    }
-    #[instrument(level = "debug", skip(self))]
-    pub fn setup_network(
-        &mut self,
-        outbound_protocols: ProtocolTypeSet,
-        inbound_protocols: ProtocolTypeSet,
-        address_types: AddressTypeSet,
-        capabilities: Vec<Capability>,
-    ) -> &mut Self {
-        self.changes.push(RoutingDomainChange::SetupNetwork {
-            outbound_protocols,
-            inbound_protocols,
-            address_types,
-            capabilities,
-        });
-        self
-    }
-
-    #[instrument(level = "debug", skip(self))]
-    pub fn set_network_class(&mut self, network_class: Option<NetworkClass>) -> &mut Self {
-        self.changes
-            .push(RoutingDomainChange::SetNetworkClass { network_class });
-        self
-    }
-
-    #[instrument(level = "debug", skip(self))]
-    pub async fn commit(&mut self, pause_tasks: bool) {
-        // No locking if we have nothing to do
-        if self.changes.is_empty() {
-            return;
-        }
-
-        // Briefly pause routing table ticker while changes are made
-        let _tick_guard = if pause_tasks {
-            Some(self.routing_table.pause_tasks().await)
-        } else {
-            None
-        };
-
-        // Apply changes
-        log_rtab!("[{:?}] COMMIT: {:?}", self.routing_domain, self.changes);
-        let mut peer_info_changed = false;
-        {
-            let mut inner = self.routing_table.inner.write();
-            inner.with_routing_domain_mut(self.routing_domain, |detail| {
-                for change in self.changes.drain(..) {
-                    match change {
-                        RoutingDomainChange::ClearDialInfoDetails {
-                            address_type,
-                            protocol_type,
-                        } => {
-                            if !detail.common_mut().dial_info_details().is_empty() {
-                                if address_type.is_some() || protocol_type.is_some() {
-                                    info!(
-                                        "[{:?}] cleared dial info: {}:{}",
-                                        self.routing_domain,
-                                        address_type
-                                            .map(|at| format!("{:?}", at))
-                                            .unwrap_or("---".to_string()),
-                                        protocol_type
-                                            .map(|at| format!("{:?}", at))
-                                            .unwrap_or("---".to_string()),
-                                    );
-                                } else {
-                                    info!("[{:?}] cleared all dial info", self.routing_domain);
-                                }
-                            }
-                            detail
-                                .common_mut()
-                                .clear_dial_info_details(address_type, protocol_type);
-                            peer_info_changed = true;
-                        }
-                        RoutingDomainChange::ClearRelayNode => {
-                            if detail.common_mut().relay_node().is_some() {
-                                info!("[{:?}] cleared relay node", self.routing_domain);
-                            }
-                            detail.common_mut().set_relay_node(None);
-                            peer_info_changed = true;
-                        }
-                        RoutingDomainChange::SetRelayNode { relay_node } => {
-                            info!("[{:?}] set relay node: {}", self.routing_domain, relay_node);
-                            detail.common_mut().set_relay_node(Some(relay_node.clone()));
-                            peer_info_changed = true;
-                        }
-                        RoutingDomainChange::SetRelayNodeKeepalive { ts } => {
-                            trace!("[{:?}] relay node keepalive: {:?}", self.routing_domain, ts);
-                            detail.common_mut().set_relay_node_last_keepalive(ts);
-                        }
-                        RoutingDomainChange::AddDialInfoDetail { dial_info_detail } => {
-                            info!(
-                                "[{:?}] dial info: {:?}:{}",
-                                self.routing_domain,
-                                dial_info_detail.class,
-                                dial_info_detail.dial_info
-                            );
-                            detail
-                                .common_mut()
-                                .add_dial_info_detail(dial_info_detail.clone());
-
-                            peer_info_changed = true;
-                        }
-                        RoutingDomainChange::SetupNetwork {
-                            outbound_protocols,
-                            inbound_protocols,
-                            address_types,
-                            capabilities,
-                        } => {
-                            let old_outbound_protocols = detail.common().outbound_protocols();
-                            let old_inbound_protocols = detail.common().inbound_protocols();
-                            let old_address_types = detail.common().address_types();
-                            let old_capabilities = detail.common().capabilities();
-
-                            let this_changed = old_outbound_protocols != outbound_protocols
-                                || old_inbound_protocols != inbound_protocols
-                                || old_address_types != address_types
-                                || old_capabilities != *capabilities;
-
-                            if this_changed {
-                                info!(
-                                    "[{:?}] setup network: outbound {:?} inbound {:?} address types {:?} capabilities {:?}",
-                                    self.routing_domain,
-                                    outbound_protocols,
-                                    inbound_protocols,
-                                    address_types,
-                                    capabilities
-                                );
-
-                                detail.common_mut().setup_network(
-                                    outbound_protocols,
-                                    inbound_protocols,
-                                    address_types,
-                                    capabilities.clone(),
-                                );
-                                peer_info_changed = true;
-                            }
-                        }
-                        RoutingDomainChange::SetNetworkClass { network_class } => {
-                            let old_network_class = detail.common().network_class();
-
-                            let this_changed = old_network_class != network_class;
-                            if this_changed {
-                                if let Some(network_class) = network_class {
-                                    info!(
-                                        "[{:?}] set network class: {:?}",
-                                        self.routing_domain, network_class,
-                                    );
-                                } else {
-                                    info!("[{:?}] cleared network class", self.routing_domain,);
-                                }
-                                detail.common_mut().set_network_class(network_class);
-                                peer_info_changed = true;
-                            }
-                        }
-                    }
-                }
-            });
-            if peer_info_changed {
-                // Allow signed node info updates at same timestamp for otherwise dead nodes if our network has changed
-                inner.reset_all_updated_since_last_network_change();
-            }
-        }
-        // Clear the routespecstore cache if our PublicInternet dial info has changed
-        if peer_info_changed && self.routing_domain == RoutingDomain::PublicInternet {
-            let rss = self.routing_table.route_spec_store();
-            rss.reset();
-        }
-    }
-}

veilid-core/src/routing_table/routing_domains.rs

@@ -1,622 +0,0 @@
-use super::*;
-
-/// Mechanism required to contact another node
-#[derive(Clone, Debug)]
-pub(crate) enum ContactMethod {
-    /// Node is not reachable by any means
-    Unreachable,
-    /// Connection should have already existed
-    Existing,
-    /// Contact the node directly
-    Direct(DialInfo),
-    /// Request via signal the node connect back directly (relay, target)
-    SignalReverse(TypedKey, TypedKey),
-    /// Request via signal the node negotiate a hole punch (relay, target)
-    SignalHolePunch(TypedKey, TypedKey),
-    /// Must use an inbound relay to reach the node
-    InboundRelay(TypedKey),
-    /// Must use outbound relay to reach the node
-    OutboundRelay(TypedKey),
-}
-
-#[derive(Debug)]
-pub(crate) struct RoutingDomainDetailCommon {
-    routing_domain: RoutingDomain,
-    network_class: Option<NetworkClass>,
-    outbound_protocols: ProtocolTypeSet,
-    inbound_protocols: ProtocolTypeSet,
-    address_types: AddressTypeSet,
-    relay_node: Option<NodeRef>,
-    relay_node_last_keepalive: Option<Timestamp>,
-    capabilities: Vec<Capability>,
-    dial_info_details: Vec<DialInfoDetail>,
-    // caches
-    cached_peer_info: Mutex<Option<PeerInfo>>,
-}
-
-impl RoutingDomainDetailCommon {
-    pub fn new(routing_domain: RoutingDomain) -> Self {
-        Self {
-            routing_domain,
-            network_class: Default::default(),
-            outbound_protocols: Default::default(),
-            inbound_protocols: Default::default(),
-            address_types: Default::default(),
-            relay_node: Default::default(),
-            relay_node_last_keepalive: Default::default(),
-            capabilities: Default::default(),
-            dial_info_details: Default::default(),
-            cached_peer_info: Mutex::new(Default::default()),
-        }
-    }
-
-    // Set from network manager
-    pub(super) fn setup_network(
-        &mut self,
-        outbound_protocols: ProtocolTypeSet,
-        inbound_protocols: ProtocolTypeSet,
-        address_types: AddressTypeSet,
-        capabilities: Vec<Capability>,
-    ) {
-        self.outbound_protocols = outbound_protocols;
-        self.inbound_protocols = inbound_protocols;
-        self.address_types = address_types;
-        self.capabilities = capabilities;
-        self.clear_cache();
-    }
-
-    pub(super) fn set_network_class(&mut self, network_class: Option<NetworkClass>) {
-        self.network_class = network_class;
-        self.clear_cache();
-    }
-    pub fn network_class(&self) -> Option<NetworkClass> {
-        self.network_class
-    }
-    pub fn outbound_protocols(&self) -> ProtocolTypeSet {
-        self.outbound_protocols
-    }
-    pub fn inbound_protocols(&self) -> ProtocolTypeSet {
-        self.inbound_protocols
-    }
-    pub fn address_types(&self) -> AddressTypeSet {
-        self.address_types
-    }
-    pub fn capabilities(&self) -> Vec<Capability> {
-        self.capabilities.clone()
-    }
-    pub fn relay_node(&self) -> Option<NodeRef> {
-        self.relay_node.clone()
-    }
-    pub fn relay_node_last_keepalive(&self) -> Option<Timestamp> {
-        self.relay_node_last_keepalive
-    }
-    pub(super) fn set_relay_node(&mut self, opt_relay_node: Option<NodeRef>) {
-        self.relay_node = opt_relay_node.map(|nr| {
-            nr.filtered_clone(NodeRefFilter::new().with_routing_domain(self.routing_domain))
-        });
-        self.relay_node_last_keepalive = None;
-        self.clear_cache();
-    }
-    pub(super) fn set_relay_node_last_keepalive(&mut self, ts: Option<Timestamp>) {
-        self.relay_node_last_keepalive = ts;
-    }
-    pub fn dial_info_details(&self) -> &Vec<DialInfoDetail> {
-        &self.dial_info_details
-    }
-    pub(super) fn clear_dial_info_details(
-        &mut self,
-        address_type: Option<AddressType>,
-        protocol_type: Option<ProtocolType>,
-    ) {
-        self.dial_info_details.retain_mut(|e| {
-            let mut remove = true;
-            if let Some(pt) = protocol_type {
-                if pt != e.dial_info.protocol_type() {
-                    remove = false;
-                }
-            }
-            if let Some(at) = address_type {
-                if at != e.dial_info.address_type() {
-                    remove = false;
-                }
-            }
-            !remove
-        });
-        self.clear_cache();
-    }
-    pub(super) fn add_dial_info_detail(&mut self, did: DialInfoDetail) {
-        self.dial_info_details.push(did);
-        self.dial_info_details.sort();
-        self.clear_cache();
-    }
-
-    pub fn has_valid_network_class(&self) -> bool {
-        self.network_class.unwrap_or(NetworkClass::Invalid) != NetworkClass::Invalid
-    }
-
-    fn make_peer_info(&self, rti: &RoutingTableInner) -> PeerInfo {
-        let node_info = NodeInfo::new(
-            self.network_class.unwrap_or(NetworkClass::Invalid),
-            self.outbound_protocols,
-            self.address_types,
-            VALID_ENVELOPE_VERSIONS.to_vec(),
-            VALID_CRYPTO_KINDS.to_vec(),
-            self.capabilities.clone(),
-            self.dial_info_details.clone(),
-        );
-
-        let relay_info = if let Some(rn) = &self.relay_node {
-            let opt_relay_pi = rn.locked(rti).make_peer_info(self.routing_domain);
-            if let Some(relay_pi) = opt_relay_pi {
-                let (relay_ids, relay_sni) = relay_pi.destructure();
-                match relay_sni {
-                    SignedNodeInfo::Direct(d) => Some((relay_ids, d)),
-                    SignedNodeInfo::Relayed(_) => {
-                        warn!("relay node should not have a relay itself! if this happens, a relay updated its signed node info and became a relay, which should cause the relay to be dropped");
-                        None
-                    }
-                }
-            } else {
-                None
-            }
-        } else {
-            None
-        };
-
-        let signed_node_info = match relay_info {
-            Some((relay_ids, relay_sdni)) => SignedNodeInfo::Relayed(
-                SignedRelayedNodeInfo::make_signatures(
-                    rti.unlocked_inner.crypto(),
-                    rti.unlocked_inner.node_id_typed_key_pairs(),
-                    node_info,
-                    relay_ids,
-                    relay_sdni,
-                )
-                .unwrap(),
-            ),
-            None => SignedNodeInfo::Direct(
-                SignedDirectNodeInfo::make_signatures(
-                    rti.unlocked_inner.crypto(),
-                    rti.unlocked_inner.node_id_typed_key_pairs(),
-                    node_info,
-                )
-                .unwrap(),
-            ),
-        };
-
-        PeerInfo::new(rti.unlocked_inner.node_ids(), signed_node_info)
-    }
-
-    pub fn with_peer_info<F, R>(&self, rti: &RoutingTableInner, f: F) -> R
-    where
-        F: FnOnce(&PeerInfo) -> R,
-    {
-        let mut cpi = self.cached_peer_info.lock();
-        if cpi.is_none() {
-            // Regenerate peer info
-            let pi = self.make_peer_info(rti);
-
-            // Cache the peer info
-            *cpi = Some(pi);
-        }
-        f(cpi.as_ref().unwrap())
-    }
-
-    #[allow(dead_code)]
-    pub fn inbound_dial_info_filter(&self) -> DialInfoFilter {
-        DialInfoFilter::all()
-            .with_protocol_type_set(self.inbound_protocols)
-            .with_address_type_set(self.address_types)
-    }
-    pub fn outbound_dial_info_filter(&self) -> DialInfoFilter {
-        DialInfoFilter::all()
-            .with_protocol_type_set(self.outbound_protocols)
-            .with_address_type_set(self.address_types)
-    }
-
-    pub(super) fn clear_cache(&self) {
-        *self.cached_peer_info.lock() = None;
-    }
-}
-
-/// General trait for all routing domains
-pub(crate) trait RoutingDomainDetail {
-    // Common accessors
-    fn common(&self) -> &RoutingDomainDetailCommon;
-    fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon;
-
-    /// Can this routing domain contain a particular address
-    fn can_contain_address(&self, address: Address) -> bool;
-
-    /// Get the contact method required for node A to reach node B in this routing domain
-    /// Routing table must be locked for reading to use this function
-    fn get_contact_method(
-        &self,
-        rti: &RoutingTableInner,
-        peer_a: &PeerInfo,
-        peer_b: &PeerInfo,
-        dial_info_filter: DialInfoFilter,
-        sequencing: Sequencing,
-        dif_sort: Option<Arc<DialInfoDetailSort>>,
-    ) -> ContactMethod;
-}
-
-/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-/// Public Internet routing domain internals
-#[derive(Debug)]
-pub struct PublicInternetRoutingDomainDetail {
-    /// Common implementation for all routing domains
-    common: RoutingDomainDetailCommon,
-}
-
-impl Default for PublicInternetRoutingDomainDetail {
-    fn default() -> Self {
-        Self {
-            common: RoutingDomainDetailCommon::new(RoutingDomain::PublicInternet),
-        }
-    }
-}
-
-fn first_filtered_dial_info_detail_between_nodes(
-    from_node: &NodeInfo,
-    to_node: &NodeInfo,
-    dial_info_filter: &DialInfoFilter,
-    sequencing: Sequencing,
-    dif_sort: Option<Arc<DialInfoDetailSort>>,
-) -> Option<DialInfoDetail> {
-    // Consider outbound capabilities
-    let dial_info_filter = (*dial_info_filter).filtered(
-        &DialInfoFilter::all()
-            .with_address_type_set(from_node.address_types())
-            .with_protocol_type_set(from_node.outbound_protocols()),
-    );
-
-    // Apply sequencing and get sort
-    // Include sorting by external dial info sort for rotating through dialinfo
-    // based on an external preference table, for example the one kept by
-    // AddressFilter to deprioritize dialinfo that have recently failed to connect
-    let (ordered, dial_info_filter) = dial_info_filter.with_sequencing(sequencing);
-    let sort: Option<Box<DialInfoDetailSort>> = if ordered {
-        if let Some(dif_sort) = dif_sort {
-            Some(Box::new(move |a, b| {
-                let mut ord = dif_sort(a, b);
-                if ord == core::cmp::Ordering::Equal {
-                    ord = DialInfoDetail::ordered_sequencing_sort(a, b);
-                }
-                ord
-            }))
-        } else {
-            Some(Box::new(move |a, b| {
-                DialInfoDetail::ordered_sequencing_sort(a, b)
-            }))
-        }
-    } else if let Some(dif_sort) = dif_sort {
-        Some(Box::new(move |a, b| dif_sort(a, b)))
-    } else {
-        None
-    };
-
-    // If the filter is dead then we won't be able to connect
-    if dial_info_filter.is_dead() {
-        return None;
-    }
-
-    // Get the best match dial info for node B if we have it
-    let direct_filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter);
-    to_node.first_filtered_dial_info_detail(sort, direct_filter)
-}
-
-impl RoutingDomainDetail for PublicInternetRoutingDomainDetail {
-    fn common(&self) -> &RoutingDomainDetailCommon {
-        &self.common
-    }
-    fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon {
-        &mut self.common
-    }
-    fn can_contain_address(&self, address: Address) -> bool {
-        address.is_global()
-    }
-    fn get_contact_method(
-        &self,
-        rti: &RoutingTableInner,
-        peer_a: &PeerInfo,
-        peer_b: &PeerInfo,
-        dial_info_filter: DialInfoFilter,
-        sequencing: Sequencing,
-        dif_sort: Option<Arc<DialInfoDetailSort>>,
-    ) -> ContactMethod {
-        let ip6_prefix_size = rti
-            .unlocked_inner
-            .config
-            .get()
-            .network
-            .max_connections_per_ip6_prefix_size as usize;
-
-        // Get the nodeinfos for convenience
-        let node_a = peer_a.signed_node_info().node_info();
-        let node_b = peer_b.signed_node_info().node_info();
-
-        // Check to see if these nodes are on the same network
-        let same_ipblock = node_a.node_is_on_same_ipblock(node_b, ip6_prefix_size);
-
-        // Get the node ids that would be used between these peers
-        let cck = common_crypto_kinds(&peer_a.node_ids().kinds(), &peer_b.node_ids().kinds());
-        let Some(best_ck) = cck.first().copied() else {
-            // No common crypto kinds between these nodes, can't contact
-            return ContactMethod::Unreachable;
-        };
-
-        //let node_a_id = peer_a.node_ids().get(best_ck).unwrap();
-        let node_b_id = peer_b.node_ids().get(best_ck).unwrap();
-
-        // Get the best match dial info for node B if we have it
-        // Don't try direct inbound at all if the two nodes are on the same ipblock to avoid hairpin NAT issues
-        // as well avoiding direct traffic between same-network nodes. This would be done in the LocalNetwork RoutingDomain.
-        if let Some(target_did) = (!same_ipblock)
-            .then(|| {
-                first_filtered_dial_info_detail_between_nodes(
-                    node_a,
-                    node_b,
-                    &dial_info_filter,
-                    sequencing,
-                    dif_sort.clone(),
-                )
-            })
-            .flatten()
-        {
-            // Do we need to signal before going inbound?
-            if !target_did.class.requires_signal() {
-                // Go direct without signaling
-                return ContactMethod::Direct(target_did.dial_info);
-            }
-
-            // Get the target's inbound relay, it must have one or it is not reachable
-            if let Some(node_b_relay) = peer_b.signed_node_info().relay_info() {
-                // Note that relay_peer_info could be node_a, in which case a connection already exists
-                // and we only get here if the connection had dropped, in which case node_a is unreachable until
-                // it gets a new relay connection up
-                if peer_b
-                    .signed_node_info()
-                    .relay_ids()
-                    .contains_any(peer_a.node_ids())
-                {
-                    return ContactMethod::Existing;
-                }
-
-                // Get best node id to contact relay with
-                let Some(node_b_relay_id) = peer_b.signed_node_info().relay_ids().get(best_ck)
-                else {
-                    // No best relay id
-                    return ContactMethod::Unreachable;
-                };
-
-                // Can node A reach the inbound relay directly?
-                if first_filtered_dial_info_detail_between_nodes(
-                    node_a,
-                    node_b_relay,
-                    &dial_info_filter,
-                    sequencing,
-                    dif_sort.clone(),
-                )
-                .is_some()
-                {
-                    // Can node A receive anything inbound ever?
-                    if matches!(node_a.network_class(), NetworkClass::InboundCapable) {
-                        ///////// Reverse connection
-
-                        // Get the best match dial info for an reverse inbound connection from node B to node A
-                        if let Some(reverse_did) = first_filtered_dial_info_detail_between_nodes(
-                            node_b,
-                            node_a,
-                            &dial_info_filter,
-                            sequencing,
-                            dif_sort.clone(),
-                        ) {
-                            // Ensure we aren't on the same public IP address (no hairpin nat)
-                            if reverse_did.dial_info.ip_addr() != target_did.dial_info.ip_addr() {
-                                // Can we receive a direct reverse connection?
-                                if !reverse_did.class.requires_signal() {
-                                    return ContactMethod::SignalReverse(
-                                        node_b_relay_id,
-                                        node_b_id,
-                                    );
-                                }
-                            }
-                        }
-
-                        ///////// UDP hole-punch
-
-                        // Does node B have a direct udp dialinfo node A can reach?
-                        let udp_dial_info_filter = dial_info_filter
-                            .filtered(&DialInfoFilter::all().with_protocol_type(ProtocolType::UDP));
-                        if let Some(target_udp_did) = first_filtered_dial_info_detail_between_nodes(
-                            node_a,
-                            node_b,
-                            &udp_dial_info_filter,
-                            sequencing,
-                            dif_sort.clone(),
-                        ) {
-                            // Does node A have a direct udp dialinfo that node B can reach?
-                            if let Some(reverse_udp_did) =
-                                first_filtered_dial_info_detail_between_nodes(
-                                    node_b,
-                                    node_a,
-                                    &udp_dial_info_filter,
-                                    sequencing,
-                                    dif_sort.clone(),
-                                )
-                            {
-                                // Ensure we aren't on the same public IP address (no hairpin nat)
-                                if reverse_udp_did.dial_info.ip_addr()
-                                    != target_udp_did.dial_info.ip_addr()
-                                {
-                                    // The target and ourselves have a udp dialinfo that they can reach
-                                    return ContactMethod::SignalHolePunch(
-                                        node_b_relay_id,
-                                        node_b_id,
-                                    );
-                                }
-                            }
-                        }
-                        // Otherwise we have to inbound relay
-                    }
-
-                    return ContactMethod::InboundRelay(node_b_relay_id);
-                }
-            }
-        }
-        // If the node B has no direct dial info or is on the same ipblock, it needs to have an inbound relay
-        else if let Some(node_b_relay) = peer_b.signed_node_info().relay_info() {
-            // Note that relay_peer_info could be node_a, in which case a connection already exists
-            // and we only get here if the connection had dropped, in which case node_b is unreachable until
-            // it gets a new relay connection up
-            if peer_b
-                .signed_node_info()
-                .relay_ids()
-                .contains_any(peer_a.node_ids())
-            {
-                return ContactMethod::Existing;
-            }
-
-            // Get best node id to contact relay with
-            let Some(node_b_relay_id) = peer_b.signed_node_info().relay_ids().get(best_ck) else {
-                // No best relay id
-                return ContactMethod::Unreachable;
-            };
-
-            // Can we reach the inbound relay?
-            if first_filtered_dial_info_detail_between_nodes(
-                node_a,
-                node_b_relay,
-                &dial_info_filter,
-                sequencing,
-                dif_sort.clone(),
-            )
-            .is_some()
-            {
-                ///////// Reverse connection
-
-                // Get the best match dial info for an reverse inbound connection from node B to node A
-                // unless both nodes are on the same ipblock
-                if let Some(reverse_did) = (!same_ipblock)
-                    .then(|| {
-                        first_filtered_dial_info_detail_between_nodes(
-                            node_b,
-                            node_a,
-                            &dial_info_filter,
-                            sequencing,
-                            dif_sort.clone(),
-                        )
-                    })
-                    .flatten()
-                {
-                    // Can we receive a direct reverse connection?
-                    if !reverse_did.class.requires_signal() {
-                        return ContactMethod::SignalReverse(node_b_relay_id, node_b_id);
-                    }
-                }
-
-                return ContactMethod::InboundRelay(node_b_relay_id);
-            }
-        }
-
-        // If node A can't reach the node by other means, it may need to use its outbound relay
-        if peer_a
-            .signed_node_info()
-            .node_info()
-            .network_class()
-            .outbound_wants_relay()
-        {
-            if let Some(node_a_relay_id) = peer_a.signed_node_info().relay_ids().get(best_ck) {
-                // Ensure it's not our relay we're trying to reach
-                if node_a_relay_id != node_b_id {
-                    return ContactMethod::OutboundRelay(node_a_relay_id);
-                }
-            }
-        }
-
-        ContactMethod::Unreachable
-    }
-}
-
-/// Local Network routing domain internals
-#[derive(Debug)]
-pub struct LocalNetworkRoutingDomainDetail {
-    /// The local networks this domain will communicate with
-    local_networks: Vec<(IpAddr, IpAddr)>,
-    /// Common implementation for all routing domains
-    common: RoutingDomainDetailCommon,
-}
-
-impl Default for LocalNetworkRoutingDomainDetail {
-    fn default() -> Self {
-        Self {
-            local_networks: Default::default(),
-            common: RoutingDomainDetailCommon::new(RoutingDomain::LocalNetwork),
-        }
-    }
-}
-
-impl LocalNetworkRoutingDomainDetail {
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
-    pub fn set_local_networks(&mut self, mut local_networks: Vec<(IpAddr, IpAddr)>) -> bool {
-        local_networks.sort();
-        if local_networks == self.local_networks {
-            return false;
-        }
-        self.local_networks = local_networks;
-        true
-    }
-}
-
-impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail {
-    fn common(&self) -> &RoutingDomainDetailCommon {
-        &self.common
-    }
-    fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon {
-        &mut self.common
-    }
-    fn can_contain_address(&self, address: Address) -> bool {
-        let ip = address.ip_addr();
-        for localnet in &self.local_networks {
-            if ipaddr_in_network(ip, localnet.0, localnet.1) {
-                return true;
-            }
-        }
-        false
-    }
-
-    fn get_contact_method(
-        &self,
-        _rti: &RoutingTableInner,
-        peer_a: &PeerInfo,
-        peer_b: &PeerInfo,
-        dial_info_filter: DialInfoFilter,
-        sequencing: Sequencing,
-        dif_sort: Option<Arc<DialInfoDetailSort>>,
-    ) -> ContactMethod {
-        // Get the nodeinfos for convenience
-        let node_a = peer_a.signed_node_info().node_info();
-        let node_b = peer_b.signed_node_info().node_info();
-
-        // Get the node ids that would be used between these peers
-        let cck = common_crypto_kinds(&peer_a.node_ids().kinds(), &peer_b.node_ids().kinds());
-        let Some(_best_ck) = cck.first().copied() else {
-            // No common crypto kinds between these nodes, can't contact
-            return ContactMethod::Unreachable;
-        };
-
-        if let Some(target_did) = first_filtered_dial_info_detail_between_nodes(
-            node_a,
-            node_b,
-            &dial_info_filter,
-            sequencing,
-            dif_sort,
-        ) {
-            return ContactMethod::Direct(target_did.dial_info);
-        }
-
-        ContactMethod::Unreachable
-    }
-}

veilid-core/src/routing_table/routing_table_inner/mod.rs

@@ -1,8 +1,15 @@
+mod routing_domains;
+
 use super::*;
+pub use routing_domains::*;
+
 use weak_table::PtrWeakHashSet;
 
 pub const RECENT_PEERS_TABLE_SIZE: usize = 64;
 
+// Critical sections
+pub const LOCK_TAG_TICK: &str = "TICK";
+
 pub type EntryCounts = BTreeMap<(RoutingDomain, CryptoKind), usize>;
 //////////////////////////////////////////////////////////////////////////
 
@@ -87,34 +94,36 @@ impl RoutingTableInner {
         }
     }
 
-    pub fn with_routing_domain_mut<F, R>(&mut self, domain: RoutingDomain, f: F) -> R
+    fn with_public_internet_routing_domain_mut<F, R>(&mut self, f: F) -> R
     where
-        F: FnOnce(&mut dyn RoutingDomainDetail) -> R,
+        F: FnOnce(&mut PublicInternetRoutingDomainDetail) -> R,
     {
-        match domain {
+        f(&mut self.public_internet_routing_domain)
-            RoutingDomain::PublicInternet => f(&mut self.public_internet_routing_domain),
+    }
-            RoutingDomain::LocalNetwork => f(&mut self.local_network_routing_domain),
+    fn with_local_network_routing_domain_mut<F, R>(&mut self, f: F) -> R
-        }
+    where
+        F: FnOnce(&mut LocalNetworkRoutingDomainDetail) -> R,
+    {
+        f(&mut self.local_network_routing_domain)
     }
 
-    pub fn relay_node(&self, domain: RoutingDomain) -> Option<NodeRef> {
+    pub fn relay_node(&self, domain: RoutingDomain) -> Option<FilteredNodeRef> {
-        self.with_routing_domain(domain, |rd| rd.common().relay_node())
+        self.with_routing_domain(domain, |rdd| rdd.relay_node())
     }
 
     pub fn relay_node_last_keepalive(&self, domain: RoutingDomain) -> Option<Timestamp> {
-        self.with_routing_domain(domain, |rd| rd.common().relay_node_last_keepalive())
+        self.with_routing_domain(domain, |rdd| rdd.relay_node_last_keepalive())
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn has_dial_info(&self, domain: RoutingDomain) -> bool {
-        self.with_routing_domain(domain, |rd| !rd.common().dial_info_details().is_empty())
+        self.with_routing_domain(domain, |rdd| !rdd.dial_info_details().is_empty())
     }
 
     pub fn dial_info_details(&self, domain: RoutingDomain) -> Vec<DialInfoDetail> {
-        self.with_routing_domain(domain, |rd| rd.common().dial_info_details().clone())
+        self.with_routing_domain(domain, |rdd| rdd.dial_info_details().clone())
     }
 
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
     pub fn first_filtered_dial_info_detail(
         &self,
         routing_domain_set: RoutingDomainSet,
@@ -124,8 +133,8 @@ impl RoutingTableInner {
             return None;
         }
         for routing_domain in routing_domain_set {
-            let did = self.with_routing_domain(routing_domain, |rd| {
+            let did = self.with_routing_domain(routing_domain, |rdd| {
-                for did in rd.common().dial_info_details() {
+                for did in rdd.dial_info_details() {
                     if did.matches_filter(filter) {
                         return Some(did.clone());
                     }
@@ -149,8 +158,8 @@ impl RoutingTableInner {
             return ret;
         }
         for routing_domain in routing_domain_set {
-            self.with_routing_domain(routing_domain, |rd| {
+            self.with_routing_domain(routing_domain, |rdd| {
-                for did in rd.common().dial_info_details() {
+                for did in rdd.dial_info_details() {
                     if did.matches_filter(filter) {
                         ret.push(did.clone());
                     }
@@ -161,37 +170,20 @@ impl RoutingTableInner {
         ret
     }
 
-    pub fn ensure_dial_info_is_valid(&self, domain: RoutingDomain, dial_info: &DialInfo) -> bool {
-        let address = dial_info.socket_address().address();
-        let can_contain_address =
-            self.with_routing_domain(domain, |rd| rd.can_contain_address(address));
-
-        if !can_contain_address {
-            log_rtab!(debug "can not add dial info to this routing domain");
-            return false;
-        }
-        if !dial_info.is_valid() {
-            log_rtab!(debug
-                "shouldn't be registering invalid addresses: {:?}",
-                dial_info
-            );
-            return false;
-        }
-        true
-    }
-
     pub fn node_info_is_valid_in_routing_domain(
         &self,
         routing_domain: RoutingDomain,
         node_info: &NodeInfo,
     ) -> bool {
         // Ensure all of the dial info works in this routing domain
-        for did in node_info.dial_info_detail_list() {
+        self.with_routing_domain(routing_domain, |rdd| {
-            if !self.ensure_dial_info_is_valid(routing_domain, &did.dial_info) {
+            for did in node_info.dial_info_detail_list() {
-                return false;
+                if !rdd.ensure_dial_info_is_valid(&did.dial_info) {
+                    return false;
+                }
             }
-        }
+            true
-        true
+        })
     }
 
     pub fn signed_node_info_is_valid_in_routing_domain(
@@ -223,8 +215,8 @@ impl RoutingTableInner {
     pub fn get_contact_method(
         &self,
         routing_domain: RoutingDomain,
-        peer_a: &PeerInfo,
+        peer_a: Arc<PeerInfo>,
-        peer_b: &PeerInfo,
+        peer_b: Arc<PeerInfo>,
         dial_info_filter: DialInfoFilter,
         sequencing: Sequencing,
         dif_sort: Option<Arc<DialInfoDetailSort>>,
@@ -244,41 +236,42 @@ impl RoutingTableInner {
         });
     }
 
+    /// Publish the node's current peer info to the world if it is valid
+    pub fn publish_peer_info(&mut self, routing_domain: RoutingDomain) -> bool {
+        self.with_routing_domain(routing_domain, |rdd| rdd.publish_peer_info(self))
+    }
+    /// Unpublish the node's current peer info
+    pub fn unpublish_peer_info(&mut self, routing_domain: RoutingDomain) {
+        self.with_routing_domain(routing_domain, |rdd| rdd.unpublish_peer_info())
+    }
+
+    /// Get the current published peer info
+    pub fn get_published_peer_info(&self, routing_domain: RoutingDomain) -> Option<Arc<PeerInfo>> {
+        self.with_routing_domain(routing_domain, |rdd| rdd.get_published_peer_info())
+    }
+
     /// Return if this routing domain has a valid network class
     pub fn has_valid_network_class(&self, routing_domain: RoutingDomain) -> bool {
-        self.with_routing_domain(routing_domain, |rdd| rdd.common().has_valid_network_class())
+        self.with_routing_domain(routing_domain, |rdd| rdd.has_valid_network_class())
     }
 
-    /// Return a copy of our node's peerinfo
+    /// Return a copy of our node's current peerinfo (may not yet be published)
-    pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo {
+    pub fn get_current_peer_info(&self, routing_domain: RoutingDomain) -> Arc<PeerInfo> {
-        self.with_routing_domain(routing_domain, |rdd| {
+        self.with_routing_domain(routing_domain, |rdd| rdd.get_peer_info(self))
-            rdd.common().with_peer_info(self, |pi| pi.clone())
-        })
-    }
-
-    /// Return our current node info timestamp
-    pub fn get_own_node_info_ts(&self, routing_domain: RoutingDomain) -> Timestamp {
-        self.with_routing_domain(routing_domain, |rdd| {
-            rdd.common()
-                .with_peer_info(self, |pi| pi.signed_node_info().timestamp())
-        })
     }
 
     /// Return the domain's currently registered network class
     pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option<NetworkClass> {
-        self.with_routing_domain(routing_domain, |rdd| rdd.common().network_class())
+        self.with_routing_domain(routing_domain, |rdd| rdd.network_class())
     }
 
     /// Return the domain's filter for what we can receivein the form of a dial info filter
-    #[allow(dead_code)]
     pub fn get_inbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter {
-        self.with_routing_domain(routing_domain, |rdd| {
+        self.with_routing_domain(routing_domain, |rdd| rdd.inbound_dial_info_filter())
-            rdd.common().inbound_dial_info_filter()
-        })
     }
 
     /// Return the domain's filter for what we can receive in the form of a node ref filter
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn get_inbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter {
         let dif = self.get_inbound_dial_info_filter(routing_domain);
         NodeRefFilter::new()
@@ -288,9 +281,7 @@ impl RoutingTableInner {
 
     /// Return the domain's filter for what we can send out in the form of a dial info filter
     pub fn get_outbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter {
-        self.with_routing_domain(routing_domain, |rdd| {
+        self.with_routing_domain(routing_domain, |rdd| rdd.outbound_dial_info_filter())
-            rdd.common().outbound_dial_info_filter()
-        })
     }
     /// Return the domain's filter for what we can receive in the form of a node ref filter
     pub fn get_outbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter {
@@ -327,30 +318,6 @@ impl RoutingTableInner {
         }
     }
 
-    #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
-    pub fn configure_local_network_routing_domain(
-        &mut self,
-        local_networks: Vec<(IpAddr, IpAddr)>,
-    ) {
-        log_net!(debug "configure_local_network_routing_domain: {:#?}", local_networks);
-
-        let changed = self
-            .local_network_routing_domain
-            .set_local_networks(local_networks);
-
-        // If the local network topology has changed, nuke the existing local node info and let new local discovery happen
-        if changed {
-            let cur_ts = Timestamp::now();
-            self.with_entries_mut(cur_ts, BucketEntryState::Dead, |rti, e| {
-                e.with_mut(rti, |_rti, e| {
-                    e.clear_signed_node_info(RoutingDomain::LocalNetwork);
-                    e.reset_updated_since_last_network_change();
-                });
-                Option::<()>::None
-            });
-        }
-    }
-
     /// Attempt to empty the routing table
     /// should only be performed when there are no node_refs (detached)
     pub fn purge_buckets(&mut self) {
@@ -401,11 +368,11 @@ impl RoutingTableInner {
         let bucket = self.get_bucket_mut(bucket_index);
         let bucket_depth = Self::bucket_depth(bucket_index);
 
-        if let Some(_dead_node_ids) = bucket.kick(bucket_depth, exempt_peers) {
+        if let Some(dead_node_ids) = bucket.kick(bucket_depth, exempt_peers) {
             // Remove expired entries
             self.all_entries.remove_expired();
 
-            log_rtab!(debug "Bucket {}:{} kicked Routing table now has {} nodes", bucket_index.0, bucket_index.1, self.bucket_entry_count());
+            log_rtab!(debug "Bucket {}:{} kicked Routing table now has {} nodes\nKicked nodes:{:#?}", bucket_index.0, bucket_index.1, self.bucket_entry_count(), dead_node_ids);
 
             // Now purge the routing table inner vectors
             //let filter = |k: &DHTKey| dead_node_ids.contains(k);
@@ -515,11 +482,13 @@ impl RoutingTableInner {
         outer_self: RoutingTable,
         routing_domain: RoutingDomain,
         cur_ts: Timestamp,
-    ) -> Vec<NodeRef> {
+    ) -> Vec<FilteredNodeRef> {
-        let own_node_info_ts = self.get_own_node_info_ts(routing_domain);
+        let opt_own_node_info_ts = self
+            .get_published_peer_info(routing_domain)
+            .map(|pi| pi.signed_node_info().timestamp());
 
         // Collect all entries that are 'needs_ping' and have some node info making them reachable somehow
-        let mut node_refs = Vec::<NodeRef>::with_capacity(self.bucket_entry_count());
+        let mut node_refs = Vec::<FilteredNodeRef>::with_capacity(self.bucket_entry_count());
         self.with_entries(cur_ts, BucketEntryState::Unreliable, |rti, entry| {
             let entry_needs_ping = |e: &BucketEntryInner| {
                 // If this entry isn't in the routing domain we are checking, don't include it
@@ -534,7 +503,9 @@ impl RoutingTableInner {
                 }
 
                 // If this entry needs a ping because this node hasn't seen our latest node info, then do it
-                if !e.has_seen_our_node_info_ts(routing_domain, own_node_info_ts) {
+                if opt_own_node_info_ts.is_some()
+                    && !e.has_seen_our_node_info_ts(routing_domain, opt_own_node_info_ts.unwrap())
+                {
                     return true;
                 }
 
@@ -547,10 +518,11 @@ impl RoutingTableInner {
             };
 
             if entry.with_inner(entry_needs_ping) {
-                node_refs.push(NodeRef::new(
+                node_refs.push(FilteredNodeRef::new(
                     outer_self.clone(),
                     entry,
-                    Some(NodeRefFilter::new().with_routing_domain(routing_domain)),
+                    NodeRefFilter::new().with_routing_domain(routing_domain),
+                    Sequencing::default(),
                 ));
             }
             Option::<()>::None
@@ -558,11 +530,11 @@ impl RoutingTableInner {
         node_refs
     }
 
-    #[allow(dead_code)]
+    #[expect(dead_code)]
     pub fn get_all_alive_nodes(&self, outer_self: RoutingTable, cur_ts: Timestamp) -> Vec<NodeRef> {
         let mut node_refs = Vec::<NodeRef>::with_capacity(self.bucket_entry_count());
         self.with_entries(cur_ts, BucketEntryState::Unreliable, |_rti, entry| {
-            node_refs.push(NodeRef::new(outer_self.clone(), entry, None));
+            node_refs.push(NodeRef::new(outer_self.clone(), entry));
             Option::<()>::None
         });
         node_refs
@@ -673,7 +645,7 @@ impl RoutingTableInner {
             }
 
             // Make a noderef to return
-            let nr = NodeRef::new(outer_self.clone(), best_entry.clone(), None);
+            let nr = NodeRef::new(outer_self.clone(), best_entry.clone());
 
             // Update the entry with the update func
             best_entry.with_mut_inner(|e| update_func(self, e));
@@ -696,7 +668,7 @@ impl RoutingTableInner {
         }
 
         // Make node ref to return
-        let nr = NodeRef::new(outer_self.clone(), new_entry.clone(), None);
+        let nr = NodeRef::new(outer_self.clone(), new_entry.clone());
 
         // Update the entry with the update func
         new_entry.with_mut_inner(|e| update_func(self, e));
@@ -742,7 +714,7 @@ impl RoutingTableInner {
         let bucket = self.get_bucket(bucket_index);
         Ok(bucket
             .entry(&node_id.value)
-            .map(|e| NodeRef::new(outer_self, e, None)))
+            .map(|e| NodeRef::new(outer_self, e)))
     }
 
     /// Resolve an existing routing table entry and return a filtered reference to it
@@ -753,10 +725,10 @@ impl RoutingTableInner {
         node_id: TypedKey,
         routing_domain_set: RoutingDomainSet,
         dial_info_filter: DialInfoFilter,
-    ) -> EyreResult<Option<NodeRef>> {
+    ) -> EyreResult<Option<FilteredNodeRef>> {
         let nr = self.lookup_node_ref(outer_self, node_id)?;
         Ok(nr.map(|nr| {
-            nr.filtered_clone(
+            nr.custom_filtered(
                 NodeRefFilter::new()
                     .with_dial_info_filter(dial_info_filter)
                     .with_routing_domain_set(routing_domain_set),
@@ -789,10 +761,11 @@ impl RoutingTableInner {
     pub fn register_node_with_peer_info(
         &mut self,
         outer_self: RoutingTable,
-        routing_domain: RoutingDomain,
+        peer_info: Arc<PeerInfo>,
-        peer_info: PeerInfo,
         allow_invalid: bool,
-    ) -> EyreResult<NodeRef> {
+    ) -> EyreResult<FilteredNodeRef> {
+        let routing_domain = peer_info.routing_domain();
+
         // if our own node is in the list, then ignore it as we don't add ourselves to our own routing table
         if self
             .unlocked_inner
@@ -830,30 +803,41 @@ impl RoutingTableInner {
         }
 
         // Register relay info first if we have that and the relay isn't us
-        if let Some(relay_peer_info) = peer_info.signed_node_info().relay_peer_info() {
+        if let Some(relay_peer_info) = peer_info.signed_node_info().relay_peer_info(routing_domain)
+        {
             if !self
                 .unlocked_inner
                 .matches_own_node_id(relay_peer_info.node_ids())
             {
-                self.register_node_with_peer_info(
+                self.register_node_with_peer_info(outer_self.clone(), relay_peer_info, false)?;
-                    outer_self.clone(),
-                    routing_domain,
-                    relay_peer_info,
-                    false,
-                )?;
             }
         }
 
-        let (node_ids, signed_node_info) = peer_info.destructure();
+        let (_routing_domain, node_ids, signed_node_info) =
-        let mut nr = self.create_node_ref(outer_self, &node_ids, |_rti, e| {
+            Arc::unwrap_or_clone(peer_info).destructure();
-            e.update_signed_node_info(routing_domain, signed_node_info);
+        let mut updated = false;
+        let nr = self.create_node_ref(outer_self, &node_ids, |_rti, e| {
+            updated = e.update_signed_node_info(routing_domain, signed_node_info);
         })?;
 
-        nr.set_filter(Some(
+        if updated {
-            NodeRefFilter::new().with_routing_domain(routing_domain),
+            // If this is our relay, then redo our own peerinfo because
-        ));
+            // if we have relayed peerinfo, then changing the relay's peerinfo
+            // changes our own peer info
+            self.with_routing_domain(routing_domain, |rd| {
+                let opt_our_relay_node_ids = rd
+                    .relay_node()
+                    .map(|relay_nr| relay_nr.locked(self).node_ids());
+                if let Some(our_relay_node_ids) = opt_our_relay_node_ids {
+                    if our_relay_node_ids.contains_any(&node_ids) {
+                        rd.refresh();
+                        rd.publish_peer_info(self);
+                    }
+                }
+            });
+        }
 
-        Ok(nr)
+        Ok(nr.custom_filtered(NodeRefFilter::new().with_routing_domain(routing_domain)))
     }
 
     /// Shortcut function to add a node to our routing table if it doesn't exist
@@ -862,16 +846,20 @@ impl RoutingTableInner {
     pub fn register_node_with_existing_connection(
         &mut self,
         outer_self: RoutingTable,
+        routing_domain: RoutingDomain,
         node_id: TypedKey,
         flow: Flow,
         timestamp: Timestamp,
-    ) -> EyreResult<NodeRef> {
+    ) -> EyreResult<FilteredNodeRef> {
         let nr = self.create_node_ref(outer_self, &TypedKeyGroup::from(node_id), |_rti, e| {
             //e.make_not_dead(timestamp);
             e.touch_last_seen(timestamp);
         })?;
         // set the most recent node address for connection finding and udp replies
         nr.locked_mut(self).set_last_flow(flow, timestamp);
+
+        // Enforce routing domain
+        let nr = nr.custom_filtered(NodeRefFilter::new().with_routing_domain(routing_domain));
         Ok(nr)
     }
 
@@ -940,34 +928,41 @@ impl RoutingTableInner {
 
     // Retrieve the fastest nodes in the routing table matching an entry filter
     #[instrument(level = "trace", skip_all)]
-    pub fn find_fast_public_nodes_filtered(
+    pub fn find_fast_non_local_nodes_filtered(
         &self,
         outer_self: RoutingTable,
+        routing_domain: RoutingDomain,
         node_count: usize,
         mut filters: VecDeque<RoutingTableEntryFilter>,
     ) -> Vec<NodeRef> {
-        let public_node_filter =
+        assert_ne!(
-            Box::new(|_rti: &RoutingTableInner, v: Option<Arc<BucketEntry>>| {
+            routing_domain,
+            RoutingDomain::LocalNetwork,
+            "LocalNetwork is not a valid non-local RoutingDomain"
+        );
+        let public_node_filter = Box::new(
+            move |_rti: &RoutingTableInner, v: Option<Arc<BucketEntry>>| {
                 let entry = v.unwrap();
                 entry.with_inner(|e| {
                     // skip nodes on local network
                     if e.node_info(RoutingDomain::LocalNetwork).is_some() {
                         return false;
                     }
-                    // skip nodes not on public internet
+                    // skip nodes not on desired routing domain
-                    if e.node_info(RoutingDomain::PublicInternet).is_none() {
+                    if e.node_info(routing_domain).is_none() {
                         return false;
                     }
                     true
                 })
-            }) as RoutingTableEntryFilter;
+            },
+        ) as RoutingTableEntryFilter;
         filters.push_front(public_node_filter);
 
         self.find_preferred_fastest_nodes(
             node_count,
             filters,
             |_rti: &RoutingTableInner, v: Option<Arc<BucketEntry>>| {
-                NodeRef::new(outer_self.clone(), v.unwrap().clone(), None)
+                NodeRef::new(outer_self.clone(), v.unwrap().clone())
             },
         )
     }
@@ -996,12 +991,14 @@ impl RoutingTableInner {
     pub fn transform_to_peer_info(
         &self,
         routing_domain: RoutingDomain,
-        own_peer_info: &PeerInfo,
+        own_peer_info: Arc<PeerInfo>,
         entry: Option<Arc<BucketEntry>>,
-    ) -> PeerInfo {
+    ) -> Arc<PeerInfo> {
         match entry {
             None => own_peer_info.clone(),
-            Some(entry) => entry.with_inner(|e| e.make_peer_info(routing_domain).unwrap()),
+            Some(entry) => {
+                entry.with_inner(|e| Arc::new(e.make_peer_info(routing_domain).unwrap()))
+            }
         }
     }
 
@@ -1242,7 +1239,7 @@ impl RoutingTableInner {
     ) -> Vec<NodeRef> {
         // Lock all noderefs
         let kind = node_id.kind;
-        let mut closest_nodes_locked: Vec<NodeRefLocked> = closest_nodes
+        let mut closest_nodes_locked: Vec<LockedNodeRef> = closest_nodes
             .iter()
             .filter_map(|nr| {
                 let nr_locked = nr.locked(self);
@@ -1267,12 +1264,12 @@ impl RoutingTableInner {
 pub(crate) fn make_closest_noderef_sort(
     crypto: Crypto,
     node_id: TypedKey,
-) -> impl Fn(&NodeRefLocked, &NodeRefLocked) -> core::cmp::Ordering {
+) -> impl Fn(&LockedNodeRef, &LockedNodeRef) -> core::cmp::Ordering {
     let kind = node_id.kind;
     // Get cryptoversion to check distance with
     let vcrypto = crypto.get(kind).unwrap();
 
-    move |a: &NodeRefLocked, b: &NodeRefLocked| -> core::cmp::Ordering {
+    move |a: &LockedNodeRef, b: &LockedNodeRef| -> core::cmp::Ordering {
         // same nodes are always the same
         if a.same_entry(b) {
             return core::cmp::Ordering::Equal;

veilid-core/src/routing_table/routing_table_inner/routing_domains/editor.rs

@@ -0,0 +1,116 @@
+use super::*;
+
+pub trait RoutingDomainEditorCommonTrait {
+    fn clear_dial_info_details(
+        &mut self,
+        address_type: Option<AddressType>,
+        protocol_type: Option<ProtocolType>,
+    ) -> &mut Self;
+    fn clear_relay_node(&mut self) -> &mut Self;
+    fn set_relay_node(&mut self, relay_node: NodeRef) -> &mut Self;
+    fn set_relay_node_keepalive(&mut self, ts: Option<Timestamp>) -> &mut Self;
+    #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
+    fn add_dial_info(&mut self, dial_info: DialInfo, class: DialInfoClass) -> &mut Self;
+    fn setup_network(
+        &mut self,
+        outbound_protocols: ProtocolTypeSet,
+        inbound_protocols: ProtocolTypeSet,
+        address_types: AddressTypeSet,
+        capabilities: Vec<Capability>,
+    ) -> &mut Self;
+    fn set_network_class(&mut self, network_class: Option<NetworkClass>) -> &mut Self;
+    fn commit(&mut self, pause_tasks: bool) -> SendPinBoxFutureLifetime<'_, bool>;
+    fn shutdown(&mut self) -> SendPinBoxFutureLifetime<'_, ()>;
+    fn publish(&mut self);
+}
+
+pub(super) trait RoutingDomainDetailApplyCommonChange {
+    /// Make a change from the routing domain editor
+    fn apply_common_change(&mut self, change: RoutingDomainChangeCommon);
+}
+
+impl<T: RoutingDomainDetailCommonAccessors> RoutingDomainDetailApplyCommonChange for T {
+    /// Make a change from the routing domain editor
+    fn apply_common_change(&mut self, change: RoutingDomainChangeCommon) {
+        match change {
+            RoutingDomainChangeCommon::ClearDialInfoDetails {
+                address_type,
+                protocol_type,
+            } => {
+                self.common_mut()
+                    .clear_dial_info_details(address_type, protocol_type);
+            }
+
+            RoutingDomainChangeCommon::ClearRelayNode => {
+                self.common_mut().set_relay_node(None);
+            }
+
+            RoutingDomainChangeCommon::SetRelayNode { relay_node } => {
+                self.common_mut().set_relay_node(Some(relay_node.clone()))
+            }
+
+            RoutingDomainChangeCommon::SetRelayNodeKeepalive { ts } => {
+                self.common_mut().set_relay_node_last_keepalive(ts);
+            }
+            RoutingDomainChangeCommon::AddDialInfo { dial_info_detail } => {
+                if !self.ensure_dial_info_is_valid(&dial_info_detail.dial_info) {
+                    return;
+                }
+
+                self.common_mut()
+                    .add_dial_info_detail(dial_info_detail.clone());
+            }
+            // RoutingDomainChange::RemoveDialInfoDetail { dial_info_detail } => {
+            //     self.common
+            //         .remove_dial_info_detail(dial_info_detail.clone());
+            // }
+            RoutingDomainChangeCommon::SetupNetwork {
+                outbound_protocols,
+                inbound_protocols,
+                address_types,
+                capabilities,
+            } => {
+                self.common_mut().setup_network(
+                    outbound_protocols,
+                    inbound_protocols,
+                    address_types,
+                    capabilities.clone(),
+                );
+            }
+            RoutingDomainChangeCommon::SetNetworkClass { network_class } => {
+                self.common_mut().set_network_class(network_class);
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub(super) enum RoutingDomainChangeCommon {
+    ClearDialInfoDetails {
+        address_type: Option<AddressType>,
+        protocol_type: Option<ProtocolType>,
+    },
+    ClearRelayNode,
+    SetRelayNode {
+        relay_node: NodeRef,
+    },
+    SetRelayNodeKeepalive {
+        ts: Option<Timestamp>,
+    },
+    AddDialInfo {
+        dial_info_detail: DialInfoDetail,
+    },
+    // #[cfg_attr(target_arch = "wasm32", expect(dead_code))]
+    // RemoveDialInfoDetail {
+    //     dial_info_detail: DialInfoDetail,
+    // },
+    SetupNetwork {
+        outbound_protocols: ProtocolTypeSet,
+        inbound_protocols: ProtocolTypeSet,
+        address_types: AddressTypeSet,
+        capabilities: Vec<Capability>,
+    },
+    SetNetworkClass {
+        network_class: Option<NetworkClass>,
+    },
+}

veilid-core/src/routing_table/routing_table_inner/routing_domains/local_network/editor.rs

@@ -0,0 +1,270 @@
+#![cfg_attr(target_arch = "wasm32", expect(dead_code))]
+
+use super::*;
+
+#[derive(Debug)]
+enum RoutingDomainChangeLocalNetwork {
+    SetLocalNetworks {
+        local_networks: Vec<(IpAddr, IpAddr)>,
+    },
+    Common(RoutingDomainChangeCommon),
+}
+
+pub struct RoutingDomainEditorLocalNetwork {
+    routing_table: RoutingTable,
+    changes: Vec<RoutingDomainChangeLocalNetwork>,
+}
+
+impl RoutingDomainEditorLocalNetwork {
+    pub(in crate::routing_table) fn new(routing_table: RoutingTable) -> Self {
+        Self {
+            routing_table: routing_table.clone(),
+            changes: Vec::new(),
+        }
+    }
+
+    pub fn set_local_networks(&mut self, local_networks: Vec<(IpAddr, IpAddr)>) -> &mut Self {
+        self.changes
+            .push(RoutingDomainChangeLocalNetwork::SetLocalNetworks { local_networks });
+        self
+    }
+}
+
+impl RoutingDomainEditorCommonTrait for RoutingDomainEditorLocalNetwork {
+    #[instrument(level = "debug", skip(self))]
+    fn clear_dial_info_details(
+        &mut self,
+        address_type: Option<AddressType>,
+        protocol_type: Option<ProtocolType>,
+    ) -> &mut Self {
+        self.changes.push(RoutingDomainChangeLocalNetwork::Common(
+            RoutingDomainChangeCommon::ClearDialInfoDetails {
+                address_type,
+                protocol_type,
+            },
+        ));
+
+        self
+    }
+    #[instrument(level = "debug", skip(self))]
+    fn clear_relay_node(&mut self) -> &mut Self {
+        self.changes.push(RoutingDomainChangeLocalNetwork::Common(
+            RoutingDomainChangeCommon::ClearRelayNode,
+        ));
+        self
+    }
+    #[instrument(level = "debug", skip(self))]
+    fn set_relay_node(&mut self, relay_node: NodeRef) -> &mut Self {
+        self.changes.push(RoutingDomainChangeLocalNetwork::Common(
+            RoutingDomainChangeCommon::SetRelayNode { relay_node },
+        ));
+        self
+    }
+    #[instrument(level = "debug", skip(self))]
+    fn set_relay_node_keepalive(&mut self, ts: Option<Timestamp>) -> &mut Self {
+        self.changes.push(RoutingDomainChangeLocalNetwork::Common(
+            RoutingDomainChangeCommon::SetRelayNodeKeepalive { ts },
+        ));
+        self
+    }
+    #[instrument(level = "debug", skip(self))]
+    fn add_dial_info(&mut self, dial_info: DialInfo, class: DialInfoClass) -> &mut Self {
+        self.changes.push(RoutingDomainChangeLocalNetwork::Common(
+            RoutingDomainChangeCommon::AddDialInfo {
+                dial_info_detail: DialInfoDetail {
+                    dial_info: dial_info.clone(),
+                    class,
+                },
+            },
+        ));
+        self
+    }
+    // #[instrument(level = "debug", skip_all)]
+    // fn retain_dial_info<F: Fn(&DialInfo, DialInfoClass) -> bool>(
+    //     &mut self,
+    //     closure: F,
+    // ) -> EyreResult<&mut Self> {
+    //     let dids = self.routing_table.dial_info_details(self.routing_domain);
+    //     for did in dids {
+    //         if !closure(&did.dial_info, did.class) {
+    //             self.changes
+    //                 .push(RoutingDomainChangePublicInternet::Common(RoutingDomainChange::RemoveDialInfoDetail {
+    //                     dial_info_detail: did,
+    //                 }));
+    //         }
+    //     }
+
+    //     Ok(self)
+    // }
+
+    #[instrument(level = "debug", skip(self))]
+    fn setup_network(
+        &mut self,
+        outbound_protocols: ProtocolTypeSet,
+        inbound_protocols: ProtocolTypeSet,
+        address_types: AddressTypeSet,
+        capabilities: Vec<Capability>,
+    ) -> &mut Self {
+        self.changes.push(RoutingDomainChangeLocalNetwork::Common(
+            RoutingDomainChangeCommon::SetupNetwork {
+                outbound_protocols,
+                inbound_protocols,
+                address_types,
+                capabilities,
+            },
+        ));
+        self
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn set_network_class(&mut self, network_class: Option<NetworkClass>) -> &mut Self {
+        self.changes.push(RoutingDomainChangeLocalNetwork::Common(
+            RoutingDomainChangeCommon::SetNetworkClass { network_class },
+        ));
+        self
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn commit(&mut self, pause_tasks: bool) -> SendPinBoxFutureLifetime<'_, bool> {
+        Box::pin(async move {
+            // No locking if we have nothing to do
+            if self.changes.is_empty() {
+                return false;
+            }
+            // Briefly pause routing table ticker while changes are made
+            let _tick_guard = if pause_tasks {
+                Some(self.routing_table.pause_tasks().await)
+            } else {
+                None
+            };
+
+            // Apply changes
+            let mut peer_info_changed = false;
+
+            let mut rti_lock = self.routing_table.inner.write();
+            let rti = &mut rti_lock;
+            rti.with_local_network_routing_domain_mut(|detail| {
+                let old_dial_info_details = detail.dial_info_details().clone();
+                let old_relay_node = detail.relay_node();
+                let old_outbound_protocols = detail.outbound_protocols();
+                let old_inbound_protocols = detail.inbound_protocols();
+                let old_address_types = detail.address_types();
+                let old_capabilities = detail.capabilities();
+                let old_network_class = detail.network_class();
+
+                for change in self.changes.drain(..) {
+                    match change {
+                        RoutingDomainChangeLocalNetwork::Common(common_change) => {
+                            detail.apply_common_change(common_change);
+                        }
+                        RoutingDomainChangeLocalNetwork::SetLocalNetworks { local_networks } => {
+                            detail.set_local_networks(local_networks);
+                        }
+                    }
+                }
+
+                let new_dial_info_details = detail.dial_info_details().clone();
+                let new_relay_node = detail.relay_node();
+                let new_outbound_protocols = detail.outbound_protocols();
+                let new_inbound_protocols = detail.inbound_protocols();
+                let new_address_types = detail.address_types();
+                let new_capabilities = detail.capabilities();
+                let new_network_class = detail.network_class();
+
+                // Compare and see if peerinfo needs republication
+                let removed_dial_info = old_dial_info_details
+                    .iter()
+                    .filter(|di| !new_dial_info_details.contains(di))
+                    .collect::<Vec<_>>();
+                if !removed_dial_info.is_empty() {
+                    info!("[LocalNetwork] removed dial info: {:#?}", removed_dial_info);
+                    peer_info_changed = true;
+                }
+                let added_dial_info = new_dial_info_details
+                    .iter()
+                    .filter(|di| !old_dial_info_details.contains(di))
+                    .collect::<Vec<_>>();
+                if !added_dial_info.is_empty() {
+                    info!("[LocalNetwork] added dial info: {:#?}", added_dial_info);
+                    peer_info_changed = true;
+                }
+                if let Some(nrn) = new_relay_node {
+                    if let Some(orn) = old_relay_node {
+                        if !nrn.same_entry(&orn) {
+                            info!("[LocalNetwork] change relay: {} -> {}", orn, nrn);
+                            peer_info_changed = true;
+                        }
+                    } else {
+                        info!("[LocalNetwork] set relay: {}", nrn);
+                        peer_info_changed = true;
+                    }
+                }
+                if old_outbound_protocols != new_outbound_protocols {
+                    info!(
+                        "[LocalNetwork] changed network: outbound {:?}->{:?}\n",
+                        old_outbound_protocols, new_outbound_protocols
+                    );
+                    peer_info_changed = true;
+                }
+                if old_inbound_protocols != new_inbound_protocols {
+                    info!(
+                        "[LocalNetwork] changed network: inbound {:?}->{:?}\n",
+                        old_inbound_protocols, new_inbound_protocols,
+                    );
+                    peer_info_changed = true;
+                }
+                if old_address_types != new_address_types {
+                    info!(
+                        "[LocalNetwork] changed network: address types {:?}->{:?}\n",
+                        old_address_types, new_address_types,
+                    );
+                    peer_info_changed = true;
+                }
+                if old_capabilities != new_capabilities {
+                    info!(
+                        "[PublicInternet] changed network: capabilities {:?}->{:?}\n",
+                        old_capabilities, new_capabilities
+                    );
+                    peer_info_changed = true;
+                }
+                if old_network_class != new_network_class {
+                    info!(
+                        "[LocalNetwork] changed network class: {:?}->{:?}\n",
+                        old_network_class, new_network_class
+                    );
+                    peer_info_changed = true;
+                }
+            });
+
+            if peer_info_changed {
+                // Allow signed node info updates at same timestamp for otherwise dead nodes if our network has changed
+                rti.reset_all_updated_since_last_network_change();
+            }
+
+            peer_info_changed
+        })
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn publish(&mut self) {
+        self.routing_table
+            .inner
+            .write()
+            .publish_peer_info(RoutingDomain::LocalNetwork);
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn shutdown(&mut self) -> SendPinBoxFutureLifetime<'_, ()> {
+        Box::pin(async move {
+            self.clear_dial_info_details(None, None)
+                .set_network_class(None)
+                .clear_relay_node()
+                .commit(true)
+                .await;
+            self.routing_table
+                .inner
+                .write()
+                .unpublish_peer_info(RoutingDomain::LocalNetwork);
+        })
+    }
+}

veilid-core/src/routing_table/routing_table_inner/routing_domains/local_network/mod.rs

@@ -0,0 +1,203 @@
+mod editor;
+
+pub use editor::*;
+
+use super::*;
+
+/// Local Network routing domain internals
+#[derive(Debug)]
+pub struct LocalNetworkRoutingDomainDetail {
+    /// The local networks this domain will communicate with
+    local_networks: Vec<(IpAddr, IpAddr)>,
+    /// Common implementation for all routing domains
+    common: RoutingDomainDetailCommon,
+    /// Published peer info for this routing domain
+    published_peer_info: Mutex<Option<Arc<PeerInfo>>>,
+}
+
+impl Default for LocalNetworkRoutingDomainDetail {
+    fn default() -> Self {
+        Self {
+            local_networks: Default::default(),
+            common: RoutingDomainDetailCommon::new(RoutingDomain::LocalNetwork),
+            published_peer_info: Default::default(),
+        }
+    }
+}
+
+impl LocalNetworkRoutingDomainDetail {
+    #[expect(dead_code)]
+    pub fn local_networks(&self) -> Vec<(IpAddr, IpAddr)> {
+        self.local_networks.clone()
+    }
+
+    pub fn set_local_networks(&mut self, mut local_networks: Vec<(IpAddr, IpAddr)>) -> bool {
+        local_networks.sort();
+        if local_networks == self.local_networks {
+            return false;
+        }
+        self.local_networks = local_networks;
+        true
+    }
+}
+
+impl RoutingDomainDetailCommonAccessors for LocalNetworkRoutingDomainDetail {
+    fn common(&self) -> &RoutingDomainDetailCommon {
+        &self.common
+    }
+    fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon {
+        &mut self.common
+    }
+}
+
+impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail {
+    fn routing_domain(&self) -> RoutingDomain {
+        RoutingDomain::LocalNetwork
+    }
+
+    fn network_class(&self) -> Option<NetworkClass> {
+        self.common.network_class()
+    }
+    fn outbound_protocols(&self) -> ProtocolTypeSet {
+        self.common.outbound_protocols()
+    }
+    fn inbound_protocols(&self) -> ProtocolTypeSet {
+        self.common.inbound_protocols()
+    }
+    fn address_types(&self) -> AddressTypeSet {
+        self.common.address_types()
+    }
+    fn capabilities(&self) -> Vec<Capability> {
+        self.common.capabilities()
+    }
+    fn relay_node(&self) -> Option<FilteredNodeRef> {
+        self.common.relay_node()
+    }
+    fn relay_node_last_keepalive(&self) -> Option<Timestamp> {
+        self.common.relay_node_last_keepalive()
+    }
+    fn dial_info_details(&self) -> &Vec<DialInfoDetail> {
+        self.common.dial_info_details()
+    }
+    fn has_valid_network_class(&self) -> bool {
+        self.common.has_valid_network_class()
+    }
+
+    fn inbound_dial_info_filter(&self) -> DialInfoFilter {
+        self.common.inbound_dial_info_filter()
+    }
+    fn outbound_dial_info_filter(&self) -> DialInfoFilter {
+        self.common.outbound_dial_info_filter()
+    }
+
+    fn get_peer_info(&self, rti: &RoutingTableInner) -> Arc<PeerInfo> {
+        self.common.get_peer_info(rti)
+    }
+
+    fn get_published_peer_info(&self) -> Option<Arc<PeerInfo>> {
+        (*self.published_peer_info.lock()).clone()
+    }
+
+    fn can_contain_address(&self, address: Address) -> bool {
+        let ip = address.ip_addr();
+        for localnet in &self.local_networks {
+            if ipaddr_in_network(ip, localnet.0, localnet.1) {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn refresh(&self) {
+        self.common.clear_cache();
+    }
+
+    fn publish_peer_info(&self, rti: &RoutingTableInner) -> bool {
+        let pi = self.get_peer_info(rti);
+
+        // If the network class is not yet determined, don't publish
+        if pi.signed_node_info().node_info().network_class() == NetworkClass::Invalid {
+            log_rtab!(debug "[LocalNetwork] Not publishing peer info with invalid network class");
+            return false;
+        }
+
+        // If we need a relay and we don't have one, don't publish yet
+        if let Some(_relay_kind) = pi.signed_node_info().node_info().requires_relay() {
+            if pi.signed_node_info().relay_ids().is_empty() {
+                log_rtab!(debug "[LocalNetwork] Not publishing peer info that wants relay until we have a relay");
+                return false;
+            }
+        }
+
+        // Don't publish if the peer info hasnt changed from our previous publication
+        let mut ppi_lock = self.published_peer_info.lock();
+        if let Some(old_peer_info) = &*ppi_lock {
+            if pi.equivalent(old_peer_info) {
+                log_rtab!(debug "[LocalNetwork] Not publishing peer info because it is equivalent");
+                return false;
+            }
+        }
+
+        log_rtab!(debug "[LocalNetwork] Published new peer info: {:#?}", pi);
+        *ppi_lock = Some(pi);
+
+        true
+    }
+
+    fn unpublish_peer_info(&self) {
+        let mut ppi_lock = self.published_peer_info.lock();
+        log_rtab!(debug "[LocalNetwork] Unpublished peer info");
+        *ppi_lock = None;
+    }
+
+    fn ensure_dial_info_is_valid(&self, dial_info: &DialInfo) -> bool {
+        let address = dial_info.socket_address().address();
+        let can_contain_address = self.can_contain_address(address);
+
+        if !can_contain_address {
+            log_rtab!(debug "[LocalNetwork] can not add dial info to this routing domain: {:?}", dial_info);
+            return false;
+        }
+        if !dial_info.is_valid() {
+            log_rtab!(debug
+                "shouldn't be registering invalid addresses: {:?}",
+                dial_info
+            );
+            return false;
+        }
+        true
+    }
+
+    fn get_contact_method(
+        &self,
+        _rti: &RoutingTableInner,
+        peer_a: Arc<PeerInfo>,
+        peer_b: Arc<PeerInfo>,
+        dial_info_filter: DialInfoFilter,
+        sequencing: Sequencing,
+        dif_sort: Option<Arc<DialInfoDetailSort>>,
+    ) -> ContactMethod {
+        // Get the nodeinfos for convenience
+        let node_a = peer_a.signed_node_info().node_info();
+        let node_b = peer_b.signed_node_info().node_info();
+
+        // Get the node ids that would be used between these peers
+        let cck = common_crypto_kinds(&peer_a.node_ids().kinds(), &peer_b.node_ids().kinds());
+        let Some(_best_ck) = cck.first().copied() else {
+            // No common crypto kinds between these nodes, can't contact
+            return ContactMethod::Unreachable;
+        };
+
+        if let Some(target_did) = first_filtered_dial_info_detail_between_nodes(
+            node_a,
+            node_b,
+            &dial_info_filter,
+            sequencing,
+            dif_sort,
+        ) {
+            return ContactMethod::Direct(target_did.dial_info);
+        }
+
+        ContactMethod::Unreachable
+    }
+}

veilid-core/src/routing_table/routing_table_inner/routing_domains/mod.rs

@@ -0,0 +1,333 @@
+mod editor;
+mod local_network;
+mod public_internet;
+
+use super::*;
+
+pub use editor::*;
+pub use local_network::*;
+pub use public_internet::*;
+
+/// General trait for all routing domains
+pub(crate) trait RoutingDomainDetail {
+    // Common accessors
+    #[expect(dead_code)]
+    fn routing_domain(&self) -> RoutingDomain;
+    fn network_class(&self) -> Option<NetworkClass>;
+    fn outbound_protocols(&self) -> ProtocolTypeSet;
+    fn inbound_protocols(&self) -> ProtocolTypeSet;
+    fn address_types(&self) -> AddressTypeSet;
+    fn capabilities(&self) -> Vec<Capability>;
+    fn relay_node(&self) -> Option<FilteredNodeRef>;
+    fn relay_node_last_keepalive(&self) -> Option<Timestamp>;
+    fn dial_info_details(&self) -> &Vec<DialInfoDetail>;
+    fn has_valid_network_class(&self) -> bool;
+    fn get_published_peer_info(&self) -> Option<Arc<PeerInfo>>;
+    fn inbound_dial_info_filter(&self) -> DialInfoFilter;
+    fn outbound_dial_info_filter(&self) -> DialInfoFilter;
+    fn get_peer_info(&self, rti: &RoutingTableInner) -> Arc<PeerInfo>;
+
+    /// Can this routing domain contain a particular address
+    fn can_contain_address(&self, address: Address) -> bool;
+    fn ensure_dial_info_is_valid(&self, dial_info: &DialInfo) -> bool;
+
+    /// Refresh caches if external data changes
+    fn refresh(&self);
+
+    /// Publish current peer info to the world
+    fn publish_peer_info(&self, rti: &RoutingTableInner) -> bool;
+    fn unpublish_peer_info(&self);
+
+    /// Get the contact method required for node A to reach node B in this routing domain
+    /// Routing table must be locked for reading to use this function
+    fn get_contact_method(
+        &self,
+        rti: &RoutingTableInner,
+        peer_a: Arc<PeerInfo>,
+        peer_b: Arc<PeerInfo>,
+        dial_info_filter: DialInfoFilter,
+        sequencing: Sequencing,
+        dif_sort: Option<Arc<DialInfoDetailSort>>,
+    ) -> ContactMethod;
+}
+
+trait RoutingDomainDetailCommonAccessors: RoutingDomainDetail {
+    #[expect(dead_code)]
+    fn common(&self) -> &RoutingDomainDetailCommon;
+    fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon;
+}
+
+fn first_filtered_dial_info_detail_between_nodes(
+    from_node: &NodeInfo,
+    to_node: &NodeInfo,
+    dial_info_filter: &DialInfoFilter,
+    sequencing: Sequencing,
+    dif_sort: Option<Arc<DialInfoDetailSort>>,
+) -> Option<DialInfoDetail> {
+    // Consider outbound capabilities
+    let dial_info_filter = (*dial_info_filter).filtered(
+        &DialInfoFilter::all()
+            .with_address_type_set(from_node.address_types())
+            .with_protocol_type_set(from_node.outbound_protocols()),
+    );
+
+    // Apply sequencing and get sort
+    // Include sorting by external dial info sort for rotating through dialinfo
+    // based on an external preference table, for example the one kept by
+    // AddressFilter to deprioritize dialinfo that have recently failed to connect
+    let (ordered, dial_info_filter) = dial_info_filter.apply_sequencing(sequencing);
+    let sort: Option<Box<DialInfoDetailSort>> = if ordered {
+        if let Some(dif_sort) = dif_sort {
+            Some(Box::new(move |a, b| {
+                let mut ord = dif_sort(a, b);
+                if ord == core::cmp::Ordering::Equal {
+                    ord = DialInfoDetail::ordered_sequencing_sort(a, b);
+                }
+                ord
+            }))
+        } else {
+            Some(Box::new(move |a, b| {
+                DialInfoDetail::ordered_sequencing_sort(a, b)
+            }))
+        }
+    } else if let Some(dif_sort) = dif_sort {
+        Some(Box::new(move |a, b| dif_sort(a, b)))
+    } else {
+        None
+    };
+
+    // If the filter is dead then we won't be able to connect
+    if dial_info_filter.is_dead() {
+        return None;
+    }
+
+    // Get the best match dial info for node B if we have it
+    let direct_filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter);
+    to_node.first_filtered_dial_info_detail(sort, direct_filter)
+}
+
+#[derive(Debug)]
+struct RoutingDomainDetailCommon {
+    routing_domain: RoutingDomain,
+    network_class: Option<NetworkClass>,
+    outbound_protocols: ProtocolTypeSet,
+    inbound_protocols: ProtocolTypeSet,
+    address_types: AddressTypeSet,
+    relay_node: Option<NodeRef>,
+    relay_node_last_keepalive: Option<Timestamp>,
+    capabilities: Vec<Capability>,
+    dial_info_details: Vec<DialInfoDetail>,
+    // caches
+    cached_peer_info: Mutex<Option<Arc<PeerInfo>>>,
+}
+
+impl RoutingDomainDetailCommon {
+    pub fn new(routing_domain: RoutingDomain) -> Self {
+        Self {
+            routing_domain,
+            network_class: Default::default(),
+            outbound_protocols: Default::default(),
+            inbound_protocols: Default::default(),
+            address_types: Default::default(),
+            relay_node: Default::default(),
+            relay_node_last_keepalive: Default::default(),
+            capabilities: Default::default(),
+            dial_info_details: Default::default(),
+            cached_peer_info: Mutex::new(Default::default()),
+        }
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+    // Accessors
+
+    pub fn network_class(&self) -> Option<NetworkClass> {
+        self.network_class
+    }
+
+    pub fn outbound_protocols(&self) -> ProtocolTypeSet {
+        self.outbound_protocols
+    }
+
+    pub fn inbound_protocols(&self) -> ProtocolTypeSet {
+        self.inbound_protocols
+    }
+
+    pub fn address_types(&self) -> AddressTypeSet {
+        self.address_types
+    }
+
+    pub fn capabilities(&self) -> Vec<Capability> {
+        self.capabilities.clone()
+    }
+
+    pub fn relay_node(&self) -> Option<FilteredNodeRef> {
+        self.relay_node.as_ref().map(|nr| {
+            nr.custom_filtered(NodeRefFilter::new().with_routing_domain(self.routing_domain))
+        })
+    }
+
+    pub fn relay_node_last_keepalive(&self) -> Option<Timestamp> {
+        self.relay_node_last_keepalive
+    }
+
+    pub fn dial_info_details(&self) -> &Vec<DialInfoDetail> {
+        &self.dial_info_details
+    }
+
+    pub fn has_valid_network_class(&self) -> bool {
+        self.network_class.unwrap_or(NetworkClass::Invalid) != NetworkClass::Invalid
+    }
+
+    pub fn inbound_dial_info_filter(&self) -> DialInfoFilter {
+        DialInfoFilter::all()
+            .with_protocol_type_set(self.inbound_protocols)
+            .with_address_type_set(self.address_types)
+    }
+
+    pub fn outbound_dial_info_filter(&self) -> DialInfoFilter {
+        DialInfoFilter::all()
+            .with_protocol_type_set(self.outbound_protocols)
+            .with_address_type_set(self.address_types)
+    }
+
+    pub fn get_peer_info(&self, rti: &RoutingTableInner) -> Arc<PeerInfo> {
+        let mut cpi = self.cached_peer_info.lock();
+        if cpi.is_none() {
+            // Regenerate peer info
+            let pi = self.make_peer_info(rti);
+
+            // Cache the peer info
+            *cpi = Some(Arc::new(pi));
+        }
+        cpi.as_ref().unwrap().clone()
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+    // Mutators
+
+    fn setup_network(
+        &mut self,
+        outbound_protocols: ProtocolTypeSet,
+        inbound_protocols: ProtocolTypeSet,
+        address_types: AddressTypeSet,
+        capabilities: Vec<Capability>,
+    ) {
+        self.outbound_protocols = outbound_protocols;
+        self.inbound_protocols = inbound_protocols;
+        self.address_types = address_types;
+        self.capabilities = capabilities;
+        self.clear_cache();
+    }
+
+    fn set_network_class(&mut self, network_class: Option<NetworkClass>) {
+        self.network_class = network_class;
+        self.clear_cache();
+    }
+
+    fn set_relay_node(&mut self, opt_relay_node: Option<NodeRef>) {
+        self.relay_node = opt_relay_node;
+        self.relay_node_last_keepalive = None;
+        self.clear_cache();
+    }
+    fn set_relay_node_last_keepalive(&mut self, ts: Option<Timestamp>) {
+        self.relay_node_last_keepalive = ts;
+    }
+
+    fn clear_dial_info_details(
+        &mut self,
+        address_type: Option<AddressType>,
+        protocol_type: Option<ProtocolType>,
+    ) {
+        self.dial_info_details.retain_mut(|e| {
+            let mut remove = true;
+            if let Some(pt) = protocol_type {
+                if pt != e.dial_info.protocol_type() {
+                    remove = false;
+                }
+            }
+            if let Some(at) = address_type {
+                if at != e.dial_info.address_type() {
+                    remove = false;
+                }
+            }
+            !remove
+        });
+        self.clear_cache();
+    }
+    fn add_dial_info_detail(&mut self, did: DialInfoDetail) {
+        self.dial_info_details.push(did);
+        self.dial_info_details.sort();
+        self.dial_info_details.dedup();
+        self.clear_cache();
+    }
+    // fn remove_dial_info_detail(&mut self, did: DialInfoDetail) {
+    //     if let Some(index) = self.dial_info_details.iter().position(|x| *x == did) {
+    //         self.dial_info_details.remove(index);
+    //     }
+    //     self.clear_cache();
+    // }
+
+    //////////////////////////////////////////////////////////////////////////////
+    // Internal functions
+
+    fn make_peer_info(&self, rti: &RoutingTableInner) -> PeerInfo {
+        let node_info = NodeInfo::new(
+            self.network_class.unwrap_or(NetworkClass::Invalid),
+            self.outbound_protocols,
+            self.address_types,
+            VALID_ENVELOPE_VERSIONS.to_vec(),
+            VALID_CRYPTO_KINDS.to_vec(),
+            self.capabilities.clone(),
+            self.dial_info_details.clone(),
+        );
+
+        let relay_info = if let Some(rn) = &self.relay_node {
+            let opt_relay_pi = rn.locked(rti).make_peer_info(self.routing_domain);
+            if let Some(relay_pi) = opt_relay_pi {
+                let (_routing_domain, relay_ids, relay_sni) = relay_pi.destructure();
+                match relay_sni {
+                    SignedNodeInfo::Direct(d) => Some((relay_ids, d)),
+                    SignedNodeInfo::Relayed(_) => {
+                        warn!("relay node should not have a relay itself! if this happens, a relay updated its signed node info and became a relay, which should cause the relay to be dropped");
+                        None
+                    }
+                }
+            } else {
+                None
+            }
+        } else {
+            None
+        };
+
+        let signed_node_info = match relay_info {
+            Some((relay_ids, relay_sdni)) => SignedNodeInfo::Relayed(
+                SignedRelayedNodeInfo::make_signatures(
+                    rti.unlocked_inner.crypto(),
+                    rti.unlocked_inner.node_id_typed_key_pairs(),
+                    node_info,
+                    relay_ids,
+                    relay_sdni,
+                )
+                .unwrap(),
+            ),
+            None => SignedNodeInfo::Direct(
+                SignedDirectNodeInfo::make_signatures(
+                    rti.unlocked_inner.crypto(),
+                    rti.unlocked_inner.node_id_typed_key_pairs(),
+                    node_info,
+                )
+                .unwrap(),
+            ),
+        };
+
+        PeerInfo::new(
+            self.routing_domain,
+            rti.unlocked_inner.node_ids(),
+            signed_node_info,
+        )
+    }
+
+    fn clear_cache(&self) {
+        *self.cached_peer_info.lock() = None;
+    }
+}

veilid-core/src/routing_table/routing_table_inner/routing_domains/public_internet/editor.rs

@@ -0,0 +1,266 @@
+use super::*;
+
+#[derive(Debug)]
+enum RoutingDomainChangePublicInternet {
+    Common(RoutingDomainChangeCommon),
+}
+
+pub struct RoutingDomainEditorPublicInternet {
+    routing_table: RoutingTable,
+    changes: Vec<RoutingDomainChangePublicInternet>,
+}
+
+impl RoutingDomainEditorPublicInternet {
+    pub(in crate::routing_table) fn new(routing_table: RoutingTable) -> Self {
+        Self {
+            routing_table,
+            changes: Vec::new(),
+        }
+    }
+}
+
+impl RoutingDomainEditorCommonTrait for RoutingDomainEditorPublicInternet {
+    #[instrument(level = "debug", skip(self))]
+    fn clear_dial_info_details(
+        &mut self,
+        address_type: Option<AddressType>,
+        protocol_type: Option<ProtocolType>,
+    ) -> &mut Self {
+        self.changes.push(RoutingDomainChangePublicInternet::Common(
+            RoutingDomainChangeCommon::ClearDialInfoDetails {
+                address_type,
+                protocol_type,
+            },
+        ));
+
+        self
+    }
+    #[instrument(level = "debug", skip(self))]
+    fn clear_relay_node(&mut self) -> &mut Self {
+        self.changes.push(RoutingDomainChangePublicInternet::Common(
+            RoutingDomainChangeCommon::ClearRelayNode,
+        ));
+        self
+    }
+    #[instrument(level = "debug", skip(self))]
+    fn set_relay_node(&mut self, relay_node: NodeRef) -> &mut Self {
+        self.changes.push(RoutingDomainChangePublicInternet::Common(
+            RoutingDomainChangeCommon::SetRelayNode { relay_node },
+        ));
+        self
+    }
+    #[instrument(level = "debug", skip(self))]
+    fn set_relay_node_keepalive(&mut self, ts: Option<Timestamp>) -> &mut Self {
+        self.changes.push(RoutingDomainChangePublicInternet::Common(
+            RoutingDomainChangeCommon::SetRelayNodeKeepalive { ts },
+        ));
+        self
+    }
+    #[instrument(level = "debug", skip(self))]
+    fn add_dial_info(&mut self, dial_info: DialInfo, class: DialInfoClass) -> &mut Self {
+        self.changes.push(RoutingDomainChangePublicInternet::Common(
+            RoutingDomainChangeCommon::AddDialInfo {
+                dial_info_detail: DialInfoDetail {
+                    dial_info: dial_info.clone(),
+                    class,
+                },
+            },
+        ));
+        self
+    }
+    // #[instrument(level = "debug", skip_all)]
+    // fn retain_dial_info<F: Fn(&DialInfo, DialInfoClass) -> bool>(
+    //     &mut self,
+    //     closure: F,
+    // ) -> EyreResult<&mut Self> {
+    //     let dids = self.routing_table.dial_info_details(self.routing_domain);
+    //     for did in dids {
+    //         if !closure(&did.dial_info, did.class) {
+    //             self.changes
+    //                 .push(RoutingDomainChangePublicInternet::Common(RoutingDomainChange::RemoveDialInfoDetail {
+    //                     dial_info_detail: did,
+    //                 }));
+    //         }
+    //     }
+
+    //     Ok(self)
+    // }
+
+    #[instrument(level = "debug", skip(self))]
+    fn setup_network(
+        &mut self,
+        outbound_protocols: ProtocolTypeSet,
+        inbound_protocols: ProtocolTypeSet,
+        address_types: AddressTypeSet,
+        capabilities: Vec<Capability>,
+    ) -> &mut Self {
+        self.changes.push(RoutingDomainChangePublicInternet::Common(
+            RoutingDomainChangeCommon::SetupNetwork {
+                outbound_protocols,
+                inbound_protocols,
+                address_types,
+                capabilities,
+            },
+        ));
+        self
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn set_network_class(&mut self, network_class: Option<NetworkClass>) -> &mut Self {
+        self.changes.push(RoutingDomainChangePublicInternet::Common(
+            RoutingDomainChangeCommon::SetNetworkClass { network_class },
+        ));
+        self
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn commit(&mut self, pause_tasks: bool) -> SendPinBoxFutureLifetime<'_, bool> {
+        Box::pin(async move {
+            // No locking if we have nothing to do
+            if self.changes.is_empty() {
+                return false;
+            }
+            // Briefly pause routing table ticker while changes are made
+            let _tick_guard = if pause_tasks {
+                Some(self.routing_table.pause_tasks().await)
+            } else {
+                None
+            };
+
+            // Apply changes
+            let mut peer_info_changed = false;
+
+            let mut rti_lock = self.routing_table.inner.write();
+            let rti = &mut rti_lock;
+            rti.with_public_internet_routing_domain_mut(|detail| {
+                let old_dial_info_details = detail.dial_info_details().clone();
+                let old_relay_node = detail.relay_node();
+                let old_outbound_protocols = detail.outbound_protocols();
+                let old_inbound_protocols = detail.inbound_protocols();
+                let old_address_types = detail.address_types();
+                let old_capabilities = detail.capabilities();
+                let old_network_class = detail.network_class();
+
+                for change in self.changes.drain(..) {
+                    match change {
+                        RoutingDomainChangePublicInternet::Common(common_change) => {
+                            detail.apply_common_change(common_change);
+                        }
+                    }
+                }
+
+                let new_dial_info_details = detail.dial_info_details().clone();
+                let new_relay_node = detail.relay_node();
+                let new_outbound_protocols = detail.outbound_protocols();
+                let new_inbound_protocols = detail.inbound_protocols();
+                let new_address_types = detail.address_types();
+                let new_capabilities = detail.capabilities();
+                let new_network_class = detail.network_class();
+
+                // Compare and see if peerinfo needs republication
+                let removed_dial_info = old_dial_info_details
+                    .iter()
+                    .filter(|di| !new_dial_info_details.contains(di))
+                    .collect::<Vec<_>>();
+                if !removed_dial_info.is_empty() {
+                    info!(
+                        "[PublicInternet] removed dial info: {:#?}",
+                        removed_dial_info
+                    );
+                    peer_info_changed = true;
+                }
+                let added_dial_info = new_dial_info_details
+                    .iter()
+                    .filter(|di| !old_dial_info_details.contains(di))
+                    .collect::<Vec<_>>();
+                if !added_dial_info.is_empty() {
+                    info!("[PublicInternet] added dial info: {:#?}", added_dial_info);
+                    peer_info_changed = true;
+                }
+                if let Some(nrn) = new_relay_node {
+                    if let Some(orn) = old_relay_node {
+                        if !nrn.same_entry(&orn) {
+                            info!("[PublicInternet] change relay: {} -> {}", orn, nrn);
+                            peer_info_changed = true;
+                        }
+                    } else {
+                        info!("[PublicInternet] set relay: {}", nrn);
+                        peer_info_changed = true;
+                    }
+                }
+                if old_outbound_protocols != new_outbound_protocols {
+                    info!(
+                        "[PublicInternet] changed network: outbound {:?}->{:?}\n",
+                        old_outbound_protocols, new_outbound_protocols
+                    );
+                    peer_info_changed = true;
+                }
+                if old_inbound_protocols != new_inbound_protocols {
+                    info!(
+                        "[PublicInternet] changed network: inbound {:?}->{:?}\n",
+                        old_inbound_protocols, new_inbound_protocols,
+                    );
+                    peer_info_changed = true;
+                }
+                if old_address_types != new_address_types {
+                    info!(
+                        "[PublicInternet] changed network: address types {:?}->{:?}\n",
+                        old_address_types, new_address_types,
+                    );
+                    peer_info_changed = true;
+                }
+                if old_capabilities != new_capabilities {
+                    info!(
+                        "[PublicInternet] changed network: capabilities {:?}->{:?}\n",
+                        old_capabilities, new_capabilities
+                    );
+                    peer_info_changed = true;
+                }
+                if old_network_class != new_network_class {
+                    info!(
+                        "[PublicInternet] changed network class: {:?}->{:?}\n",
+                        old_network_class, new_network_class
+                    );
+                    peer_info_changed = true;
+                }
+            });
+
+            if peer_info_changed {
+                // Allow signed node info updates at same timestamp for otherwise dead nodes if our network has changed
+                rti.reset_all_updated_since_last_network_change();
+            }
+
+            peer_info_changed
+        })
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn publish(&mut self) {
+        let changed = self
+            .routing_table
+            .inner
+            .write()
+            .publish_peer_info(RoutingDomain::PublicInternet);
+
+        // Clear the routespecstore cache if our PublicInternet dial info has changed
+        if changed {
+            let rss = self.routing_table.route_spec_store();
+            rss.reset();
+        }
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn shutdown(&mut self) -> SendPinBoxFutureLifetime<'_, ()> {
+        Box::pin(async move {
+            self.clear_dial_info_details(None, None)
+                .set_network_class(None)
+                .clear_relay_node()
+                .commit(true)
+                .await;
+            self.routing_table
+                .inner
+                .write()
+                .unpublish_peer_info(RoutingDomain::PublicInternet);
+        })
+    }
+}

veilid-core/src/routing_table/routing_table_inner/routing_domains/public_internet/mod.rs

@@ -0,0 +1,369 @@
+mod editor;
+
+pub use editor::*;
+
+use super::*;
+
+/// Public Internet routing domain internals
+#[derive(Debug)]
+pub struct PublicInternetRoutingDomainDetail {
+    /// Common implementation for all routing domains
+    common: RoutingDomainDetailCommon,
+    /// Published peer info for this routing domain
+    published_peer_info: Mutex<Option<Arc<PeerInfo>>>,
+}
+
+impl Default for PublicInternetRoutingDomainDetail {
+    fn default() -> Self {
+        Self {
+            common: RoutingDomainDetailCommon::new(RoutingDomain::PublicInternet),
+            published_peer_info: Default::default(),
+        }
+    }
+}
+
+impl RoutingDomainDetailCommonAccessors for PublicInternetRoutingDomainDetail {
+    fn common(&self) -> &RoutingDomainDetailCommon {
+        &self.common
+    }
+    fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon {
+        &mut self.common
+    }
+}
+
+impl RoutingDomainDetail for PublicInternetRoutingDomainDetail {
+    fn routing_domain(&self) -> RoutingDomain {
+        RoutingDomain::PublicInternet
+    }
+
+    fn network_class(&self) -> Option<NetworkClass> {
+        self.common.network_class()
+    }
+    fn outbound_protocols(&self) -> ProtocolTypeSet {
+        self.common.outbound_protocols()
+    }
+    fn inbound_protocols(&self) -> ProtocolTypeSet {
+        self.common.inbound_protocols()
+    }
+    fn address_types(&self) -> AddressTypeSet {
+        self.common.address_types()
+    }
+    fn capabilities(&self) -> Vec<Capability> {
+        self.common.capabilities()
+    }
+    fn relay_node(&self) -> Option<FilteredNodeRef> {
+        self.common.relay_node()
+    }
+    fn relay_node_last_keepalive(&self) -> Option<Timestamp> {
+        self.common.relay_node_last_keepalive()
+    }
+    fn dial_info_details(&self) -> &Vec<DialInfoDetail> {
+        self.common.dial_info_details()
+    }
+    fn has_valid_network_class(&self) -> bool {
+        self.common.has_valid_network_class()
+    }
+
+    fn inbound_dial_info_filter(&self) -> DialInfoFilter {
+        self.common.inbound_dial_info_filter()
+    }
+    fn outbound_dial_info_filter(&self) -> DialInfoFilter {
+        self.common.outbound_dial_info_filter()
+    }
+
+    fn get_peer_info(&self, rti: &RoutingTableInner) -> Arc<PeerInfo> {
+        self.common.get_peer_info(rti)
+    }
+
+    fn get_published_peer_info(&self) -> Option<Arc<PeerInfo>> {
+        (*self.published_peer_info.lock()).clone()
+    }
+
+    ////////////////////////////////////////////////
+
+    fn can_contain_address(&self, address: Address) -> bool {
+        address.is_global()
+    }
+
+    fn refresh(&self) {
+        self.common.clear_cache();
+    }
+
+    fn publish_peer_info(&self, rti: &RoutingTableInner) -> bool {
+        let pi = self.get_peer_info(rti);
+
+        // If the network class is not yet determined, don't publish
+        if pi.signed_node_info().node_info().network_class() == NetworkClass::Invalid {
+            log_rtab!(debug "[PublicInternet] Not publishing peer info with invalid network class");
+            return false;
+        }
+
+        // If we need a relay and we don't have one, don't publish yet
+        if let Some(_relay_kind) = pi.signed_node_info().node_info().requires_relay() {
+            if pi.signed_node_info().relay_ids().is_empty() {
+                log_rtab!(debug "[PublicInternet] Not publishing peer info that wants relay until we have a relay");
+                return false;
+            }
+        }
+
+        // Don't publish if the peer info hasnt changed from our previous publication
+        let mut ppi_lock = self.published_peer_info.lock();
+        if let Some(old_peer_info) = &*ppi_lock {
+            if pi.equivalent(old_peer_info) {
+                log_rtab!(debug "[PublicInternet] Not publishing peer info because it is equivalent");
+                return false;
+            }
+        }
+
+        log_rtab!(debug "[PublicInternet] Published new peer info: {:#?}", pi);
+        *ppi_lock = Some(pi);
+
+        true
+    }
+
+    fn unpublish_peer_info(&self) {
+        let mut ppi_lock = self.published_peer_info.lock();
+        log_rtab!(debug "[PublicInternet] Unpublished peer info");
+        *ppi_lock = None;
+    }
+
+    fn ensure_dial_info_is_valid(&self, dial_info: &DialInfo) -> bool {
+        let address = dial_info.socket_address().address();
+        let can_contain_address = self.can_contain_address(address);
+
+        if !can_contain_address {
+            log_rtab!(debug "[PublicInternet] can not add dial info to this routing domain: {:?}", dial_info);
+            return false;
+        }
+        if !dial_info.is_valid() {
+            log_rtab!(debug
+                "shouldn't be registering invalid addresses: {:?}",
+                dial_info
+            );
+            return false;
+        }
+        true
+    }
+
+    fn get_contact_method(
+        &self,
+        rti: &RoutingTableInner,
+        peer_a: Arc<PeerInfo>,
+        peer_b: Arc<PeerInfo>,
+        dial_info_filter: DialInfoFilter,
+        sequencing: Sequencing,
+        dif_sort: Option<Arc<DialInfoDetailSort>>,
+    ) -> ContactMethod {
+        let ip6_prefix_size = rti
+            .unlocked_inner
+            .config
+            .get()
+            .network
+            .max_connections_per_ip6_prefix_size as usize;
+
+        // Get the nodeinfos for convenience
+        let node_a = peer_a.signed_node_info().node_info();
+        let node_b = peer_b.signed_node_info().node_info();
+
+        // Check to see if these nodes are on the same network
+        let same_ipblock = node_a.node_is_on_same_ipblock(node_b, ip6_prefix_size);
+
+        // Get the node ids that would be used between these peers
+        let cck = common_crypto_kinds(&peer_a.node_ids().kinds(), &peer_b.node_ids().kinds());
+        let Some(best_ck) = cck.first().copied() else {
+            // No common crypto kinds between these nodes, can't contact
+            return ContactMethod::Unreachable;
+        };
+
+        //let node_a_id = peer_a.node_ids().get(best_ck).unwrap();
+        let node_b_id = peer_b.node_ids().get(best_ck).unwrap();
+
+        // Get the best match dial info for node B if we have it
+        // Don't try direct inbound at all if the two nodes are on the same ipblock to avoid hairpin NAT issues
+        // as well avoiding direct traffic between same-network nodes. This would be done in the LocalNetwork RoutingDomain.
+        if let Some(target_did) = (!same_ipblock)
+            .then(|| {
+                first_filtered_dial_info_detail_between_nodes(
+                    node_a,
+                    node_b,
+                    &dial_info_filter,
+                    sequencing,
+                    dif_sort.clone(),
+                )
+            })
+            .flatten()
+        {
+            // Do we need to signal before going inbound?
+            if !target_did.class.requires_signal() {
+                // Go direct without signaling
+                return ContactMethod::Direct(target_did.dial_info);
+            }
+
+            // Get the target's inbound relay, it must have one or it is not reachable
+            if let Some(node_b_relay) = peer_b.signed_node_info().relay_info() {
+                // Note that relay_peer_info could be node_a, in which case a connection already exists
+                // and we only get here if the connection had dropped, in which case node_a is unreachable until
+                // it gets a new relay connection up
+                if peer_b
+                    .signed_node_info()
+                    .relay_ids()
+                    .contains_any(peer_a.node_ids())
+                {
+                    return ContactMethod::Existing;
+                }
+
+                // Get best node id to contact relay with
+                let Some(node_b_relay_id) = peer_b.signed_node_info().relay_ids().get(best_ck)
+                else {
+                    // No best relay id
+                    return ContactMethod::Unreachable;
+                };
+
+                // Can node A reach the inbound relay directly?
+                if first_filtered_dial_info_detail_between_nodes(
+                    node_a,
+                    node_b_relay,
+                    &dial_info_filter,
+                    sequencing,
+                    dif_sort.clone(),
+                )
+                .is_some()
+                {
+                    // Can node A receive anything inbound ever?
+                    if matches!(node_a.network_class(), NetworkClass::InboundCapable) {
+                        ///////// Reverse connection
+
+                        // Get the best match dial info for an reverse inbound connection from node B to node A
+                        if let Some(reverse_did) = first_filtered_dial_info_detail_between_nodes(
+                            node_b,
+                            node_a,
+                            &dial_info_filter,
+                            sequencing,
+                            dif_sort.clone(),
+                        ) {
+                            // Ensure we aren't on the same public IP address (no hairpin nat)
+                            if reverse_did.dial_info.ip_addr() != target_did.dial_info.ip_addr() {
+                                // Can we receive a direct reverse connection?
+                                if !reverse_did.class.requires_signal() {
+                                    return ContactMethod::SignalReverse(
+                                        node_b_relay_id,
+                                        node_b_id,
+                                    );
+                                }
+                            }
+                        }
+
+                        ///////// UDP hole-punch
+
+                        // Does node B have a direct udp dialinfo node A can reach?
+                        let udp_dial_info_filter = dial_info_filter
+                            .filtered(&DialInfoFilter::all().with_protocol_type(ProtocolType::UDP));
+                        if let Some(target_udp_did) = first_filtered_dial_info_detail_between_nodes(
+                            node_a,
+                            node_b,
+                            &udp_dial_info_filter,
+                            sequencing,
+                            dif_sort.clone(),
+                        ) {
+                            // Does node A have a direct udp dialinfo that node B can reach?
+                            if let Some(reverse_udp_did) =
+                                first_filtered_dial_info_detail_between_nodes(
+                                    node_b,
+                                    node_a,
+                                    &udp_dial_info_filter,
+                                    sequencing,
+                                    dif_sort.clone(),
+                                )
+                            {
+                                // Ensure we aren't on the same public IP address (no hairpin nat)
+                                if reverse_udp_did.dial_info.ip_addr()
+                                    != target_udp_did.dial_info.ip_addr()
+                                {
+                                    // The target and ourselves have a udp dialinfo that they can reach
+                                    return ContactMethod::SignalHolePunch(
+                                        node_b_relay_id,
+                                        node_b_id,
+                                    );
+                                }
+                            }
+                        }
+                        // Otherwise we have to inbound relay
+                    }
+
+                    return ContactMethod::InboundRelay(node_b_relay_id);
+                }
+            }
+        }
+        // If the node B has no direct dial info or is on the same ipblock, it needs to have an inbound relay
+        else if let Some(node_b_relay) = peer_b.signed_node_info().relay_info() {
+            // Note that relay_peer_info could be node_a, in which case a connection already exists
+            // and we only get here if the connection had dropped, in which case node_b is unreachable until
+            // it gets a new relay connection up
+            if peer_b
+                .signed_node_info()
+                .relay_ids()
+                .contains_any(peer_a.node_ids())
+            {
+                return ContactMethod::Existing;
+            }
+
+            // Get best node id to contact relay with
+            let Some(node_b_relay_id) = peer_b.signed_node_info().relay_ids().get(best_ck) else {
+                // No best relay id
+                return ContactMethod::Unreachable;
+            };
+
+            // Can we reach the inbound relay?
+            if first_filtered_dial_info_detail_between_nodes(
+                node_a,
+                node_b_relay,
+                &dial_info_filter,
+                sequencing,
+                dif_sort.clone(),
+            )
+            .is_some()
+            {
+                ///////// Reverse connection
+
+                // Get the best match dial info for an reverse inbound connection from node B to node A
+                // unless both nodes are on the same ipblock
+                if let Some(reverse_did) = (!same_ipblock)
+                    .then(|| {
+                        first_filtered_dial_info_detail_between_nodes(
+                            node_b,
+                            node_a,
+                            &dial_info_filter,
+                            sequencing,
+                            dif_sort.clone(),
+                        )
+                    })
+                    .flatten()
+                {
+                    // Can we receive a direct reverse connection?
+                    if !reverse_did.class.requires_signal() {
+                        return ContactMethod::SignalReverse(node_b_relay_id, node_b_id);
+                    }
+                }
+
+                return ContactMethod::InboundRelay(node_b_relay_id);
+            }
+        }
+
+        // If node A can't reach the node by other means, it may need to use its outbound relay
+        if peer_a
+            .signed_node_info()
+            .node_info()
+            .network_class()
+            .outbound_wants_relay()
+        {
+            if let Some(node_a_relay_id) = peer_a.signed_node_info().relay_ids().get(best_ck) {
+                // Ensure it's not our relay we're trying to reach
+                if node_a_relay_id != node_b_id {
+                    return ContactMethod::OutboundRelay(node_a_relay_id);
+                }
+            }
+        }
+
+        ContactMethod::Unreachable
+    }
+}

veilid-core/src/routing_table/tasks/bootstrap.rs

@@ -257,7 +257,7 @@ impl RoutingTable {
     pub(crate) fn bootstrap_with_peer(
         self,
         crypto_kinds: Vec<CryptoKind>,
-        pi: PeerInfo,
+        pi: Arc<PeerInfo>,
         unord: &FuturesUnordered<SendPinBoxFuture<()>>,
     ) {
         log_rtab!(
@@ -266,7 +266,9 @@ impl RoutingTable {
             pi.signed_node_info().node_info().dial_info_detail_list()
         );
 
-        let nr = match self.register_node_with_peer_info(RoutingDomain::PublicInternet, pi, true) {
+        let routing_domain = pi.routing_domain();
+
+        let nr = match self.register_node_with_peer_info(pi, true) {
             Ok(nr) => nr,
             Err(e) => {
                 log_rtab!(error "failed to register bootstrap peer info: {}", e);
@@ -277,14 +279,14 @@ impl RoutingTable {
         // Add this our futures to process in parallel
         for crypto_kind in crypto_kinds {
             // Bootstrap this crypto kind
-            let nr = nr.clone();
+            let nr = nr.unfiltered();
             let routing_table = self.clone();
             unord.push(Box::pin(
                 async move {
                     // Get what contact method would be used for contacting the bootstrap
                     let bsdi = match routing_table
                         .network_manager()
-                        .get_node_contact_method(nr.clone())
+                        .get_node_contact_method(nr.default_filtered())
                     {
                         Ok(NodeContactMethod::Direct(v)) => v,
                         Ok(v) => {
@@ -302,10 +304,10 @@ impl RoutingTable {
 
                     // Need VALID signed peer info, so ask bootstrap to find_node of itself
                     // which will ensure it has the bootstrap's signed peer info as part of the response
-                    let _ = routing_table.find_target(crypto_kind, nr.clone(), vec![]).await;
+                    let _ = routing_table.find_nodes_close_to_node_ref(crypto_kind, nr.clone(), vec![]).await;
 
                     // Ensure we got the signed peer info
-                    if !nr.signed_node_info_has_valid_signature(RoutingDomain::PublicInternet) {
+                    if !nr.signed_node_info_has_valid_signature(routing_domain) {
                         log_rtab!(warn "bootstrap server is not responding");
                         log_rtab!(debug "bootstrap server is not responding for dialinfo: {}", bsdi);
 
@@ -324,7 +326,7 @@ impl RoutingTable {
     #[instrument(level = "trace", skip(self), err)]
     pub(crate) async fn bootstrap_with_peer_list(
         self,
-        peers: Vec<PeerInfo>,
+        peers: Vec<Arc<PeerInfo>>,
         stop_token: StopToken,
     ) -> EyreResult<()> {
         log_rtab!(debug "  bootstrapped peers: {:?}", &peers);
@@ -389,7 +391,7 @@ impl RoutingTable {
             // Direct bootstrap
             let network_manager = self.network_manager();
 
-            let mut peer_map = HashMap::<TypedKeyGroup, PeerInfo>::new();
+            let mut peer_map = HashMap::<TypedKeyGroup, Arc<PeerInfo>>::new();
             for bootstrap_di in bootstrap_dialinfos {
                 log_rtab!(debug "direct bootstrap with: {}", bootstrap_di);
                 let peers = network_manager.boot_request(bootstrap_di).await?;
@@ -403,7 +405,7 @@ impl RoutingTable {
         } else {
             // If not direct, resolve bootstrap servers and recurse their TXT entries
             let bsrecs = self.resolve_bootstrap(bootstrap).await?;
-            let peers: Vec<PeerInfo> = bsrecs
+            let peers: Vec<Arc<PeerInfo>> = bsrecs
                 .into_iter()
                 .map(|bsrec| {
                     // Get crypto support from list of node ids
@@ -413,8 +415,8 @@ impl RoutingTable {
                     let sni = SignedNodeInfo::Direct(SignedDirectNodeInfo::with_no_signature(
                         NodeInfo::new(
                             NetworkClass::InboundCapable, // Bootstraps are always inbound capable
-                            ProtocolTypeSet::only(ProtocolType::UDP), // Bootstraps do not participate in relaying and will not make outbound requests, but will have UDP enabled
+                            ProtocolTypeSet::all(), // Bootstraps are always capable of all protocols
-                            AddressTypeSet::all(), // Bootstraps are always IPV4 and IPV6 capable
+                            AddressTypeSet::all(),  // Bootstraps are always IPV4 and IPV6 capable
                             bsrec.envelope_support, // Envelope support is as specified in the bootstrap list
                             crypto_support, // Crypto support is derived from list of node ids
                             vec![],         // Bootstrap needs no capabilities
@@ -422,7 +424,11 @@ impl RoutingTable {
                         ),
                     ));
 
-                    PeerInfo::new(bsrec.node_ids, sni)
+                    Arc::new(PeerInfo::new(
+                        RoutingDomain::PublicInternet,
+                        bsrec.node_ids,
+                        sni,
+                    ))
                 })
                 .collect();
 

veilid-core/src/routing_table/tasks/closest_peers_refresh.rs

@@ -56,7 +56,7 @@ impl RoutingTable {
                     self_node_id,
                     filters,
                     |_rti, entry: Option<Arc<BucketEntry>>| {
-                        NodeRef::new(routing_table.clone(), entry.unwrap().clone(), None)
+                        NodeRef::new(routing_table.clone(), entry.unwrap().clone())
                     },
                 )
                 .unwrap();

veilid-core/src/routing_table/tasks/mod.rs

@@ -185,7 +185,13 @@ impl RoutingTable {
                 .closest_peers_refresh_task
                 .tick()
                 .await?;
+        }
 
+        // Only perform these operations if we already have a published peer info
+        if self
+            .get_published_peer_info(RoutingDomain::PublicInternet)
+            .is_some()
+        {
             // Run the private route management task
             self.unlocked_inner
                 .private_route_management_task

veilid-core/src/routing_table/tasks/peer_minimum_refresh.rs

@@ -71,7 +71,7 @@ impl RoutingTable {
                 min_peer_count,
                 filters,
                 |_rti, entry: Option<Arc<BucketEntry>>| {
-                    NodeRef::new(routing_table.clone(), entry.unwrap().clone(), None)
+                    NodeRef::new(routing_table.clone(), entry.unwrap().clone())
                 },
             );
 

veilid-core/src/routing_table/tasks/ping_validator.rs

@@ -23,9 +23,13 @@ impl RoutingTable {
     async fn relay_keepalive_public_internet(
         &self,
         cur_ts: Timestamp,
-        relay_nr: NodeRef,
         futurequeue: &mut VecDeque<PingValidatorFuture>,
     ) -> EyreResult<()> {
+        // Get the PublicInternet relay if we are using one
+        let Some(relay_nr) = self.relay_node(RoutingDomain::PublicInternet) else {
+            return Ok(());
+        };
+
         let rpc = self.rpc_processor();
         // Get our publicinternet dial info
         let dids = self.all_filtered_dial_info_details(
@@ -45,7 +49,7 @@ impl RoutingTable {
             return Ok(());
         }
         // Say we're doing this keepalive now
-        self.edit_routing_domain(RoutingDomain::PublicInternet)
+        self.edit_public_internet_routing_domain()
             .set_relay_node_keepalive(Some(cur_ts))
             .commit(false)
             .await;
@@ -88,17 +92,14 @@ impl RoutingTable {
                     got_unordered = true;
                 }
                 let dif = did.dial_info.make_filter();
-                let relay_nr_filtered =
+
-                    relay_nr.filtered_clone(NodeRefFilter::new().with_dial_info_filter(dif));
+                relay_noderefs
-                relay_noderefs.push(relay_nr_filtered);
+                    .push(relay_nr.filtered_clone(NodeRefFilter::new().with_dial_info_filter(dif)));
             }
         }
         // Add noderef filters for ordered or unordered sequencing if we havent already seen those
         if !got_ordered {
-            let (_, nrf) = NodeRefFilter::new().with_sequencing(Sequencing::EnsureOrdered);
+            relay_noderefs.push(relay_nr.sequencing_clone(Sequencing::EnsureOrdered));
-            let mut relay_nr_filtered = relay_nr.filtered_clone(nrf);
-            relay_nr_filtered.set_sequencing(Sequencing::EnsureOrdered);
-            relay_noderefs.push(relay_nr_filtered);
         }
         if !got_unordered {
             relay_noderefs.push(relay_nr);
@@ -158,7 +159,11 @@ impl RoutingTable {
             log_rtab!("--> Watch ping to {:?}", watch_nr);
 
             futurequeue.push_back(
-                async move { rpc.rpc_call_status(Destination::direct(watch_nr)).await }.boxed(),
+                async move {
+                    rpc.rpc_call_status(Destination::direct(watch_nr.default_filtered()))
+                        .await
+                }
+                .boxed(),
             );
         }
         Ok(())
@@ -177,14 +182,9 @@ impl RoutingTable {
         // Get all nodes needing pings in the PublicInternet routing domain
         let node_refs = self.get_nodes_needing_ping(RoutingDomain::PublicInternet, cur_ts);
 
-        // Get the PublicInternet relay if we are using one
+        // If we have a relay, let's ping for NAT keepalives
-        let opt_relay_nr = self.relay_node(RoutingDomain::PublicInternet);
+        self.relay_keepalive_public_internet(cur_ts, futurequeue)
-
+            .await?;
-        // If this is our relay, let's check for NAT keepalives
-        if let Some(relay_nr) = opt_relay_nr {
-            self.relay_keepalive_public_internet(cur_ts, relay_nr, futurequeue)
-                .await?;
-        }
 
         // Check active watch keepalives
         self.active_watches_keepalive_public_internet(cur_ts, futurequeue)

veilid-core/src/routing_table/tasks/private_route_management.rs

@@ -117,7 +117,6 @@ impl RoutingTable {
         let rss = self.route_spec_store();
         #[derive(Default, Debug)]
         struct TestRouteContext {
-            failed: bool,
             dead_routes: Vec<RouteId>,
         }
 
@@ -130,17 +129,13 @@ impl RoutingTable {
                 unord.push(
                     async move {
                         let success = match rss.test_route(r).await {
-                            Ok(v) => v,
+                            // Test had result
-                            // Route was already removed
+                            Ok(Some(v)) => v,
-                            Err(VeilidAPIError::InvalidArgument {
+                            // Test could not be performed at this time
-                                context: _,
+                            Ok(None) => true,
-                                argument: _,
+                            // Test failure
-                                value: _,
-                            }) => false,
-                            // Other failures
                             Err(e) => {
                                 log_rtab!(error "Test route failed: {}", e);
-                                ctx.lock().failed = true;
                                 return;
                             }
                         };

veilid-core/src/routing_table/tasks/relay_management.rs

@@ -4,7 +4,7 @@ impl RoutingTable {
     // Check if a relay is desired or not
     #[instrument(level = "trace", skip_all)]
     fn public_internet_wants_relay(&self) -> Option<RelayKind> {
-        let own_peer_info = self.get_own_peer_info(RoutingDomain::PublicInternet);
+        let own_peer_info = self.get_current_peer_info(RoutingDomain::PublicInternet);
         let own_node_info = own_peer_info.signed_node_info().node_info();
         let network_class = own_node_info.network_class();
 
@@ -34,7 +34,7 @@ impl RoutingTable {
         for did in own_node_info.dial_info_detail_list() {
             inbound_addresses.insert(did.dial_info.to_socket_addr());
         }
-        let own_local_peer_info = self.get_own_peer_info(RoutingDomain::LocalNetwork);
+        let own_local_peer_info = self.get_current_peer_info(RoutingDomain::LocalNetwork);
         let own_local_node_info = own_local_peer_info.signed_node_info().node_info();
         for ldid in own_local_node_info.dial_info_detail_list() {
             inbound_addresses.remove(&ldid.dial_info.to_socket_addr());
@@ -59,7 +59,7 @@ impl RoutingTable {
         let relay_desired = self.public_internet_wants_relay();
 
         // Get routing domain editor
-        let mut editor = self.edit_routing_domain(RoutingDomain::PublicInternet);
+        let mut editor = self.edit_public_internet_routing_domain();
 
         // If we already have a relay, see if it is dead, or if we don't need it any more
         let has_relay = {
@@ -107,16 +107,14 @@ impl RoutingTable {
             let mut got_outbound_relay = false;
             if matches!(relay_desired, RelayKind::Outbound) {
                 // The outbound relay is the host of the PWA
-                if let Some(outbound_relay_peerinfo) = intf::get_outbound_relay_peer().await {
+                if let Some(outbound_relay_peerinfo) =
+                    intf::get_outbound_relay_peer(RoutingDomain::PublicInternet).await
+                {
                     // Register new outbound relay
-                    match self.register_node_with_peer_info(
+                    match self.register_node_with_peer_info(outbound_relay_peerinfo, false) {
-                        RoutingDomain::PublicInternet,
-                        outbound_relay_peerinfo,
-                        false,
-                    ) {
                         Ok(nr) => {
                             log_rtab!(debug "Outbound relay node selected: {}", nr);
-                            editor.set_relay_node(nr);
+                            editor.set_relay_node(nr.unfiltered());
                             got_outbound_relay = true;
                         }
                         Err(e) => {
@@ -141,7 +139,14 @@ impl RoutingTable {
         }
 
         // Commit the changes
-        editor.commit(false).await;
+        if editor.commit(false).await {
+            // Try to publish the peer info
+            editor.publish();
+
+            self.network_manager()
+                .connection_manager()
+                .update_protections();
+        }
 
         Ok(())
     }
@@ -152,7 +157,7 @@ impl RoutingTable {
         let outbound_dif = self.get_outbound_dial_info_filter(RoutingDomain::PublicInternet);
         let mapped_port_info = self.get_low_level_port_info();
         let own_node_info = self
-            .get_own_peer_info(RoutingDomain::PublicInternet)
+            .get_current_peer_info(RoutingDomain::PublicInternet)
             .signed_node_info()
             .node_info()
             .clone();
@@ -174,11 +179,9 @@ impl RoutingTable {
 
             // Exclude any nodes that have 'failed to send' state indicating a
             // connection drop or inability to reach the node
-
+            if e.peer_stats().rpc_stats.failed_to_send > 0 {
-            // XXX: we should be able to enable this!
+                return false;
-            // if e.peer_stats().rpc_stats.failed_to_send > 0 {
+            }
-            //     return false;
-            // }
 
             // Until we have a way of reducing a SignedRelayedNodeInfo to a SignedDirectNodeInfo
             // See https://gitlab.com/veilid/veilid/-/issues/381
@@ -200,7 +203,7 @@ impl RoutingTable {
             // Disqualify nodes that don't cover all our inbound ports for tcp and udp
             // as we need to be able to use the relay for keepalives for all nat mappings
             let mut low_level_protocol_ports = mapped_port_info.low_level_protocol_ports.clone();
-            let dids = node_info.all_filtered_dial_info_details(DialInfoDetail::NO_SORT, |did| {
+            let dids = node_info.filtered_dial_info_details(DialInfoDetail::NO_SORT, |did| {
                 did.matches_filter(&outbound_dif)
             });
             for did in &dids {
@@ -282,6 +285,6 @@ impl RoutingTable {
             Option::<()>::None
         });
         // Return the best inbound relay noderef
-        best_inbound_relay.map(|e| NodeRef::new(self.clone(), e, None))
+        best_inbound_relay.map(|e| NodeRef::new(self.clone(), e))
     }
 }

veilid-core/src/routing_table/tests/test_serialize_routing_table.rs

@@ -65,6 +65,7 @@ pub async fn test_round_trip_peerinfo() {
         ]),
     ));
     let pi: PeerInfo = PeerInfo::new(
+        RoutingDomain::PublicInternet,
         tks,
         SignedNodeInfo::Direct(SignedDirectNodeInfo::new(
             NodeInfo::new(

veilid-core/src/routing_table/types/contact_method.rs

@@ -0,0 +1,20 @@
+use super::*;
+
+/// Mechanism required to contact another node
+#[derive(Clone, Debug)]
+pub(crate) enum ContactMethod {
+    /// Node is not reachable by any means
+    Unreachable,
+    /// Connection should have already existed
+    Existing,
+    /// Contact the node directly
+    Direct(DialInfo),
+    /// Request via signal the node connect back directly (relay, target)
+    SignalReverse(TypedKey, TypedKey),
+    /// Request via signal the node negotiate a hole punch (relay, target)
+    SignalHolePunch(TypedKey, TypedKey),
+    /// Must use an inbound relay to reach the node
+    InboundRelay(TypedKey),
+    /// Must use outbound relay to reach the node
+    OutboundRelay(TypedKey),
+}

veilid-core/src/routing_table/types/mod.rs

@@ -1,3 +1,4 @@
+mod contact_method;
 mod dial_info_detail;
 mod direction;
 mod node_info;
@@ -10,6 +11,7 @@ mod signed_relayed_node_info;
 
 use super::*;
 
+pub(crate) use contact_method::*;
 pub use dial_info_detail::*;
 pub use direction::*;
 pub use node_info::*;

veilid-core/src/routing_table/types/node_info.rs

@@ -96,7 +96,7 @@ impl NodeInfo {
         None
     }
 
-    pub fn all_filtered_dial_info_details<S, F>(
+    pub fn filtered_dial_info_details<S, F>(
         &self,
         sort: Option<S>,
         filter: F,

veilid-core/src/routing_table/types/peer_info.rs

@@ -2,14 +2,20 @@ use super::*;
 
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct PeerInfo {
+    routing_domain: RoutingDomain,
     node_ids: TypedKeyGroup,
     signed_node_info: SignedNodeInfo,
 }
 
 impl PeerInfo {
-    pub fn new(node_ids: TypedKeyGroup, signed_node_info: SignedNodeInfo) -> Self {
+    pub fn new(
+        routing_domain: RoutingDomain,
+        node_ids: TypedKeyGroup,
+        signed_node_info: SignedNodeInfo,
+    ) -> Self {
         assert!(!node_ids.is_empty() && node_ids.len() <= MAX_CRYPTO_KINDS);
         Self {
+            routing_domain,
             node_ids,
             signed_node_info,
         }
@@ -24,17 +30,20 @@ impl PeerInfo {
         Ok(())
     }
 
+    pub fn routing_domain(&self) -> RoutingDomain {
+        self.routing_domain
+    }
     pub fn node_ids(&self) -> &TypedKeyGroup {
         &self.node_ids
     }
     pub fn signed_node_info(&self) -> &SignedNodeInfo {
         &self.signed_node_info
     }
-    pub fn destructure(self) -> (TypedKeyGroup, SignedNodeInfo) {
+    pub fn destructure(self) -> (RoutingDomain, TypedKeyGroup, SignedNodeInfo) {
-        (self.node_ids, self.signed_node_info)
+        (self.routing_domain, self.node_ids, self.signed_node_info)
     }
 
-    pub fn validate_vec(peer_info_vec: &mut Vec<PeerInfo>, crypto: Crypto) {
+    pub fn validate_vec(peer_info_vec: &mut Vec<Arc<PeerInfo>>, crypto: Crypto) {
         let mut n = 0usize;
         while n < peer_info_vec.len() {
             let pi = peer_info_vec.get(n).unwrap();
@@ -45,4 +54,16 @@ impl PeerInfo {
             }
         }
     }
+
+    /// Compare this PeerInfo to another one
+    /// Exclude the signature and timestamp and any other fields that are not
+    /// semantically valuable
+    /// If the two are not equivalent they should be considered different
+    /// enough for republication, but this is not the only criteria required
+    /// for publication.
+    pub fn equivalent(&self, other: &PeerInfo) -> bool {
+        self.routing_domain == other.routing_domain
+            && self.node_ids == other.node_ids
+            && self.signed_node_info.equivalent(&other.signed_node_info)
+    }
 }

veilid-core/src/routing_table/types/signed_direct_node_info.rs

@@ -96,4 +96,13 @@ impl SignedDirectNodeInfo {
     pub fn signatures(&self) -> &[TypedSignature] {
         &self.signatures
     }
+
+    /// Compare this SignedDirectNodeInfo to another one
+    /// Exclude the signature and timestamp and any other fields that are not
+    /// semantically valuable
+    pub fn equivalent(&self, other: &SignedDirectNodeInfo) -> bool {
+        let a = self.node_info();
+        let b = other.node_info();
+        a == b
+    }
 }

veilid-core/src/routing_table/types/signed_node_info.rs

@@ -49,13 +49,14 @@ impl SignedNodeInfo {
             SignedNodeInfo::Relayed(r) => Some(r.relay_info().node_info()),
         }
     }
-    pub fn relay_peer_info(&self) -> Option<PeerInfo> {
+    pub fn relay_peer_info(&self, routing_domain: RoutingDomain) -> Option<Arc<PeerInfo>> {
         match self {
             SignedNodeInfo::Direct(_) => None,
-            SignedNodeInfo::Relayed(r) => Some(PeerInfo::new(
+            SignedNodeInfo::Relayed(r) => Some(Arc::new(PeerInfo::new(
+                routing_domain,
                 r.relay_ids().clone(),
                 SignedNodeInfo::Direct(r.relay_info().clone()),
-            )),
+            ))),
         }
     }
     pub fn has_any_dial_info(&self) -> bool {
@@ -96,4 +97,20 @@ impl SignedNodeInfo {
             })
             .unwrap_or_default();
     }
+
+    /// Compare this SignedNodeInfo to another one
+    /// Exclude the signature and timestamp and any other fields that are not
+    /// semantically valuable
+    pub fn equivalent(&self, other: &SignedNodeInfo) -> bool {
+        match self {
+            SignedNodeInfo::Direct(d) => match other {
+                SignedNodeInfo::Direct(pd) => d.equivalent(pd),
+                SignedNodeInfo::Relayed(_) => true,
+            },
+            SignedNodeInfo::Relayed(r) => match other {
+                SignedNodeInfo::Direct(_) => true,
+                SignedNodeInfo::Relayed(pr) => r.equivalent(pr),
+            },
+        }
+    }
 }

veilid-core/src/routing_table/types/signed_relayed_node_info.rs

@@ -141,4 +141,18 @@ impl SignedRelayedNodeInfo {
     pub fn signatures(&self) -> &[TypedSignature] {
         &self.signatures
     }
+
+    /// Compare this SignedRelayedNodeInfo to another one
+    /// Exclude the signature and timestamp and any other fields that are not
+    /// semantically valuable
+    pub fn equivalent(&self, other: &SignedRelayedNodeInfo) -> bool {
+        let a = self.node_info();
+        let b = other.node_info();
+        let ari = self.relay_ids();
+        let bri = other.relay_ids();
+        let ar = self.relay_info();
+        let br = other.relay_info();
+
+        a == b && ari == bri && ar.equivalent(br)
+    }
 }

veilid-core/src/rpc_processor/coders/mod.rs

@@ -74,3 +74,8 @@ pub(in crate::rpc_processor) struct RPCValidateContext {
     // pub rpc_processor: RPCProcessor,
     pub question_context: Option<QuestionContext>,
 }
+
+#[derive(Clone)]
+pub(crate) struct RPCDecodeContext {
+    pub routing_domain: RoutingDomain,
+}

veilid-core/src/rpc_processor/coders/operations/answer.rs

@@ -12,16 +12,18 @@ impl RPCAnswer {
     pub fn validate(&mut self, validate_context: &RPCValidateContext) -> Result<(), RPCError> {
         self.detail.validate(validate_context)
     }
-    #[cfg(feature = "verbose-tracing")]
     pub fn desc(&self) -> &'static str {
         self.detail.desc()
     }
     pub fn destructure(self) -> RPCAnswerDetail {
         self.detail
     }
-    pub fn decode(reader: &veilid_capnp::answer::Reader) -> Result<RPCAnswer, RPCError> {
+    pub fn decode(
+        decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::answer::Reader,
+    ) -> Result<RPCAnswer, RPCError> {
         let d_reader = reader.get_detail();
-        let detail = RPCAnswerDetail::decode(&d_reader)?;
+        let detail = RPCAnswerDetail::decode(decode_context, &d_reader)?;
         Ok(RPCAnswer { detail })
     }
     pub fn encode(&self, builder: &mut veilid_capnp::answer::Builder) -> Result<(), RPCError> {
@@ -52,7 +54,6 @@ pub(in crate::rpc_processor) enum RPCAnswerDetail {
 }
 
 impl RPCAnswerDetail {
-    #[cfg(feature = "verbose-tracing")]
     pub fn desc(&self) -> &'static str {
         match self {
             RPCAnswerDetail::StatusA(_) => "StatusA",
@@ -96,73 +97,74 @@ impl RPCAnswerDetail {
         }
     }
     pub fn decode(
+        decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::answer::detail::Reader,
     ) -> Result<RPCAnswerDetail, RPCError> {
         let which_reader = reader.which().map_err(RPCError::protocol)?;
         let out = match which_reader {
             veilid_capnp::answer::detail::StatusA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationStatusA::decode(&op_reader)?;
+                let out = RPCOperationStatusA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::StatusA(Box::new(out))
             }
             veilid_capnp::answer::detail::FindNodeA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationFindNodeA::decode(&op_reader)?;
+                let out = RPCOperationFindNodeA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::FindNodeA(Box::new(out))
             }
             veilid_capnp::answer::detail::AppCallA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationAppCallA::decode(&op_reader)?;
+                let out = RPCOperationAppCallA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::AppCallA(Box::new(out))
             }
             veilid_capnp::answer::detail::GetValueA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationGetValueA::decode(&op_reader)?;
+                let out = RPCOperationGetValueA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::GetValueA(Box::new(out))
             }
             veilid_capnp::answer::detail::SetValueA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationSetValueA::decode(&op_reader)?;
+                let out = RPCOperationSetValueA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::SetValueA(Box::new(out))
             }
             veilid_capnp::answer::detail::WatchValueA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationWatchValueA::decode(&op_reader)?;
+                let out = RPCOperationWatchValueA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::WatchValueA(Box::new(out))
             }
             veilid_capnp::answer::detail::InspectValueA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationInspectValueA::decode(&op_reader)?;
+                let out = RPCOperationInspectValueA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::InspectValueA(Box::new(out))
             }
             #[cfg(feature = "unstable-blockstore")]
             veilid_capnp::answer::detail::SupplyBlockA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationSupplyBlockA::decode(&op_reader)?;
+                let out = RPCOperationSupplyBlockA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::SupplyBlockA(Box::new(out))
             }
             #[cfg(feature = "unstable-blockstore")]
             veilid_capnp::answer::detail::FindBlockA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationFindBlockA::decode(&op_reader)?;
+                let out = RPCOperationFindBlockA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::FindBlockA(Box::new(out))
             }
             #[cfg(feature = "unstable-tunnels")]
             veilid_capnp::answer::detail::StartTunnelA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationStartTunnelA::decode(&op_reader)?;
+                let out = RPCOperationStartTunnelA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::StartTunnelA(Box::new(out))
             }
             #[cfg(feature = "unstable-tunnels")]
             veilid_capnp::answer::detail::CompleteTunnelA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationCompleteTunnelA::decode(&op_reader)?;
+                let out = RPCOperationCompleteTunnelA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::CompleteTunnelA(Box::new(out))
             }
             #[cfg(feature = "unstable-tunnels")]
             veilid_capnp::answer::detail::CancelTunnelA(r) => {
                 let op_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCOperationCancelTunnelA::decode(&op_reader)?;
+                let out = RPCOperationCancelTunnelA::decode(decode_context, &op_reader)?;
                 RPCAnswerDetail::CancelTunnelA(Box::new(out))
             }
         };

veilid-core/src/rpc_processor/coders/operations/operation.rs

@@ -8,7 +8,6 @@ pub(in crate::rpc_processor) enum RPCOperationKind {
 }
 
 impl RPCOperationKind {
-    #[cfg(feature = "verbose-tracing")]
     pub fn desc(&self) -> &'static str {
         match self {
             RPCOperationKind::Question(q) => q.desc(),
@@ -25,22 +24,25 @@ impl RPCOperationKind {
         }
     }
 
-    pub fn decode(kind_reader: &veilid_capnp::operation::kind::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        decode_context: &RPCDecodeContext,
+        kind_reader: &veilid_capnp::operation::kind::Reader,
+    ) -> Result<Self, RPCError> {
         let which_reader = kind_reader.which().map_err(RPCError::protocol)?;
         let out = match which_reader {
             veilid_capnp::operation::kind::Which::Question(r) => {
                 let q_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCQuestion::decode(&q_reader)?;
+                let out = RPCQuestion::decode(decode_context, &q_reader)?;
                 RPCOperationKind::Question(Box::new(out))
             }
             veilid_capnp::operation::kind::Which::Statement(r) => {
                 let q_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCStatement::decode(&q_reader)?;
+                let out = RPCStatement::decode(decode_context, &q_reader)?;
                 RPCOperationKind::Statement(Box::new(out))
             }
             veilid_capnp::operation::kind::Which::Answer(r) => {
                 let q_reader = r.map_err(RPCError::protocol)?;
-                let out = RPCAnswer::decode(&q_reader)?;
+                let out = RPCAnswer::decode(decode_context, &q_reader)?;
                 RPCOperationKind::Answer(Box::new(out))
             }
         };
@@ -63,8 +65,7 @@ impl RPCOperationKind {
 #[derive(Debug, Clone)]
 pub(in crate::rpc_processor) struct RPCOperation {
     op_id: OperationId,
-    opt_sender_peer_info: Option<PeerInfo>,
+    sender_peer_info: SenderPeerInfo,
-    target_node_info_ts: Timestamp,
     kind: RPCOperationKind,
 }
 
@@ -72,16 +73,14 @@ impl RPCOperation {
     pub fn new_question(question: RPCQuestion, sender_peer_info: SenderPeerInfo) -> Self {
         Self {
             op_id: OperationId::new(get_random_u64()),
-            opt_sender_peer_info: sender_peer_info.opt_sender_peer_info,
+            sender_peer_info,
-            target_node_info_ts: sender_peer_info.target_node_info_ts,
             kind: RPCOperationKind::Question(Box::new(question)),
         }
     }
     pub fn new_statement(statement: RPCStatement, sender_peer_info: SenderPeerInfo) -> Self {
         Self {
             op_id: OperationId::new(get_random_u64()),
-            opt_sender_peer_info: sender_peer_info.opt_sender_peer_info,
+            sender_peer_info,
-            target_node_info_ts: sender_peer_info.target_node_info_ts,
             kind: RPCOperationKind::Statement(Box::new(statement)),
         }
     }
@@ -93,15 +92,14 @@ impl RPCOperation {
     ) -> Self {
         Self {
             op_id: request.op_id,
-            opt_sender_peer_info: sender_peer_info.opt_sender_peer_info,
+            sender_peer_info,
-            target_node_info_ts: sender_peer_info.target_node_info_ts,
             kind: RPCOperationKind::Answer(Box::new(answer)),
         }
     }
 
     pub fn validate(&mut self, validate_context: &RPCValidateContext) -> Result<(), RPCError> {
         // Validate sender peer info
-        if let Some(sender_peer_info) = &self.opt_sender_peer_info {
+        if let Some(sender_peer_info) = &self.sender_peer_info.opt_peer_info {
             sender_peer_info
                 .validate(validate_context.crypto.clone())
                 .map_err(RPCError::protocol)?;
@@ -115,34 +113,29 @@ impl RPCOperation {
         self.op_id
     }
 
-    pub fn sender_peer_info(&self) -> Option<&PeerInfo> {
+    pub fn sender_peer_info(&self) -> SenderPeerInfo {
-        self.opt_sender_peer_info.as_ref()
+        self.sender_peer_info.clone()
-    }
-    pub fn target_node_info_ts(&self) -> Timestamp {
-        self.target_node_info_ts
     }
 
     pub fn kind(&self) -> &RPCOperationKind {
         &self.kind
     }
 
-    pub fn destructure(self) -> (OperationId, Option<PeerInfo>, Timestamp, RPCOperationKind) {
+    pub fn destructure(self) -> (OperationId, SenderPeerInfo, RPCOperationKind) {
-        (
+        (self.op_id, self.sender_peer_info, self.kind)
-            self.op_id,
-            self.opt_sender_peer_info,
-            self.target_node_info_ts,
-            self.kind,
-        )
     }
 
-    pub fn decode(operation_reader: &veilid_capnp::operation::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        decode_context: &RPCDecodeContext,
+        operation_reader: &veilid_capnp::operation::Reader,
+    ) -> Result<Self, RPCError> {
         let op_id = OperationId::new(operation_reader.get_op_id());
 
-        let sender_peer_info = if operation_reader.has_sender_peer_info() {
+        let opt_peer_info = if operation_reader.has_sender_peer_info() {
             let pi_reader = operation_reader
                 .get_sender_peer_info()
                 .map_err(RPCError::protocol)?;
-            let pi = decode_peer_info(&pi_reader)?;
+            let pi = Arc::new(decode_peer_info(decode_context, &pi_reader)?);
             Some(pi)
         } else {
             None
@@ -151,23 +144,25 @@ impl RPCOperation {
         let target_node_info_ts = Timestamp::new(operation_reader.get_target_node_info_ts());
 
         let kind_reader = operation_reader.get_kind();
-        let kind = RPCOperationKind::decode(&kind_reader)?;
+        let kind = RPCOperationKind::decode(decode_context, &kind_reader)?;
 
         Ok(RPCOperation {
             op_id,
-            opt_sender_peer_info: sender_peer_info,
+            sender_peer_info: SenderPeerInfo {
-            target_node_info_ts,
+                opt_peer_info,
+                target_node_info_ts,
+            },
             kind,
         })
     }
 
     pub fn encode(&self, builder: &mut veilid_capnp::operation::Builder) -> Result<(), RPCError> {
         builder.set_op_id(self.op_id.as_u64());
-        if let Some(sender_peer_info) = &self.opt_sender_peer_info {
+        if let Some(sender_peer_info) = &self.sender_peer_info.opt_peer_info {
             let mut pi_builder = builder.reborrow().init_sender_peer_info();
             encode_peer_info(sender_peer_info, &mut pi_builder)?;
         }
-        builder.set_target_node_info_ts(self.target_node_info_ts.as_u64());
+        builder.set_target_node_info_ts(self.sender_peer_info.target_node_info_ts.as_u64());
         let mut k_builder = builder.reborrow().init_kind();
         self.kind.encode(&mut k_builder)?;
         Ok(())

veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs

@@ -27,7 +27,10 @@ impl RPCOperationAppCallQ {
         self.message
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_app_call_q::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        _decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_app_call_q::Reader,
+    ) -> Result<Self, RPCError> {
         let mr = reader.get_message().map_err(RPCError::protocol)?;
         if mr.len() > MAX_APP_CALL_Q_MESSAGE_LEN {
             return Err(RPCError::protocol("AppCallQ message too long to set"));
@@ -72,7 +75,10 @@ impl RPCOperationAppCallA {
         self.message
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_app_call_a::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        _decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_app_call_a::Reader,
+    ) -> Result<Self, RPCError> {
         let mr = reader.get_message().map_err(RPCError::protocol)?;
         if mr.len() > MAX_APP_CALL_A_MESSAGE_LEN {
             return Err(RPCError::protocol("AppCallA message too long to set"));

veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs

@@ -26,7 +26,10 @@ impl RPCOperationAppMessage {
         self.message
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_app_message::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        _decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_app_message::Reader,
+    ) -> Result<Self, RPCError> {
         let mr = reader.get_message().map_err(RPCError::protocol)?;
         if mr.len() > MAX_APP_MESSAGE_MESSAGE_LEN {
             return Err(RPCError::protocol("AppMessage message too long to set"));

veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs

@@ -23,6 +23,7 @@ impl RPCOperationCancelTunnelQ {
     }
 
     pub fn decode(
+        decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_cancel_tunnel_q::Reader,
     ) -> Result<Self, RPCError> {
         let id = TunnelId::new(reader.get_id());

veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs

@@ -40,6 +40,7 @@ impl RPCOperationCompleteTunnelQ {
     }
 
     pub fn decode(
+        decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_complete_tunnel_q::Reader,
     ) -> Result<Self, RPCError> {
         let id = TunnelId::new(reader.get_id());

veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs

@@ -26,6 +26,7 @@ impl RPCOperationFindBlockQ {
     }
 
     pub fn decode(
+        decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_find_block_q::Reader,
     ) -> Result<RPCOperationFindBlockQ, RPCError> {
         let bi_reader = reader.get_block_id().map_err(RPCError::protocol)?;

veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs

@@ -30,7 +30,10 @@ impl RPCOperationFindNodeQ {
         (self.node_id, self.capabilities)
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_find_node_q::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        _decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_find_node_q::Reader,
+    ) -> Result<Self, RPCError> {
         let ni_reader = reader.get_node_id().map_err(RPCError::protocol)?;
         let node_id = decode_typed_key(&ni_reader)?;
         let cap_reader = reader
@@ -77,11 +80,11 @@ impl RPCOperationFindNodeQ {
 
 #[derive(Debug, Clone)]
 pub(in crate::rpc_processor) struct RPCOperationFindNodeA {
-    peers: Vec<PeerInfo>,
+    peers: Vec<Arc<PeerInfo>>,
 }
 
 impl RPCOperationFindNodeA {
-    pub fn new(peers: Vec<PeerInfo>) -> Result<Self, RPCError> {
+    pub fn new(peers: Vec<Arc<PeerInfo>>) -> Result<Self, RPCError> {
         if peers.len() > MAX_FIND_NODE_A_PEERS_LEN {
             return Err(RPCError::protocol(
                 "encoded find node peers length too long",
@@ -100,11 +103,12 @@ impl RPCOperationFindNodeA {
     //     &self.peers
     // }
 
-    pub fn destructure(self) -> Vec<PeerInfo> {
+    pub fn destructure(self) -> Vec<Arc<PeerInfo>> {
         self.peers
     }
 
     pub fn decode(
+        decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_find_node_a::Reader,
     ) -> Result<RPCOperationFindNodeA, RPCError> {
         let peers_reader = reader.get_peers().map_err(RPCError::protocol)?;
@@ -115,15 +119,15 @@ impl RPCOperationFindNodeA {
             ));
         }
 
-        let mut peers = Vec::<PeerInfo>::with_capacity(
+        let mut peers = Vec::<Arc<PeerInfo>>::with_capacity(
             peers_reader
                 .len()
                 .try_into()
                 .map_err(RPCError::map_internal("too many peers"))?,
         );
         for p in peers_reader.iter() {
-            let peer_info = decode_peer_info(&p)?;
+            let peer_info = decode_peer_info(decode_context, &p)?;
-            peers.push(peer_info);
+            peers.push(Arc::new(peer_info));
         }
 
         Ok(Self { peers })

veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs

@@ -52,7 +52,10 @@ impl RPCOperationGetValueQ {
         (self.key, self.subkey, self.want_descriptor)
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_get_value_q::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        _decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_get_value_q::Reader,
+    ) -> Result<Self, RPCError> {
         let k_reader = reader.reborrow().get_key().map_err(RPCError::protocol)?;
         let key = decode_typed_key(&k_reader)?;
         let subkey = reader.reborrow().get_subkey();
@@ -80,14 +83,14 @@ impl RPCOperationGetValueQ {
 #[derive(Debug, Clone)]
 pub(in crate::rpc_processor) struct RPCOperationGetValueA {
     value: Option<SignedValueData>,
-    peers: Vec<PeerInfo>,
+    peers: Vec<Arc<PeerInfo>>,
     descriptor: Option<SignedValueDescriptor>,
 }
 
 impl RPCOperationGetValueA {
     pub fn new(
         value: Option<SignedValueData>,
-        peers: Vec<PeerInfo>,
+        peers: Vec<Arc<PeerInfo>>,
         descriptor: Option<SignedValueDescriptor>,
     ) -> Result<Self, RPCError> {
         if peers.len() > MAX_GET_VALUE_A_PEERS_LEN {
@@ -171,13 +174,16 @@ impl RPCOperationGetValueA {
         self,
     ) -> (
         Option<SignedValueData>,
-        Vec<PeerInfo>,
+        Vec<Arc<PeerInfo>>,
         Option<SignedValueDescriptor>,
     ) {
         (self.value, self.peers, self.descriptor)
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_get_value_a::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_get_value_a::Reader,
+    ) -> Result<Self, RPCError> {
         let value = if reader.has_value() {
             let value_reader = reader.get_value().map_err(RPCError::protocol)?;
             let value = decode_signed_value_data(&value_reader)?;
@@ -192,14 +198,14 @@ impl RPCOperationGetValueA {
                 "decoded GetValueA peers length too long",
             ));
         }
-        let mut peers = Vec::<PeerInfo>::with_capacity(
+        let mut peers = Vec::<Arc<PeerInfo>>::with_capacity(
             peers_reader
                 .len()
                 .try_into()
                 .map_err(RPCError::map_internal("too many peers"))?,
         );
         for p in peers_reader.iter() {
-            let peer_info = decode_peer_info(&p)?;
+            let peer_info = Arc::new(decode_peer_info(decode_context, &p)?);
             peers.push(peer_info);
         }
 

veilid-core/src/rpc_processor/coders/operations/operation_inspect_value.rs

@@ -58,6 +58,7 @@ impl RPCOperationInspectValueQ {
     }
 
     pub fn decode(
+        _decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_inspect_value_q::Reader,
     ) -> Result<Self, RPCError> {
         let k_reader = reader.reborrow().get_key().map_err(RPCError::protocol)?;
@@ -118,14 +119,14 @@ impl RPCOperationInspectValueQ {
 #[derive(Debug, Clone)]
 pub(in crate::rpc_processor) struct RPCOperationInspectValueA {
     seqs: Vec<ValueSeqNum>,
-    peers: Vec<PeerInfo>,
+    peers: Vec<Arc<PeerInfo>>,
     descriptor: Option<SignedValueDescriptor>,
 }
 
 impl RPCOperationInspectValueA {
     pub fn new(
         seqs: Vec<ValueSeqNum>,
-        peers: Vec<PeerInfo>,
+        peers: Vec<Arc<PeerInfo>>,
         descriptor: Option<SignedValueDescriptor>,
     ) -> Result<Self, RPCError> {
         if seqs.len() > MAX_INSPECT_VALUE_A_SEQS_LEN {
@@ -198,13 +199,14 @@ impl RPCOperationInspectValueA {
         self,
     ) -> (
         Vec<ValueSeqNum>,
-        Vec<PeerInfo>,
+        Vec<Arc<PeerInfo>>,
         Option<SignedValueDescriptor>,
     ) {
         (self.seqs, self.peers, self.descriptor)
     }
 
     pub fn decode(
+        decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_inspect_value_a::Reader,
     ) -> Result<Self, RPCError> {
         let seqs = if reader.has_seqs() {
@@ -228,14 +230,14 @@ impl RPCOperationInspectValueA {
                 "decoded InspectValueA peers length too long",
             ));
         }
-        let mut peers = Vec::<PeerInfo>::with_capacity(
+        let mut peers = Vec::<Arc<PeerInfo>>::with_capacity(
             peers_reader
                 .len()
                 .try_into()
                 .map_err(RPCError::map_internal("too many peers"))?,
         );
         for p in peers_reader.iter() {
-            let peer_info = decode_peer_info(&p)?;
+            let peer_info = Arc::new(decode_peer_info(decode_context, &p)?);
             peers.push(peer_info);
         }
 

veilid-core/src/rpc_processor/coders/operations/operation_return_receipt.rs

@@ -29,6 +29,7 @@ impl RPCOperationReturnReceipt {
     }
 
     pub fn decode(
+        _decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_return_receipt::Reader,
     ) -> Result<Self, RPCError> {
         let rr = reader.get_receipt().map_err(RPCError::protocol)?;

veilid-core/src/rpc_processor/coders/operations/operation_route.rs

@@ -2,6 +2,7 @@ use super::*;
 
 #[derive(Clone)]
 pub(in crate::rpc_processor) struct RoutedOperation {
+    routing_domain: RoutingDomain,
     sequencing: Sequencing,
     signatures: Vec<Signature>,
     nonce: Nonce,
@@ -11,6 +12,7 @@ pub(in crate::rpc_processor) struct RoutedOperation {
 impl fmt::Debug for RoutedOperation {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("RoutedOperation")
+            .field("routing_domain", &self.routing_domain)
             .field("sequencing", &self.sequencing)
             .field("signatures.len", &self.signatures.len())
             .field("nonce", &self.nonce)
@@ -20,8 +22,14 @@ impl fmt::Debug for RoutedOperation {
 }
 
 impl RoutedOperation {
-    pub fn new(sequencing: Sequencing, nonce: Nonce, data: Vec<u8>) -> Self {
+    pub fn new(
+        routing_domain: RoutingDomain,
+        sequencing: Sequencing,
+        nonce: Nonce,
+        data: Vec<u8>,
+    ) -> Self {
         Self {
+            routing_domain,
             sequencing,
             signatures: Vec::new(),
             nonce,
@@ -32,6 +40,9 @@ impl RoutedOperation {
         //xxx
         Ok(())
     }
+    pub fn routing_domain(&self) -> RoutingDomain {
+        self.routing_domain
+    }
     pub fn sequencing(&self) -> Sequencing {
         self.sequencing
     }
@@ -54,7 +65,10 @@ impl RoutedOperation {
     //     (self.sequencing, self.signatures, self.nonce, self.data)
     // }
 
-    pub fn decode(reader: &veilid_capnp::routed_operation::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::routed_operation::Reader,
+    ) -> Result<Self, RPCError> {
         let sigs_reader = reader.get_signatures().map_err(RPCError::protocol)?;
         let mut signatures = Vec::<Signature>::with_capacity(
             sigs_reader
@@ -73,6 +87,7 @@ impl RoutedOperation {
         let data = reader.get_data().map_err(RPCError::protocol)?;
 
         Ok(Self {
+            routing_domain: decode_context.routing_domain,
             sequencing,
             signatures,
             nonce,
@@ -132,12 +147,15 @@ impl RPCOperationRoute {
         (self.safety_route, self.operation)
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_route::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_route::Reader,
+    ) -> Result<Self, RPCError> {
         let sr_reader = reader.get_safety_route().map_err(RPCError::protocol)?;
-        let safety_route = decode_safety_route(&sr_reader)?;
+        let safety_route = decode_safety_route(decode_context, &sr_reader)?;
 
         let o_reader = reader.get_operation().map_err(RPCError::protocol)?;
-        let operation = RoutedOperation::decode(&o_reader)?;
+        let operation = RoutedOperation::decode(decode_context, &o_reader)?;
 
         Ok(Self {
             safety_route,

veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs

@@ -72,7 +72,10 @@ impl RPCOperationSetValueQ {
         (self.key, self.subkey, self.value, self.descriptor)
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_set_value_q::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        _decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_set_value_q::Reader,
+    ) -> Result<Self, RPCError> {
         let k_reader = reader.get_key().map_err(RPCError::protocol)?;
         let key = decode_typed_key(&k_reader)?;
         let subkey = reader.get_subkey();
@@ -115,14 +118,14 @@ impl RPCOperationSetValueQ {
 pub(in crate::rpc_processor) struct RPCOperationSetValueA {
     set: bool,
     value: Option<SignedValueData>,
-    peers: Vec<PeerInfo>,
+    peers: Vec<Arc<PeerInfo>>,
 }
 
 impl RPCOperationSetValueA {
     pub fn new(
         set: bool,
         value: Option<SignedValueData>,
-        peers: Vec<PeerInfo>,
+        peers: Vec<Arc<PeerInfo>>,
     ) -> Result<Self, RPCError> {
         if peers.len() > MAX_SET_VALUE_A_PEERS_LEN {
             return Err(RPCError::protocol(
@@ -174,11 +177,14 @@ impl RPCOperationSetValueA {
     // pub fn peers(&self) -> &[PeerInfo] {
     //     &self.peers
     // }
-    pub fn destructure(self) -> (bool, Option<SignedValueData>, Vec<PeerInfo>) {
+    pub fn destructure(self) -> (bool, Option<SignedValueData>, Vec<Arc<PeerInfo>>) {
         (self.set, self.value, self.peers)
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_set_value_a::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
+        decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_set_value_a::Reader,
+    ) -> Result<Self, RPCError> {
         let set = reader.get_set();
         let value = if reader.has_value() {
             let v_reader = reader.get_value().map_err(RPCError::protocol)?;
@@ -193,14 +199,14 @@ impl RPCOperationSetValueA {
                 "decoded SetValueA peers length too long",
             ));
         }
-        let mut peers = Vec::<PeerInfo>::with_capacity(
+        let mut peers = Vec::<Arc<PeerInfo>>::with_capacity(
             peers_reader
                 .len()
                 .try_into()
                 .map_err(RPCError::map_internal("too many peers"))?,
         );
         for p in peers_reader.iter() {
-            let peer_info = decode_peer_info(&p)?;
+            let peer_info = Arc::new(decode_peer_info(decode_context, &p)?);
             peers.push(peer_info);
         }
 

veilid-core/src/rpc_processor/coders/operations/operation_signal.rs

@@ -19,8 +19,11 @@ impl RPCOperationSignal {
         self.signal_info
     }
 
-    pub fn decode(reader: &veilid_capnp::operation_signal::Reader) -> Result<Self, RPCError> {
+    pub fn decode(
-        let signal_info = decode_signal_info(reader)?;
+        decode_context: &RPCDecodeContext,
+        reader: &veilid_capnp::operation_signal::Reader,
+    ) -> Result<Self, RPCError> {
+        let signal_info = decode_signal_info(decode_context, reader)?;
         Ok(Self { signal_info })
     }
     pub fn encode(

veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs

@@ -35,6 +35,7 @@ impl RPCOperationStartTunnelQ {
     }
 
     pub fn decode(
+        decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_start_tunnel_q::Reader,
     ) -> Result<Self, RPCError> {
         let id = TunnelId::new(reader.get_id());
@@ -86,6 +87,7 @@ impl RPCOperationStartTunnelA {
     }
 
     pub fn decode(
+        decode_context: &RPCDecodeContext,
         reader: &veilid_capnp::operation_start_tunnel_a::Reader,
     ) -> Result<Self, RPCError> {
         match reader.which().map_err(RPCError::protocol)? {