From 4bc3f950dfaac72a182b8afc1926f9d4c1f5a298 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 25 Feb 2023 19:51:14 -0500 Subject: [PATCH] more refactor --- veilid-core/src/crypto/byte_array_types.rs | 4 + veilid-core/src/crypto/types.rs | 1 + .../src/routing_table/route_spec_store/mod.rs | 8 +- .../route_spec_store/permutation.rs | 70 +++ .../remote_private_route_info.rs | 7 + .../route_spec_store/route_set_spec_detail.rs | 31 +- .../route_spec_store/route_spec_store.rs | 489 ++++++------------ .../route_spec_store_cache.rs | 239 +++++---- .../route_spec_store_content.rs | 142 ++++- .../tasks/private_route_management.rs | 80 +-- veilid-core/src/rpc_processor/destination.rs | 18 +- veilid-core/src/rpc_processor/mod.rs | 4 +- .../src/rpc_processor/rpc_find_node.rs | 2 +- veilid-core/src/rpc_processor/rpc_signal.rs | 2 +- veilid-core/src/rpc_processor/rpc_status.rs | 4 +- veilid-core/src/veilid_api/routing_context.rs | 9 +- veilid-core/src/veilid_api/types.rs | 6 +- 17 files changed, 587 insertions(+), 529 deletions(-) create mode 100644 veilid-core/src/routing_table/route_spec_store/permutation.rs diff --git a/veilid-core/src/crypto/byte_array_types.rs b/veilid-core/src/crypto/byte_array_types.rs index 7880a695..17b32fa4 100644 --- a/veilid-core/src/crypto/byte_array_types.rs +++ b/veilid-core/src/crypto/byte_array_types.rs @@ -41,6 +41,9 @@ pub const SHARED_SECRET_LENGTH: usize = 32; /// Length of a shared secret in bytes after encoding to base64url #[allow(dead_code)] pub const SHARED_SECRET_LENGTH_ENCODED: usize = 43; +/// Length of a route id in bytes +#[allow(dead_code)] +pub const ROUTE_ID_LENGTH: usize = 32; ////////////////////////////////////////////////////////////////////// @@ -260,3 +263,4 @@ byte_array_type!(Signature, SIGNATURE_LENGTH); byte_array_type!(PublicKeyDistance, PUBLIC_KEY_LENGTH); byte_array_type!(Nonce, NONCE_LENGTH); byte_array_type!(SharedSecret, SHARED_SECRET_LENGTH); +byte_array_type!(RouteId, ROUTE_ID_LENGTH); diff --git a/veilid-core/src/crypto/types.rs b/veilid-core/src/crypto/types.rs index aeb4dcf1..c3802c43 100644 --- a/veilid-core/src/crypto/types.rs +++ b/veilid-core/src/crypto/types.rs @@ -11,6 +11,7 @@ use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as pub type CryptoKind = FourCC; /// Sort best crypto kinds first +/// Better crypto kinds are 'less', ordered toward the front of a list pub fn compare_crypto_kind(a: &CryptoKind, b: &CryptoKind) -> cmp::Ordering { let a_idx = VALID_CRYPTO_KINDS.iter().position(|k| k == a); let b_idx = VALID_CRYPTO_KINDS.iter().position(|k| k == b); diff --git a/veilid-core/src/routing_table/route_spec_store/mod.rs b/veilid-core/src/routing_table/route_spec_store/mod.rs index d453caa9..545fdaee 100644 --- a/veilid-core/src/routing_table/route_spec_store/mod.rs +++ b/veilid-core/src/routing_table/route_spec_store/mod.rs @@ -1,5 +1,6 @@ use super::*; +mod permutation; mod remote_private_route_info; mod route_set_spec_detail; mod route_spec_store; @@ -7,6 +8,7 @@ mod route_spec_store_cache; mod route_spec_store_content; mod route_stats; +pub use permutation::*; pub use remote_private_route_info::*; pub use route_set_spec_detail::*; pub use route_spec_store::*; @@ -27,9 +29,3 @@ const REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY: TimestampDuration = TimestampDuration:: const ROUTE_MIN_IDLE_TIME_MS: u32 = 30_000; /// The size of the compiled route cache const COMPILED_ROUTE_CACHE_SIZE: usize = 256; - -/// The type of an allocated route set id -pub type RouteSetSpecId = String; - -/// Type type of an imported remote route set id -pub type RemotePrivateRouteId = String; diff --git a/veilid-core/src/routing_table/route_spec_store/permutation.rs b/veilid-core/src/routing_table/route_spec_store/permutation.rs new file mode 100644 index 00000000..9a7f9812 --- /dev/null +++ b/veilid-core/src/routing_table/route_spec_store/permutation.rs @@ -0,0 +1,70 @@ +use super::*; + +/// number of route permutations is the number of unique orderings +/// for a set of nodes, given that the first node is fixed +fn _get_route_permutation_count(hop_count: usize) -> usize { + if hop_count == 0 { + unreachable!(); + } + // a single node or two nodes is always fixed + if hop_count == 1 || hop_count == 2 { + return 1; + } + // more than two nodes has factorial permutation + // hop_count = 3 -> 2! -> 2 + // hop_count = 4 -> 3! -> 6 + (3..hop_count).into_iter().fold(2usize, |acc, x| acc * x) +} +pub type PermReturnType = (Vec, bool); +pub type PermFunc<'t> = Box Option + Send + 't>; + +/// get the route permutation at particular 'perm' index, starting at the 'start' index +/// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum +/// number of permutations is given by get_route_permutation_count() + +pub fn with_route_permutations( + hop_count: usize, + start: usize, + f: &PermFunc, +) -> Option { + if hop_count == 0 { + unreachable!(); + } + // initial permutation + let mut permutation: Vec = Vec::with_capacity(hop_count); + for n in 0..hop_count { + permutation.push(start + n); + } + // if we have one hop or two, then there's only one permutation + if hop_count == 1 || hop_count == 2 { + return f(&permutation); + } + + // heaps algorithm, but skipping the first element + fn heaps_permutation( + permutation: &mut [usize], + size: usize, + f: &PermFunc, + ) -> Option { + if size == 1 { + return f(&permutation); + } + + for i in 0..size { + let out = heaps_permutation(permutation, size - 1, f); + if out.is_some() { + return out; + } + if size % 2 == 1 { + permutation.swap(1, size); + } else { + permutation.swap(1 + i, size); + } + } + + None + } + + // recurse + heaps_permutation(&mut permutation, hop_count - 1, f) +} diff --git a/veilid-core/src/routing_table/route_spec_store/remote_private_route_info.rs b/veilid-core/src/routing_table/route_spec_store/remote_private_route_info.rs index e931b504..1b1f585d 100644 --- a/veilid-core/src/routing_table/route_spec_store/remote_private_route_info.rs +++ b/veilid-core/src/routing_table/route_spec_store/remote_private_route_info.rs @@ -32,6 +32,13 @@ impl RemotePrivateRouteInfo { &mut self.stats } + pub fn has_seen_our_node_info_ts(&mut self, our_node_info_ts: Timestamp) -> bool { + self.last_seen_our_node_info_ts == our_node_info_ts + } + pub fn set_last_seen_our_node_info_ts(&mut self, last_seen_our_node_info_ts: Timestamp) { + self.last_seen_our_node_info_ts = last_seen_our_node_info_ts; + } + // Check to see if this remote private route has expired pub fn did_expire(&self, cur_ts: Timestamp) -> bool { cur_ts.saturating_sub(self.last_touched_ts) >= REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY diff --git a/veilid-core/src/routing_table/route_spec_store/route_set_spec_detail.rs b/veilid-core/src/routing_table/route_spec_store/route_set_spec_detail.rs index 86b1d73d..fd6a39ae 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_set_spec_detail.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_set_spec_detail.rs @@ -49,11 +49,22 @@ impl RouteSetSpecDetail { } tks } + pub fn get_best_route_set_key(&self) -> Option { + self.get_route_set_keys().best().map(|k| k.key) + } + pub fn set_hop_node_refs(&mut self, node_refs: Vec) { + self.hop_node_refs = node_refs; + } pub fn iter_route_set( &self, ) -> alloc::collections::btree_map::Iter { self.route_set.iter() } + pub fn iter_route_set_mut( + &self, + ) -> alloc::collections::btree_map::IterMut { + self.route_set.iter_mut() + } pub fn get_stats(&self) -> &RouteStats { &self.stats } @@ -89,24 +100,4 @@ impl RouteSetSpecDetail { } cache } - - /// Generate a user-facing identifier for this allocated route - pub fn make_id(&self) -> RouteSetSpecId { - let mut idbytes = [0u8; 16]; - for (pk, _) in self.route_set.iter() { - for (i, x) in pk.bytes.iter().enumerate() { - idbytes[i % 16] ^= *x; - } - } - let id = format!( - "{:08x}-{:04x}-{:04x}-{:04x}-{:08x}{:04x}", - u32::from_be_bytes(idbytes[0..4].try_into().expect("32 bits")), - u16::from_be_bytes(idbytes[4..6].try_into().expect("16 bits")), - u16::from_be_bytes(idbytes[6..8].try_into().expect("16 bits")), - u16::from_be_bytes(idbytes[8..10].try_into().expect("16 bits")), - u32::from_be_bytes(idbytes[10..14].try_into().expect("32 bits")), - u16::from_be_bytes(idbytes[14..16].try_into().expect("16 bits")) - ); - id - } } diff --git a/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs index dc4046fd..62adaf85 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs @@ -1,4 +1,5 @@ use super::*; +use permutation::*; #[derive(Debug)] pub struct RouteSpecStoreInner { @@ -33,75 +34,6 @@ pub struct RouteSpecStore { unlocked_inner: Arc, } -/// number of route permutations is the number of unique orderings -/// for a set of nodes, given that the first node is fixed -fn _get_route_permutation_count(hop_count: usize) -> usize { - if hop_count == 0 { - unreachable!(); - } - // a single node or two nodes is always fixed - if hop_count == 1 || hop_count == 2 { - return 1; - } - // more than two nodes has factorial permutation - // hop_count = 3 -> 2! -> 2 - // hop_count = 4 -> 3! -> 6 - (3..hop_count).into_iter().fold(2usize, |acc, x| acc * x) -} -type PermReturnType = (Vec, Vec, bool); -type PermFunc<'t> = Box Option + Send + 't>; - -/// get the route permutation at particular 'perm' index, starting at the 'start' index -/// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum -/// number of permutations is given by get_route_permutation_count() - -fn with_route_permutations( - hop_count: usize, - start: usize, - f: &PermFunc, -) -> Option { - if hop_count == 0 { - unreachable!(); - } - // initial permutation - let mut permutation: Vec = Vec::with_capacity(hop_count); - for n in 0..hop_count { - permutation.push(start + n); - } - // if we have one hop or two, then there's only one permutation - if hop_count == 1 || hop_count == 2 { - return f(&permutation); - } - - // heaps algorithm, but skipping the first element - fn heaps_permutation( - permutation: &mut [usize], - size: usize, - f: &PermFunc, - ) -> Option { - if size == 1 { - return f(&permutation); - } - - for i in 0..size { - let out = heaps_permutation(permutation, size - 1, f); - if out.is_some() { - return out; - } - if size % 2 == 1 { - permutation.swap(1, size); - } else { - permutation.swap(1 + i, size); - } - } - - None - } - - // recurse - heaps_permutation(&mut permutation, hop_count - 1, f) -} - impl RouteSpecStore { pub fn new(routing_table: RoutingTable) -> Self { let config = routing_table.network_manager().config(); @@ -135,69 +67,7 @@ impl RouteSpecStore { }; // Get frozen blob from table store - let table_store = routing_table.network_manager().table_store(); - let rsstdb = table_store.open("RouteSpecStore", 1).await?; - let mut content: RouteSpecStoreContent = - rsstdb.load_rkyv(0, b"content")?.unwrap_or_default(); - - // Look up all route hop noderefs since we can't serialize those - let mut dead_ids = Vec::new(); - for (rsid, rssd) in content.iter_details_mut() { - // Get first route since they all should resolve - let Some((pk, rsd)) = rssd.route_set.first_key_value() else { - dead_ids.push(rsid.clone()); - continue; - }; - // Go through first route - for h in &rsd.hops { - let Some(nr) = routing_table.lookup_node_ref(TypedKey::new(rsd.crypto_kind, *h)) else { - dead_ids.push(rsid.clone()); - break; - }; - rssd.hop_node_refs.push(nr); - } - } - for id in dead_ids { - log_rtab!(debug "no entry, killing off private route: {}", id); - content.remove_detail(&id); - } - - // Load secrets from pstore - let pstore = routing_table.network_manager().protected_store(); - let secret_key_map: HashMap = pstore - .load_user_secret_rkyv("RouteSpecStore") - .await? - .unwrap_or_default(); - - // Ensure we got secret keys for all the public keys - let mut got_secret_key_ids = HashSet::new(); - for (rsid, rssd) in content.iter_details_mut() { - let mut found_all = true; - for (pk, rsd) in &mut rssd.route_set { - if let Some(sk) = secret_key_map.get(pk) { - rsd.secret_key = *sk; - } else { - found_all = false; - break; - } - } - if found_all { - got_secret_key_ids.insert(rsid.clone()); - } - } - - // If we missed any, nuke those route ids - let dead_ids:Vec = content.keys().filter_map(|id| { - if !got_secret_key_ids.contains(id) { - Some(id.clone()) - } else { - None - } - }).collect(); - for id in dead_ids { - log_rtab!(debug "missing secret key, killing off private route: {}", id); - content.remove_detail(&id); - } + let content = RouteSpecStoreContent::load(routing_table.clone()).await?; let mut inner = RouteSpecStoreInner { content, @@ -230,31 +100,8 @@ impl RouteSpecStore { inner.content.clone() }; - // Save all the fields we care about to the frozen blob in table storage - // This skips #[with(Skip)] saving the secret keys, we save them in the protected store instead - let table_store = self - .unlocked_inner - .routing_table - .network_manager() - .table_store(); - let rsstdb = table_store.open("RouteSpecStore", 1).await?; - rsstdb.store_rkyv(0, b"content", &content).await?; - - // // Keep secrets in protected store as well - let pstore = self - .unlocked_inner - .routing_table - .network_manager() - .protected_store(); - - let mut out: HashMap = HashMap::new(); - for (rsid, rssd) in content.iter_details() { - for (pk, rsd) in &rssd.route_set { - out.insert(*pk, rsd.secret_key); - } - } - - let _ = pstore.save_user_secret_rkyv("RouteSpecStore", &out).await?; // ignore if this previously existed or not + // Save our content + content.save(self.unlocked_inner.routing_table.clone()).await?; Ok(()) } @@ -263,7 +110,11 @@ impl RouteSpecStore { pub fn send_route_update(&self) { let (dead_routes, dead_remote_routes) = { let mut inner = self.inner.lock(); - inner.cache.take_dead_routes() + let Some(dr) = inner.cache.take_dead_routes() else { + // Nothing to do + return; + }; + dr }; let update = VeilidUpdate::Route(VeilidStateRoute { @@ -275,8 +126,6 @@ impl RouteSpecStore { update_callback(update); } - - /// Purge the route spec store pub async fn purge(&self) -> EyreResult<()> { { @@ -301,7 +150,7 @@ impl RouteSpecStore { hop_count: usize, directions: DirectionSet, avoid_nodes: &[TypedKey], - ) -> EyreResult> { + ) -> EyreResult> { let inner = &mut *self.inner.lock(); let routing_table = self.unlocked_inner.routing_table.clone(); let rti = &mut *routing_table.inner.write(); @@ -329,7 +178,7 @@ impl RouteSpecStore { hop_count: usize, directions: DirectionSet, avoid_nodes: &[TypedKey], - ) -> EyreResult> { + ) -> EyreResult> { use core::cmp::Ordering; if hop_count < 1 { @@ -625,7 +474,7 @@ impl RouteSpecStore { for start in 0..(nodes.len() - hop_count) { // Try the permutations available starting with 'start' - if let Some((rn, ck, cds)) = with_route_permutations(hop_count, start, &perm_func) { + if let Some((rn, cds)) = with_route_permutations(hop_count, start, &perm_func) { route_nodes = rn; can_do_sequenced = cds; break; @@ -666,11 +515,14 @@ impl RouteSpecStore { drop(perm_func); + // make id + let id = self.generate_allocated_route_id(&rssd)?; + // Add to cache inner.cache.add_to_cache(&rssd); // Keep route in spec store - let id = inner.content.add_detail(rssd); + inner.content.add_detail(id.clone(), rssd); Ok(Some(id)) } @@ -737,23 +589,18 @@ impl RouteSpecStore { } #[instrument(level = "trace", skip(self), ret, err)] - async fn test_allocated_route(&self, id: &String) -> EyreResult { + async fn test_allocated_route(&self, private_route_id: RouteId) -> EyreResult { // Make loopback route to test with let dest = { - // Get best route from set // Match the private route's hop length for safety route length - let (key, hop_count) = { + let hop_count = { let inner = &mut *self.inner.lock(); - let Some(rssd) = inner.content.get_detail(id) else { + let Some(rssd) = inner.content.get_detail(&private_route_id) else { bail!("route id not allocated"); }; - let Some(tkey) = rssd.get_route_set_keys().best() else { - bail!("route does not have best key"); - }; - (tkey.key, rssd.hop_count()) + rssd.hop_count() }; - let private_route = self.assemble_private_route(&key, None)?; // Always test routes with safety routes that are more likely to succeed let stability = Stability::Reliable; @@ -761,7 +608,7 @@ impl RouteSpecStore { let sequencing = Sequencing::NoPreference; let safety_spec = SafetySpec { - preferred_route: Some(key.clone()), + preferred_route: Some(private_route_id), hop_count, stability, sequencing, @@ -769,7 +616,7 @@ impl RouteSpecStore { let safety_selection = SafetySelection::Safe(safety_spec); Destination::PrivateRoute { - private_route, + private_route_id, safety_selection, } }; @@ -788,18 +635,10 @@ impl RouteSpecStore { } #[instrument(level = "trace", skip(self), ret, err)] - async fn test_remote_route(&self, id: &String) -> EyreResult { + async fn test_remote_route(&self, private_route_id: RouteId) -> EyreResult { // Make private route test let dest = { - // Get best remote route from imported set - - // Get the route to test - let private_route = match self.peek_remote_private_route(id) { - Some(pr) => pr, - None => return Ok(false), - }; - // Get a safety route that is good enough let safety_spec = SafetySpec { preferred_route: None, @@ -811,7 +650,7 @@ impl RouteSpecStore { let safety_selection = SafetySelection::Safe(safety_spec); Destination::PrivateRoute { - private_route, + private_route_id, safety_selection, } }; @@ -829,52 +668,44 @@ impl RouteSpecStore { Ok(true) } - /// Test an allocated route for continuity - #[instrument(level = "trace", skip(self), ret, err)] - pub async fn test_route(&self, key: &[TypedKey]) -> EyreResult { - let is_remote = { - let inner = &mut *self.inner.lock(); - let cur_ts = get_aligned_timestamp(); - Self::with_peek_remote_private_route(inner, cur_ts, key, |_| {}).is_some() - }; - if is_remote { - self.test_remote_route(key).await - } else { - self.test_allocated_route(key).await - } - } - /// Release an allocated route that is no longer in use #[instrument(level = "trace", skip(self), ret)] - fn release_allocated_route(&self, id: &String) -> bool { + fn release_allocated_route(&self, id: RouteId) -> bool { let mut inner = self.inner.lock(); - let Some(rssd) = inner.content.remove_detail(id) else { + let Some(rssd) = inner.content.remove_detail(&id) else { return false; }; // Remove from hop cache - if !inner.cache.remove_from_cache(&rssd) { + if !inner.cache.remove_from_cache(id, &rssd) { panic!("hop cache should have contained cache key"); } true } + /// Check if a route id is remote or not + fn is_route_id_remote(&self, id: &RouteId) -> bool { + let inner = &mut *self.inner.lock(); + let cur_ts = get_aligned_timestamp(); + inner.cache.peek_remote_private_route_mut(cur_ts, &id).is_some() + } + + /// Test an allocated route for continuity + #[instrument(level = "trace", skip(self), ret, err)] + pub async fn test_route(&self, id: RouteId) -> EyreResult { + let is_remote = self.is_route_id_remote(&id); + if is_remote { + self.test_remote_route(id).await + } else { + self.test_allocated_route(id).await + } + } + /// Release an allocated or remote route that is no longer in use #[instrument(level = "trace", skip(self), ret)] - pub fn release_route(&self, id: &String) -> bool { - - let is_remote = { - let inner = &mut *self.inner.lock(); - - // Release from compiled route cache if it's used there - self.invalidate_compiled_route_cache(inner, id); - - // Check to see if this is a remote route - let cur_ts = get_aligned_timestamp(); - Self::with_peek_remote_private_route(inner, cur_ts, id, |_| {}).is_some() - }; - + pub fn release_route(&self, id: RouteId) -> bool { + let is_remote = self.is_route_id_remote(&id); if is_remote { self.release_remote_private_route(id) } else { @@ -943,11 +774,11 @@ impl RouteSpecStore { /// List all allocated routes pub fn list_allocated_routes(&self, mut filter: F) -> Vec where - F: FnMut(&PublicKey, &RouteSetSpecDetail) -> Option, + F: FnMut(&RouteId, &RouteSetSpecDetail) -> Option, { let inner = self.inner.lock(); - let mut out = Vec::with_capacity(inner.content.details.len()); - for detail in &inner.content.details { + let mut out = Vec::with_capacity(inner.content.get_detail_count()); + for detail in inner.content.iter_details() { if let Some(x) = filter(detail.0, detail.1) { out.push(x); } @@ -958,11 +789,11 @@ impl RouteSpecStore { /// List all allocated routes pub fn list_remote_routes(&self, mut filter: F) -> Vec where - F: FnMut(&PublicKey, &RemotePrivateRouteInfo) -> Option, + F: FnMut(&RouteId, &RemotePrivateRouteInfo) -> Option, { let inner = self.inner.lock(); - let mut out = Vec::with_capacity(inner.cache.remote_private_route_cache.len()); - for info in &inner.cache.remote_private_route_cache { + let mut out = Vec::with_capacity(inner.cache.get_remote_private_route_count()); + for info in inner.cache.iter_remote_private_routes() { if let Some(x) = filter(info.0, info.1) { out.push(x); } @@ -971,56 +802,20 @@ impl RouteSpecStore { } /// Get the debug description of a route - pub fn debug_route(&self, key: &PublicKey) -> Option { + pub fn debug_route(&self, id: &RouteId) -> Option { let inner = &mut *self.inner.lock(); let cur_ts = get_aligned_timestamp(); - // If this is a remote route, print it - if let Some(s) = - Self::with_peek_remote_private_route(inner, cur_ts, key, |rpi| format!("{:#?}", rpi)) - { - return Some(s); + if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &id) { + return Some(format!("{:#?}", rpri)); } - // Otherwise check allocated routes - Self::detail(inner, key).map(|rsd| format!("{:#?}", rsd)) + if let Some(rssd) = inner.content.get_detail(id) { + return Some(format!("{:#?}", rssd)); + } + None } ////////////////////////////////////////////////////////////////////// - // Route cache - fn add_to_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, pr_pubkey: PublicKey, safety_route: SafetyRoute) - { - let key = CompiledRouteCacheKey { - sr_pubkey: safety_route.public_key, - pr_pubkey, - }; - - if let Some(v) = inner.cache.compiled_route_cache.insert(key, safety_route) { - log_rtab!(error "route cache already contained key: sr_pubkey={:?}, pr_pubkey={:?}", v.public_key, pr_pubkey); - } - } - - fn lookup_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, sr_pubkey: PublicKey, pr_pubkey: PublicKey) -> Option { - - let key = CompiledRouteCacheKey { - sr_pubkey, - pr_pubkey, - }; - - inner.cache.compiled_route_cache.get(&key).cloned() - } - - fn invalidate_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, dead_key: &PublicKey) { - let mut dead_entries = Vec::new(); - for (k, _v) in inner.cache.compiled_route_cache.iter() { - if k.sr_pubkey == *dead_key || k.pr_pubkey == *dead_key { - dead_entries.push(k.clone()); - } - } - for d in dead_entries { - inner.cache.compiled_route_cache.remove(&d); - } - } - /// Compiles a safety route to the private route, with caching /// Returns an Err() if the parameters are wrong /// Returns Ok(None) if no allocation could happen at this time (not an error) @@ -1441,16 +1236,20 @@ impl RouteSpecStore { } /// Import a remote private route for compilation - /// returns a route set id + /// It is safe to import the same route more than once and it will return the same route id + /// Returns a route set id #[instrument(level = "trace", skip(self, blob), ret, err)] - pub fn import_remote_private_route(&self, blob: Vec) -> EyreResult { + pub fn import_remote_private_route(&self, blob: Vec) -> EyreResult { + let cur_ts = get_aligned_timestamp(); // decode the pr blob let private_routes = RouteSpecStore::blob_to_private_routes(self.unlocked_inner.routing_table.crypto(), blob)?; - let inner = &mut *self.inner.lock(); - + // make the route id + let id = self.generate_remote_route_id(&private_routes)?; + // validate the private routes + let inner = &mut *self.inner.lock(); for private_route in private_routes { // ensure private route has first hop @@ -1464,62 +1263,63 @@ impl RouteSpecStore { } } - let cur_ts = get_aligned_timestamp(); - let id = inner.cache.import_remote_private_route(cur_ts, private_routes); + inner.cache.cache_remote_private_route(cur_ts, id, private_routes); Ok(id) } /// Release a remote private route that is no longer in use #[instrument(level = "trace", skip(self), ret)] - fn release_remote_private_route(&self, id: &String) -> bool { + pub fn release_remote_private_route(&self, id: RouteId) -> bool { let inner = &mut *self.inner.lock(); inner.cache.remove_remote_private_route(id) } - /// Retrieve an imported remote private route by its public key - pub fn get_remote_private_route(&self, id: &String) -> Option { + /// Check if a remote private route id is valid + #[instrument(level = "trace", skip(self), ret)] + pub fn is_valid_remote_private_route(&self, id: &RouteId) -> bool { let inner = &mut *self.inner.lock(); let cur_ts = get_aligned_timestamp(); - Self::with_get_remote_private_route(inner, cur_ts, key, |r| { - r.private_route.as_ref().unwrap().clone() - }) + inner.cache.peek_remote_private_route_mut(cur_ts, id).is_some() } - /// Retrieve an imported remote private route by its public key but don't 'touch' it - pub fn peek_remote_private_route(&self, id: &String) -> Option { - xx fix these - let inner = &mut *self.inner.lock(); - let cur_ts = get_aligned_timestamp(); - Self::with_peek_remote_private_route(inner, cur_ts, key, |r| { - r.private_route.as_ref().unwrap().clone() - }) - } + // /// Retrieve an imported remote private route by its public key + // pub fn get_remote_private_route(&self, id: &String) -> Option { + // let inner = &mut *self.inner.lock(); + // let cur_ts = get_aligned_timestamp(); + // Self::with_get_remote_private_route(inner, cur_ts, key, |r| { + // r.private_route.as_ref().unwrap().clone() + // }) + // } + + // /// Retrieve an imported remote private route by its public key but don't 'touch' it + // fn peek_remote_private_route(&self, id: &String) -> Option { + // let inner = &mut *self.inner.lock(); + // let cur_ts = get_aligned_timestamp(); + // inner.cache.with_peek_remote_private_route(cur_ts, id, f) + // Self::with_peek_remote_private_route(inner, cur_ts, key, |r| { + // r.private_route.as_ref().unwrap().clone() + // }) + // } /// Check to see if this remote (not ours) private route has seen our current node info yet /// This happens when you communicate with a private route without a safety route - pub fn has_remote_private_route_seen_our_node_info(&self, key: &PublicKey) -> bool { + pub fn has_remote_private_route_seen_our_node_info(&self, id: &RouteId) -> bool { let our_node_info_ts = { let rti = &*self.unlocked_inner.routing_table.inner.read(); let Some(ts) = rti.get_own_node_info_ts(RoutingDomain::PublicInternet) else { + // Node info is invalid, skip this return false; }; ts }; - let opt_rpr_node_info_ts = { - let inner = &mut *self.inner.lock(); - let cur_ts = get_aligned_timestamp(); - Self::with_peek_remote_private_route(inner, cur_ts, key, |rpr| { - rpr.last_seen_our_node_info_ts - }) - }; - - let Some(rpr_node_info_ts) = opt_rpr_node_info_ts else { + let inner = &mut *self.inner.lock(); + let cur_ts = get_aligned_timestamp(); + let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &id) else { return false; }; - - our_node_info_ts == rpr_node_info_ts + rpri.has_seen_our_node_info_ts(our_node_info_ts) } /// Mark a remote private route as having seen our current node info @@ -1543,20 +1343,23 @@ impl RouteSpecStore { }; let inner = &mut *self.inner.lock(); + // Check for local route. If this is not a remote private route // then we just skip the recording. We may be running a test and using // our own local route as the destination private route. - if let Some(_) = Self::detail_mut(inner, key) { + if let Some(_) = inner.content.get_id_by_key(key) { return Ok(()); } - if Self::with_get_remote_private_route(inner, cur_ts, key, |rpr| { - rpr.last_seen_our_node_info_ts = our_node_info_ts; - }) - .is_none() - { - bail!("private route is missing from store: {}", key); + + if let Some(rrid) = inner.cache.get_remote_private_route_id_by_key(key) { + if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &rrid) + { + rpri.set_last_seen_our_node_info_ts(our_node_info_ts); + return Ok(()); + } } - Ok(()) + + bail!("private route is missing from store: {}", key); } /// Get the route statistics for any route we know about, local or remote @@ -1570,15 +1373,20 @@ impl RouteSpecStore { if self.unlocked_inner.routing_table.matches_own_node_id_key(key) { return None; } + // Check for local route - if let Some(rsd) = Self::detail_mut(inner, key) { - return Some(f(&mut rsd.stats)); + if let Some(rsid) = inner.content.get_id_by_key(key) { + if let Some(rsd) = inner.content.get_detail_mut(&rsid) { + return Some(f(rsd.get_stats_mut())); + } } + // Check for remote route - if let Some(res) = - Self::with_peek_remote_private_route(inner, cur_ts, key, |rpr| f(&mut rpr.stats)) - { - return Some(res); + if let Some(rrid) = inner.cache.get_remote_private_route_id_by_key(key) { + if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &rrid) + { + return Some(f(rpri.get_stats_mut())); + } } None @@ -1613,13 +1421,10 @@ impl RouteSpecStore { let inner = &mut *self.inner.lock(); // Roll transfers for locally allocated routes - for rssd in inner.content.details.values_mut() { - rssd.stats.roll_transfers(last_ts, cur_ts); - } + inner.content.roll_transfers(last_ts, cur_ts); + // Roll transfers for remote private routes - for (_k, v) in inner.cache.remote_private_route_cache.iter_mut() { - v.stats.roll_transfers(last_ts, cur_ts); - } + inner.cache.roll_transfers(last_ts, cur_ts); } /// Convert private route list to binary blob @@ -1681,6 +1486,54 @@ impl RouteSpecStore { out.push(private_route); } + // Don't trust the order of the blob + out.sort_by(|a,b| { + a.public_key.cmp(&b.public_key) + }); + Ok(out) } + + /// Generate RouteId from typed key set of route public keys + fn generate_allocated_route_id(&self, rssd: &RouteSetSpecDetail) -> EyreResult { + let route_set_keys = rssd.get_route_set_keys(); + let crypto = self.unlocked_inner.routing_table.crypto(); + + let mut idbytes = Vec::with_capacity(PUBLIC_KEY_LENGTH * route_set_keys.len()); + let mut best_kind : Option = None; + for tk in route_set_keys.iter() { + if best_kind.is_none() || compare_crypto_kind(&tk.kind, best_kind.as_ref().unwrap()) == cmp::Ordering::Less { + best_kind = Some(tk.kind); + } + idbytes.extend_from_slice(&tk.key.bytes); + } + let Some(best_kind) = best_kind else { + bail!("no compatible crypto kinds in route"); + }; + let vcrypto = crypto.get(best_kind).unwrap(); + + Ok(RouteId::new(vcrypto.generate_hash(&idbytes).bytes)) + + } + + /// Generate RouteId from set of private routes + fn generate_remote_route_id(&self, private_routes: &[PrivateRoute]) -> EyreResult { + let crypto = self.unlocked_inner.routing_table.crypto(); + + let mut idbytes = Vec::with_capacity(PUBLIC_KEY_LENGTH * private_routes.len()); + let mut best_kind : Option = None; + for private_route in private_routes { + if best_kind.is_none() || compare_crypto_kind(&private_route.public_key.kind, best_kind.as_ref().unwrap()) == cmp::Ordering::Less { + best_kind = Some(private_route.public_key.kind); + } + idbytes.extend_from_slice(&private_route.public_key.key.bytes); + } + let Some(best_kind) = best_kind else { + bail!("no compatible crypto kinds in route"); + }; + let vcrypto = crypto.get(best_kind).unwrap(); + + Ok(RouteId::new(vcrypto.generate_hash(&idbytes).bytes)) + } + } diff --git a/veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs b/veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs index 1abe6e8a..74b47291 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs @@ -28,15 +28,15 @@ pub struct RouteSpecStoreCache { /// Route spec hop cache, used to quickly disqualify routes hop_cache: HashSet>, /// Remote private routes we've imported and statistics - remote_private_route_set_cache: LruCache, + remote_private_route_set_cache: LruCache, /// Remote private routes indexed by public key - remote_private_routes_by_key: HashMap, + remote_private_routes_by_key: HashMap, /// Compiled route cache compiled_route_cache: LruCache, /// List of dead allocated routes - dead_routes: Vec, + dead_routes: Vec, /// List of dead remote routes - dead_remote_routes: Vec, + dead_remote_routes: Vec, } impl RouteSpecStoreCache { @@ -66,7 +66,7 @@ impl RouteSpecStoreCache { } /// removes an allocated route set from our cache - pub fn remove_from_cache(&mut self, rssd: &RouteSetSpecDetail) -> bool { + pub fn remove_from_cache(&mut self, id: RouteId, rssd: &RouteSetSpecDetail) -> bool { let cache_key = rssd.make_cache_key(); // Remove from hop cache @@ -100,10 +100,13 @@ impl RouteSpecStoreCache { panic!("used_end_nodes cache should have contained hop"); } } + + // Invalidate compiled route cache + self.invalidate_compiled_route_cache(pk); } // Mark it as dead for the update - self.dead_routes.push(rssd.make_id()); + self.dead_routes.push(id); true } @@ -122,43 +125,27 @@ impl RouteSpecStoreCache { }) } - /// generate unique remote private route set id for a remote private route set - fn make_remote_private_route_id(private_routes: &[PrivateRoute]) -> String { - let mut idbytes = [0u8; 16]; - for (pk, _) in &rprinfo.private_routes { - for (i, x) in pk.bytes.iter().enumerate() { - idbytes[i % 16] ^= *x; - } - } - let id = format!( - "{:08x}-{:04x}-{:04x}-{:04x}-{:08x}{:04x}", - u32::from_be_bytes(idbytes[0..4].try_into().expect("32 bits")), - u16::from_be_bytes(idbytes[4..6].try_into().expect("16 bits")), - u16::from_be_bytes(idbytes[6..8].try_into().expect("16 bits")), - u16::from_be_bytes(idbytes[8..10].try_into().expect("16 bits")), - u32::from_be_bytes(idbytes[10..14].try_into().expect("32 bits")), - u16::from_be_bytes(idbytes[14..16].try_into().expect("16 bits")) - ); - id - } - /// add remote private route to caches /// returns a remote private route set id fn add_remote_private_route( &mut self, + id: RouteId, rprinfo: RemotePrivateRouteInfo, - ) -> RemotePrivateRouteId { - let id = Self::make_remote_private_route_id(rprinfo.get_private_routes()); - + ) -> RouteId { // also store in id by key table - for (pk, _) in rprinfo.get_private_routes() { - self.remote_private_routes_by_key.insert(*pk, id.clone()); + for private_route in rprinfo.get_private_routes() { + self.remote_private_routes_by_key + .insert(private_route.public_key.key, id.clone()); } self.remote_private_route_set_cache - .insert(id.clone(), rprinfo, |dead_id, dead_rpri| { + .insert(id, rprinfo, |dead_id, dead_rpri| { // If anything LRUs out, remove from the by-key table - for (dead_pk, _) in dead_rpri.get_private_routes() { - self.remote_private_routes_by_key.remove(&dead_pk).unwrap(); + // Follow the same logic as 'remove_remote_private_route' here + for dead_private_route in dead_rpri.get_private_routes() { + self.remote_private_routes_by_key + .remove(&dead_private_route.public_key.key) + .unwrap(); + self.invalidate_compiled_route_cache(&dead_private_route.public_key.key); } self.dead_remote_routes.push(dead_id); }); @@ -166,33 +153,68 @@ impl RouteSpecStoreCache { id } + /// get count of remote private routes in cache + pub fn get_remote_private_route_count(&self) -> usize { + self.remote_private_route_set_cache.len() + } + + /// iterate all of the remote private routes we have in the cache + pub fn iter_remote_private_routes( + &self, + ) -> hashlink::linked_hash_map::Iter { + self.remote_private_route_set_cache.iter() + } + /// remote private route cache accessor - fn get_remote_private_route( + /// will LRU entries and may expire entries and not return them if they are stale + pub fn get_remote_private_route( &mut self, - id: &RemotePrivateRouteId, + cur_ts: Timestamp, + id: &RouteId, ) -> Option<&RemotePrivateRouteInfo> { - self.remote_private_route_set_cache.get(id) + if let Some(rpri) = self.remote_private_route_set_cache.get(id) { + if !rpri.did_expire(cur_ts) { + rpri.touch(cur_ts); + return Some(rpri); + } + } + None } + /// mutable remote private route cache accessor - fn get_remote_private_route_mut( + /// will LRU entries and may expire entries and not return them if they are stale + pub fn get_remote_private_route_mut( &mut self, - id: &RemotePrivateRouteId, + cur_ts: Timestamp, + id: &RouteId, ) -> Option<&mut RemotePrivateRouteInfo> { - self.remote_private_route_set_cache.get_mut(id) + if let Some(rpri) = self.remote_private_route_set_cache.get_mut(id) { + if !rpri.did_expire(cur_ts) { + rpri.touch(cur_ts); + return Some(rpri); + } + } + None } + /// mutable remote private route cache accessor without lru action - fn peek_remote_private_route_mut( + /// will not LRU entries but may expire entries and not return them if they are stale + pub fn peek_remote_private_route_mut( &mut self, - id: &RemotePrivateRouteId, + cur_ts: Timestamp, + id: &RouteId, ) -> Option<&mut RemotePrivateRouteInfo> { - self.remote_private_route_set_cache.peek_mut(id) + if let Some(rpri) = self.remote_private_route_set_cache.peek_mut(id) { + if !rpri.did_expire(cur_ts) { + rpri.touch(cur_ts); + return Some(rpri); + } + } + None } /// look up a remote private route id by one of the route public keys - pub fn get_remote_private_route_id_by_key( - &self, - key: &PublicKey, - ) -> Option { + pub fn get_remote_private_route_id_by_key(&self, key: &PublicKey) -> Option { self.remote_private_routes_by_key.get(key).cloned() } @@ -200,14 +222,14 @@ impl RouteSpecStoreCache { /// may LRU and/or expire other cache entries to make room for the new one /// or update an existing entry with the same private route set /// returns the route set id - pub fn import_remote_private_route( + pub fn cache_remote_private_route( &mut self, cur_ts: Timestamp, + id: RouteId, private_routes: Vec, - ) -> RemotePrivateRouteId { + ) { // get id for this route set - let id = RouteSpecStoreCache::make_remote_private_route_id(&private_routes); - let rpri = if let Some(rpri) = self.get_remote_private_route_mut(&id) { + if let Some(rpri) = self.get_remote_private_route_mut(cur_ts, &id) { if rpri.did_expire(cur_ts) { // Start fresh if this had expired rpri.unexpire(cur_ts); @@ -223,88 +245,95 @@ impl RouteSpecStoreCache { last_touched_ts: cur_ts, stats: RouteStats::new(cur_ts), }; - let new_id = self.add_remote_private_route(rpri); - assert_eq!(id, new_id); - if self.get_remote_private_route_mut(&id).is_none() { - bail!("remote private route should exist"); + self.add_remote_private_route(id, rpri); + if self.peek_remote_private_route_mut(cur_ts, &id).is_none() { + panic!("remote private route should exist"); }; }; - id } /// remove a remote private route from the cache - pub fn remove_remote_private_route(&mut self, id: &RemotePrivateRouteId) -> bool { - let Some(rprinfo) = self.remote_private_route_set_cache.remove(id) else { + pub fn remove_remote_private_route(&mut self, id: RouteId) -> bool { + let Some(rprinfo) = self.remote_private_route_set_cache.remove(&id) else { return false; }; - for (pk, _) in rprinfo.get_private_routes() { - self.remote_private_routes_by_key.remove(&pk).unwrap(); + for private_route in rprinfo.get_private_routes() { + self.remote_private_routes_by_key + .remove(&private_route.public_key.key) + .unwrap(); + self.invalidate_compiled_route_cache(&private_route.public_key.key); } - self.dead_remote_routes.push(id.clone()); + self.dead_remote_routes.push(id); true } - /// get an existing remote private route cache entry - /// will LRU entries and may expire entries and not return them if they are stale - /// calls a callback with the remote private route info if returned - pub fn with_get_remote_private_route( - &mut self, - cur_ts: Timestamp, - id: &RemotePrivateRouteId, - f: F, - ) -> Option - where - F: FnOnce(&mut RemotePrivateRouteInfo) -> R, - { - if let Some(rpri) = self.get_remote_private_route_mut(&id) { - if !rpri.did_expire(cur_ts) { - rpri.touch(cur_ts); - return Some(f(rpri)); - } + /// Stores a compiled 'safety + private' route so we don't have to compile it again later + pub fn add_to_compiled_route_cache(&self, pr_pubkey: PublicKey, safety_route: SafetyRoute) { + let key = CompiledRouteCacheKey { + sr_pubkey: safety_route.public_key.key, + pr_pubkey, + }; + + if let Some(v) = self + .compiled_route_cache + .insert(key, safety_route, |_k, _v| { + // Do nothing on LRU evict + }) + { + log_rtab!(error "route cache already contained key: sr_pubkey={:?}, pr_pubkey={:?}", v.public_key, pr_pubkey); } - self.remove_remote_private_route(&id); - None } - // peek a remote private route cache entry - // will not LRU entries but may expire entries and not return them if they are stale - /// calls a callback with the remote private route info if returned - pub fn with_peek_remote_private_route( - &mut self, - cur_ts: Timestamp, - id: &RemotePrivateRouteId, - f: F, - ) -> Option - where - F: FnOnce(&mut RemotePrivateRouteInfo) -> R, - { - if let Some(rpri) = self.peek_remote_private_route_mut(&id) { - if !rpri.did_expire(cur_ts) { - rpri.touch(cur_ts); - return Some(f(rpri)); + /// Looks up an existing compiled route from the safety and private route components + pub fn lookup_compiled_route_cache( + &self, + sr_pubkey: PublicKey, + pr_pubkey: PublicKey, + ) -> Option { + let key = CompiledRouteCacheKey { + sr_pubkey, + pr_pubkey, + }; + self.compiled_route_cache.get(&key).cloned() + } + + /// When routes are dropped, they should be removed from the compiled route cache + fn invalidate_compiled_route_cache(&self, dead_key: &PublicKey) { + let mut dead_entries = Vec::new(); + for (k, _v) in self.compiled_route_cache.iter() { + if k.sr_pubkey == *dead_key || k.pr_pubkey == *dead_key { + dead_entries.push(k.clone()); } } - self.remove_remote_private_route(&id); - None + for d in dead_entries { + self.compiled_route_cache.remove(&d); + } } /// Take the dead local and remote routes so we can update clients - pub fn take_dead_routes(&mut self) -> (Vec, Vec) { + pub fn take_dead_routes(&mut self) -> Option<(Vec, Vec)> { if self.dead_routes.is_empty() && self.dead_remote_routes.is_empty() { // Nothing to do - return; + return None; } let dead_routes = core::mem::take(&mut self.dead_routes); let dead_remote_routes = core::mem::take(&mut self.dead_remote_routes); - (dead_routes, dead_remote_routes) + Some((dead_routes, dead_remote_routes)) } /// Clean up imported remote routes /// Resets statistics for when our node info changes - pub fn reset_details(&mut self) { - for (_k, v) in self.remote_private_route_cache { - // Restart stats for routes so we test the route again - v.stats.reset(); + pub fn reset_remote_private_routes(&mut self) { + // Restart stats for routes so we test the route again + for (_k, v) in self.remote_private_route_set_cache { + v.get_stats_mut().reset(); + } + } + + /// Roll transfer statistics + pub fn roll_transfers(&mut self, last_ts: Timestamp, cur_ts: Timestamp) { + for (_k, v) in self.remote_private_route_set_cache { + v.get_stats_mut().roll_transfers(last_ts, cur_ts); } } } diff --git a/veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs b/veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs index 7a8688b8..99049a05 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs @@ -5,15 +5,116 @@ use super::*; #[archive_attr(repr(C, align(8)), derive(CheckBytes))] pub struct RouteSpecStoreContent { /// All of the route sets we have allocated so far indexed by key - id_by_key: HashMap, + id_by_key: HashMap, /// All of the route sets we have allocated so far - details: HashMap, + details: HashMap, } impl RouteSpecStoreContent { - pub fn add_detail(&mut self, detail: RouteSetSpecDetail) -> RouteSetSpecId { - // generate unique key string - let id = detail.make_id(); + pub async fn load(routing_table: RoutingTable) -> EyreResult { + // Deserialize what we can + let table_store = routing_table.network_manager().table_store(); + let rsstdb = table_store.open("RouteSpecStore", 1).await?; + let mut content: RouteSpecStoreContent = + rsstdb.load_rkyv(0, b"content")?.unwrap_or_default(); + + // Look up all route hop noderefs since we can't serialize those + let mut dead_ids = Vec::new(); + for (rsid, rssd) in content.details.iter_mut() { + // Get best route since they all should resolve + let Some(pk) = rssd.get_best_route_set_key() else { + dead_ids.push(rsid.clone()); + continue; + }; + let Some(rsd) = rssd.get_route_by_key(&pk) else { + dead_ids.push(rsid.clone()); + continue; + }; + // Go through best route and resolve noderefs + let mut hop_node_refs = Vec::with_capacity(rsd.hops.len()); + for h in &rsd.hops { + let Some(nr) = routing_table.lookup_node_ref(TypedKey::new(rsd.crypto_kind, *h)) else { + dead_ids.push(rsid.clone()); + break; + }; + hop_node_refs.push(nr); + } + + // Apply noderefs + rssd.set_hop_node_refs(hop_node_refs); + } + for id in dead_ids { + log_rtab!(debug "no entry, killing off private route: {}", id); + content.remove_detail(&id); + } + + // Load secrets from pstore + let pstore = routing_table.network_manager().protected_store(); + let secret_key_map: HashMap = pstore + .load_user_secret_rkyv("RouteSpecStore") + .await? + .unwrap_or_default(); + + // Ensure we got secret keys for all the public keys + let mut got_secret_key_ids = HashSet::new(); + for (rsid, rssd) in content.details.iter_mut() { + let mut found_all = true; + for (pk, rsd) in rssd.iter_route_set_mut() { + if let Some(sk) = secret_key_map.get(pk) { + rsd.secret_key = *sk; + } else { + found_all = false; + break; + } + } + if found_all { + got_secret_key_ids.insert(rsid.clone()); + } + } + + // If we missed any, nuke those route ids + let dead_ids: Vec = content + .details + .keys() + .filter_map(|id| { + if !got_secret_key_ids.contains(id) { + Some(*id) + } else { + None + } + }) + .collect(); + for id in dead_ids { + log_rtab!(debug "missing secret key, killing off private route: {}", id); + content.remove_detail(&id); + } + + Ok(content) + } + + pub async fn save(&self, routing_table: RoutingTable) -> EyreResult<()> { + // Save all the fields we care about to the frozen blob in table storage + // This skips #[with(Skip)] saving the secret keys, we save them in the protected store instead + let table_store = routing_table.network_manager().table_store(); + let rsstdb = table_store.open("RouteSpecStore", 1).await?; + rsstdb.store_rkyv(0, b"content", self).await?; + + // // Keep secrets in protected store as well + let pstore = routing_table.network_manager().protected_store(); + + let mut out: HashMap = HashMap::new(); + for (rsid, rssd) in self.details.iter() { + for (pk, rsd) in rssd.iter_route_set() { + out.insert(*pk, rsd.secret_key); + } + } + + let _ = pstore.save_user_secret_rkyv("RouteSpecStore", &out).await?; // ignore if this previously existed or not + + Ok(()) + } + + pub fn add_detail(&mut self, id: RouteId, detail: RouteSetSpecDetail) { assert!(!self.details.contains_key(&id)); // also store in id by key table @@ -21,38 +122,32 @@ impl RouteSpecStoreContent { self.id_by_key.insert(*pk, id.clone()); } self.details.insert(id.clone(), detail); - - id } - pub fn remove_detail(&mut self, id: &RouteSetSpecId) -> Option { + pub fn remove_detail(&mut self, id: &RouteId) -> Option { let detail = self.details.remove(id)?; for (pk, _) in detail.iter_route_set() { self.id_by_key.remove(&pk).unwrap(); } Some(detail) } - pub fn get_detail(&self, id: &RouteSetSpecId) -> Option<&RouteSetSpecDetail> { + pub fn get_detail_count(&self) -> usize { + self.details.len() + } + pub fn get_detail(&self, id: &RouteId) -> Option<&RouteSetSpecDetail> { self.details.get(id) } - pub fn get_detail_mut(&mut self, id: &RouteSetSpecId) -> Option<&mut RouteSetSpecDetail> { + pub fn get_detail_mut(&mut self, id: &RouteId) -> Option<&mut RouteSetSpecDetail> { self.details.get_mut(id) } - pub fn get_id_by_key(&self, key: &PublicKey) -> Option { + pub fn get_id_by_key(&self, key: &PublicKey) -> Option { self.id_by_key.get(key).cloned() } - pub fn iter_ids(&self) -> std::collections::hash_map::Keys { + pub fn iter_ids(&self) -> std::collections::hash_map::Keys { self.details.keys() } - pub fn iter_details( - &self, - ) -> std::collections::hash_map::Iter { + pub fn iter_details(&self) -> std::collections::hash_map::Iter { self.details.iter() } - pub fn iter_details_mut( - &mut self, - ) -> std::collections::hash_map::IterMut { - self.details.iter_mut() - } /// Clean up local allocated routes /// Resets publication status and statistics for when our node info changes @@ -65,4 +160,11 @@ impl RouteSpecStoreContent { v.get_stats_mut().reset(); } } + + /// Roll transfer statistics + pub fn roll_transfers(&mut self, last_ts: Timestamp, cur_ts: Timestamp) { + for rssd in self.details.values_mut() { + rssd.get_stats_mut().roll_transfers(last_ts, cur_ts); + } + } } diff --git a/veilid-core/src/routing_table/tasks/private_route_management.rs b/veilid-core/src/routing_table/tasks/private_route_management.rs index c6ec9b65..3f44a529 100644 --- a/veilid-core/src/routing_table/tasks/private_route_management.rs +++ b/veilid-core/src/routing_table/tasks/private_route_management.rs @@ -8,7 +8,7 @@ const BACKGROUND_SAFETY_ROUTE_COUNT: usize = 2; impl RoutingTable { /// Fastest routes sort - fn route_sort_latency_fn(a: &(TypedKey, u64), b: &(TypedKey, u64)) -> cmp::Ordering { + fn route_sort_latency_fn(a: &(RouteId, u64), b: &(RouteId, u64)) -> cmp::Ordering { let mut al = a.1; let mut bl = b.1; // Treat zero latency as uncalculated @@ -35,14 +35,14 @@ impl RoutingTable { /// /// If a route doesn't 'need_testing', then we neither test nor drop it #[instrument(level = "trace", skip(self))] - fn get_allocated_routes_to_test(&self, cur_ts: Timestamp) -> Vec { + fn get_allocated_routes_to_test(&self, cur_ts: Timestamp) -> Vec { let default_route_hop_count = self.with_config(|c| c.network.rpc.default_route_hop_count as usize); let rss = self.route_spec_store(); - let mut must_test_routes = Vec::::new(); - let mut unpublished_routes = Vec::<(TypedKey, u64)>::new(); - let mut expired_routes = Vec::::new(); + let mut must_test_routes = Vec::::new(); + let mut unpublished_routes = Vec::<(RouteId, u64)>::new(); + let mut expired_routes = Vec::::new(); rss.list_allocated_routes(|k, v| { let stats = v.get_stats(); // Ignore nodes that don't need testing @@ -81,7 +81,7 @@ impl RoutingTable { } // Process dead routes - for r in &expired_routes { + for r in expired_routes { log_rtab!(debug "Expired route: {}", r); rss.release_route(r); } @@ -95,7 +95,7 @@ impl RoutingTable { async fn test_route_set( &self, stop_token: StopToken, - routes_needing_testing: Vec, + routes_needing_testing: Vec, ) -> EyreResult<()> { if routes_needing_testing.is_empty() { return Ok(()); @@ -107,43 +107,45 @@ impl RoutingTable { #[derive(Default, Debug)] struct TestRouteContext { failed: bool, - dead_routes: Vec, + dead_routes: Vec, } - let mut unord = FuturesUnordered::new(); let ctx = Arc::new(Mutex::new(TestRouteContext::default())); - for r in routes_needing_testing { - let rss = rss.clone(); - let ctx = ctx.clone(); - unord.push( - async move { - let success = match rss.test_route(&r).await { - Ok(v) => v, - Err(e) => { - log_rtab!(error "Test route failed: {}", e); - ctx.lock().failed = true; + { + let mut unord = FuturesUnordered::new(); + for r in routes_needing_testing { + let rss = rss.clone(); + let ctx = ctx.clone(); + unord.push( + async move { + let success = match rss.test_route(r).await { + Ok(v) => v, + Err(e) => { + log_rtab!(error "Test route failed: {}", e); + ctx.lock().failed = true; + return; + } + }; + if success { + // Route is okay, leave it alone return; } - }; - if success { - // Route is okay, leave it alone - return; + // Route test failed + ctx.lock().dead_routes.push(r); } - // Route test failed - ctx.lock().dead_routes.push(r); - } - .instrument(Span::current()) - .boxed(), - ); + .instrument(Span::current()) + .boxed(), + ); + } + + // Wait for test_route futures to complete in parallel + while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {} } - // Wait for test_route futures to complete in parallel - while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {} - // Process failed routes - let ctx = &mut *ctx.lock(); - for r in &ctx.dead_routes { - log_rtab!(debug "Dead route failed to test: {}", &r); + let ctx = Arc::try_unwrap(ctx).unwrap().into_inner(); + for r in ctx.dead_routes { + log_rtab!(debug "Dead route failed to test: {}", r); rss.release_route(r); } @@ -176,13 +178,16 @@ impl RoutingTable { .await?; } - // Ensure we have a minimum of N allocated local, unpublished routes with the default number of hops + // Ensure we have a minimum of N allocated local, unpublished routes with the default number of hops and all our supported crypto kinds let default_route_hop_count = self.with_config(|c| c.network.rpc.default_route_hop_count as usize); let mut local_unpublished_route_count = 0usize; let rss = self.route_spec_store(); rss.list_allocated_routes(|_k, v| { - if !v.is_published() && v.hop_count() == default_route_hop_count { + if !v.is_published() + && v.hop_count() == default_route_hop_count + && v.get_route_set_keys().kinds() == VALID_CRYPTO_KINDS + { local_unpublished_route_count += 1; } Option::<()>::None @@ -196,6 +201,7 @@ impl RoutingTable { // Parameters here must be the default safety route spec // These will be used by test_remote_route as well if let Some(k) = rss.allocate_route( + &VALID_CRYPTO_KINDS, Stability::default(), Sequencing::default(), default_route_hop_count, diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index af9cbd17..5321b17d 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -22,7 +22,7 @@ pub enum Destination { /// Send to private route (privateroute) PrivateRoute { /// A private route set id to send to - private_route: String, + private_route_id: RouteId, /// Require safety route or not safety_selection: SafetySelection, }, @@ -44,9 +44,9 @@ impl Destination { safety_selection: SafetySelection::Unsafe(sequencing), } } - pub fn private_route(private_route: PrivateRoute, safety_selection: SafetySelection) -> Self { + pub fn private_route(private_route_id: RouteId, safety_selection: SafetySelection) -> Self { Self::PrivateRoute { - private_route, + private_route_id, safety_selection, } } @@ -70,10 +70,10 @@ impl Destination { safety_selection, }, Destination::PrivateRoute { - private_route, + private_route_id, safety_selection: _, } => Self::PrivateRoute { - private_route, + private_route_id, safety_selection, }, } @@ -91,7 +91,7 @@ impl Destination { safety_selection, } => safety_selection, Destination::PrivateRoute { - private_route: _, + private_route_id: _, safety_selection, } => safety_selection, } @@ -127,7 +127,7 @@ impl fmt::Display for Destination { write!(f, "{}@{}{}", target, relay, sr) } Destination::PrivateRoute { - private_route, + private_route_id, safety_selection, } => { let sr = if matches!(safety_selection, SafetySelection::Safe(_)) { @@ -136,7 +136,7 @@ impl fmt::Display for Destination { "" }; - write!(f, "{}{}", private_route, sr) + write!(f, "{}{}", private_route_id, sr) } } } @@ -207,7 +207,7 @@ impl RPCProcessor { } }, Destination::PrivateRoute { - private_route, + private_route_id, safety_selection, } => { let Some(avoid_node_id) = private_route.first_hop_node_id() else { diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 561c648e..254b9d90 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -674,7 +674,7 @@ impl RPCProcessor { }; } Destination::PrivateRoute { - private_route, + private_route_id, safety_selection, } => { // Send to private route @@ -726,7 +726,7 @@ impl RPCProcessor { } } Destination::PrivateRoute { - private_route: _, + private_route_id: _, safety_selection: _, } => { return Ok(SenderPeerInfo::default()); diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index a3a5af6e..e7f834a7 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -17,7 +17,7 @@ impl RPCProcessor { if matches!( dest, Destination::PrivateRoute { - private_route: _, + private_route_id: _, safety_selection: _ } ) { diff --git a/veilid-core/src/rpc_processor/rpc_signal.rs b/veilid-core/src/rpc_processor/rpc_signal.rs index d717b395..674a85ff 100644 --- a/veilid-core/src/rpc_processor/rpc_signal.rs +++ b/veilid-core/src/rpc_processor/rpc_signal.rs @@ -13,7 +13,7 @@ impl RPCProcessor { if matches!( dest, Destination::PrivateRoute { - private_route: _, + private_route_id: _, safety_selection: _ } ) { diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 2335100a..6caa40f7 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -53,7 +53,7 @@ impl RPCProcessor { (Some(target.clone()), routing_domain) } Destination::PrivateRoute { - private_route: _, + private_route_id: _, safety_selection: _, } => (None, RoutingDomain::PublicInternet), }; @@ -169,7 +169,7 @@ impl RPCProcessor { safety_selection: _, } | Destination::PrivateRoute { - private_route: _, + private_route_id: _, safety_selection: _, } => { // sender info is irrelevant over relays and routes diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index 79441c1b..66a03f44 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -4,8 +4,8 @@ use super::*; #[derive(Clone, Debug)] pub enum Target { - NodeId(PublicKey), // Node by any of its public keys - PrivateRoute(String), // Private route by its route set id + NodeId(PublicKey), // Node by any of its public keys + PrivateRoute(RouteId), // Remote private route by its id } pub struct RoutingContextInner {} @@ -121,14 +121,13 @@ impl RoutingContext { Target::PrivateRoute(rsid) => { // Get remote private route let rss = self.api.routing_table()?.route_spec_store(); - let Some(private_route) = rss - .get_remote_private_route(&rsid) + if !rss.is_valid_remote_private_route(&rsid) else { apibail_key_not_found!(pr); }; Ok(rpc_processor::Destination::PrivateRoute { - private_route: rsid, + private_route_id: rsid, safety_selection: self.unlocked_inner.safety_selection, }) } diff --git a/veilid-core/src/veilid_api/types.rs b/veilid-core/src/veilid_api/types.rs index 7374cda8..c10d217e 100644 --- a/veilid-core/src/veilid_api/types.rs +++ b/veilid-core/src/veilid_api/types.rs @@ -279,8 +279,8 @@ pub struct VeilidStateNetwork { )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidStateRoute { - pub dead_routes: Vec, - pub dead_remote_routes: Vec, + pub dead_routes: Vec, + pub dead_remote_routes: Vec, } #[derive( @@ -513,7 +513,7 @@ impl SafetySelection { #[archive_attr(repr(C), derive(CheckBytes))] pub struct SafetySpec { /// preferred safety route set id if it still exists - pub preferred_route: Option, + pub preferred_route: Option, /// must be greater than 0 pub hop_count: usize, /// prefer reliability over speed