more refactor

veilid-core/src/crypto/byte_array_types.rs

@@ -41,6 +41,9 @@ pub const SHARED_SECRET_LENGTH: usize = 32;
 /// Length of a shared secret in bytes after encoding to base64url
 #[allow(dead_code)]
 pub const SHARED_SECRET_LENGTH_ENCODED: usize = 43;
+/// Length of a route id in bytes
+#[allow(dead_code)]
+pub const ROUTE_ID_LENGTH: usize = 32;
 
 //////////////////////////////////////////////////////////////////////
 
@@ -260,3 +263,4 @@ byte_array_type!(Signature, SIGNATURE_LENGTH);
 byte_array_type!(PublicKeyDistance, PUBLIC_KEY_LENGTH);
 byte_array_type!(Nonce, NONCE_LENGTH);
 byte_array_type!(SharedSecret, SHARED_SECRET_LENGTH);
+byte_array_type!(RouteId, ROUTE_ID_LENGTH);

veilid-core/src/crypto/types.rs

@@ -11,6 +11,7 @@ use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as
 pub type CryptoKind = FourCC;
 
 /// Sort best crypto kinds first
+/// Better crypto kinds are 'less', ordered toward the front of a list
 pub fn compare_crypto_kind(a: &CryptoKind, b: &CryptoKind) -> cmp::Ordering {
     let a_idx = VALID_CRYPTO_KINDS.iter().position(|k| k == a);
     let b_idx = VALID_CRYPTO_KINDS.iter().position(|k| k == b);

veilid-core/src/routing_table/route_spec_store/mod.rs

@@ -1,5 +1,6 @@
 use super::*;
 
+mod permutation;
 mod remote_private_route_info;
 mod route_set_spec_detail;
 mod route_spec_store;
@@ -7,6 +8,7 @@ mod route_spec_store_cache;
 mod route_spec_store_content;
 mod route_stats;
 
+pub use permutation::*;
 pub use remote_private_route_info::*;
 pub use route_set_spec_detail::*;
 pub use route_spec_store::*;
@@ -27,9 +29,3 @@ const REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY: TimestampDuration = TimestampDuration::
 const ROUTE_MIN_IDLE_TIME_MS: u32 = 30_000;
 /// The size of the compiled route cache
 const COMPILED_ROUTE_CACHE_SIZE: usize = 256;
-
-/// The type of an allocated route set id
-pub type RouteSetSpecId = String;
-
-/// Type type of an imported remote route set id
-pub type RemotePrivateRouteId = String;

veilid-core/src/routing_table/route_spec_store/permutation.rs

@@ -0,0 +1,70 @@
+use super::*;
+
+/// number of route permutations is the number of unique orderings
+/// for a set of nodes, given that the first node is fixed
+fn _get_route_permutation_count(hop_count: usize) -> usize {
+    if hop_count == 0 {
+        unreachable!();
+    }
+    // a single node or two nodes is always fixed
+    if hop_count == 1 || hop_count == 2 {
+        return 1;
+    }
+    // more than two nodes has factorial permutation
+    // hop_count = 3 -> 2! -> 2
+    // hop_count = 4 -> 3! -> 6
+    (3..hop_count).into_iter().fold(2usize, |acc, x| acc * x)
+}
+pub type PermReturnType = (Vec<usize>, bool);
+pub type PermFunc<'t> = Box<dyn Fn(&[usize]) -> Option<PermReturnType> + Send + 't>;
+
+/// get the route permutation at particular 'perm' index, starting at the 'start' index
+/// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum
+/// number of permutations is given by get_route_permutation_count()
+
+pub fn with_route_permutations(
+    hop_count: usize,
+    start: usize,
+    f: &PermFunc,
+) -> Option<PermReturnType> {
+    if hop_count == 0 {
+        unreachable!();
+    }
+    // initial permutation
+    let mut permutation: Vec<usize> = Vec::with_capacity(hop_count);
+    for n in 0..hop_count {
+        permutation.push(start + n);
+    }
+    // if we have one hop or two, then there's only one permutation
+    if hop_count == 1 || hop_count == 2 {
+        return f(&permutation);
+    }
+
+    // heaps algorithm, but skipping the first element
+    fn heaps_permutation(
+        permutation: &mut [usize],
+        size: usize,
+        f: &PermFunc,
+    ) -> Option<PermReturnType> {
+        if size == 1 {
+            return f(&permutation);
+        }
+
+        for i in 0..size {
+            let out = heaps_permutation(permutation, size - 1, f);
+            if out.is_some() {
+                return out;
+            }
+            if size % 2 == 1 {
+                permutation.swap(1, size);
+            } else {
+                permutation.swap(1 + i, size);
+            }
+        }
+
+        None
+    }
+
+    // recurse
+    heaps_permutation(&mut permutation, hop_count - 1, f)
+}

veilid-core/src/routing_table/route_spec_store/remote_private_route_info.rs

@@ -32,6 +32,13 @@ impl RemotePrivateRouteInfo {
         &mut self.stats
     }
 
+    pub fn has_seen_our_node_info_ts(&mut self, our_node_info_ts: Timestamp) -> bool {
+        self.last_seen_our_node_info_ts == our_node_info_ts
+    }
+    pub fn set_last_seen_our_node_info_ts(&mut self, last_seen_our_node_info_ts: Timestamp) {
+        self.last_seen_our_node_info_ts = last_seen_our_node_info_ts;
+    }
+
     // Check to see if this remote private route has expired
     pub fn did_expire(&self, cur_ts: Timestamp) -> bool {
         cur_ts.saturating_sub(self.last_touched_ts) >= REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY

veilid-core/src/routing_table/route_spec_store/route_set_spec_detail.rs

@@ -49,11 +49,22 @@ impl RouteSetSpecDetail {
         }
         tks
     }
+    pub fn get_best_route_set_key(&self) -> Option<PublicKey> {
+        self.get_route_set_keys().best().map(|k| k.key)
+    }
+    pub fn set_hop_node_refs(&mut self, node_refs: Vec<NodeRef>) {
+        self.hop_node_refs = node_refs;
+    }
     pub fn iter_route_set(
         &self,
     ) -> alloc::collections::btree_map::Iter<PublicKey, RouteSpecDetail> {
         self.route_set.iter()
     }
+    pub fn iter_route_set_mut(
+        &self,
+    ) -> alloc::collections::btree_map::IterMut<PublicKey, RouteSpecDetail> {
+        self.route_set.iter_mut()
+    }
     pub fn get_stats(&self) -> &RouteStats {
         &self.stats
     }
@@ -89,24 +100,4 @@ impl RouteSetSpecDetail {
         }
         cache
     }
-
-    /// Generate a user-facing identifier for this allocated route
-    pub fn make_id(&self) -> RouteSetSpecId {
-        let mut idbytes = [0u8; 16];
-        for (pk, _) in self.route_set.iter() {
-            for (i, x) in pk.bytes.iter().enumerate() {
-                idbytes[i % 16] ^= *x;
-            }
-        }
-        let id = format!(
-            "{:08x}-{:04x}-{:04x}-{:04x}-{:08x}{:04x}",
-            u32::from_be_bytes(idbytes[0..4].try_into().expect("32 bits")),
-            u16::from_be_bytes(idbytes[4..6].try_into().expect("16 bits")),
-            u16::from_be_bytes(idbytes[6..8].try_into().expect("16 bits")),
-            u16::from_be_bytes(idbytes[8..10].try_into().expect("16 bits")),
-            u32::from_be_bytes(idbytes[10..14].try_into().expect("32 bits")),
-            u16::from_be_bytes(idbytes[14..16].try_into().expect("16 bits"))
-        );
-        id
-    }
 }

veilid-core/src/routing_table/route_spec_store/route_spec_store.rs

@@ -1,4 +1,5 @@
 use super::*;
+use permutation::*;
 
 #[derive(Debug)]
 pub struct RouteSpecStoreInner {
@@ -33,75 +34,6 @@ pub struct RouteSpecStore {
     unlocked_inner: Arc<RouteSpecStoreUnlockedInner>,
 }
 
-/// number of route permutations is the number of unique orderings
-/// for a set of nodes, given that the first node is fixed
-fn _get_route_permutation_count(hop_count: usize) -> usize {
-    if hop_count == 0 {
-        unreachable!();
-    }
-    // a single node or two nodes is always fixed
-    if hop_count == 1 || hop_count == 2 {
-        return 1;
-    }
-    // more than two nodes has factorial permutation
-    // hop_count = 3 -> 2! -> 2
-    // hop_count = 4 -> 3! -> 6
-    (3..hop_count).into_iter().fold(2usize, |acc, x| acc * x)
-}
-type PermReturnType = (Vec<usize>, Vec<u8>, bool);
-type PermFunc<'t> = Box<dyn Fn(&[usize]) -> Option<PermReturnType> + Send + 't>;
-
-/// get the route permutation at particular 'perm' index, starting at the 'start' index
-/// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum
-/// number of permutations is given by get_route_permutation_count()
-
-fn with_route_permutations(
-    hop_count: usize,
-    start: usize,
-    f: &PermFunc,
-) -> Option<PermReturnType> {
-    if hop_count == 0 {
-        unreachable!();
-    }
-    // initial permutation
-    let mut permutation: Vec<usize> = Vec::with_capacity(hop_count);
-    for n in 0..hop_count {
-        permutation.push(start + n);
-    }
-    // if we have one hop or two, then there's only one permutation
-    if hop_count == 1 || hop_count == 2 {
-        return f(&permutation);
-    }
-
-    // heaps algorithm, but skipping the first element
-    fn heaps_permutation(
-        permutation: &mut [usize],
-        size: usize,
-        f: &PermFunc,
-    ) -> Option<PermReturnType> {
-        if size == 1 {
-            return f(&permutation);
-        }
-
-        for i in 0..size {
-            let out = heaps_permutation(permutation, size - 1, f);
-            if out.is_some() {
-                return out;
-            }
-            if size % 2 == 1 {
-                permutation.swap(1, size);
-            } else {
-                permutation.swap(1 + i, size);
-            }
-        }
-
-        None
-    }
-
-    // recurse
-    heaps_permutation(&mut permutation, hop_count - 1, f)
-}
-
 impl RouteSpecStore {
     pub fn new(routing_table: RoutingTable) -> Self {
         let config = routing_table.network_manager().config();
@@ -135,69 +67,7 @@ impl RouteSpecStore {
         };
 
         // Get frozen blob from table store
-        let table_store = routing_table.network_manager().table_store();
+        let content = RouteSpecStoreContent::load(routing_table.clone()).await?;
-        let rsstdb = table_store.open("RouteSpecStore", 1).await?;
-        let mut content: RouteSpecStoreContent =
-            rsstdb.load_rkyv(0, b"content")?.unwrap_or_default();
-
-        // Look up all route hop noderefs since we can't serialize those
-        let mut dead_ids = Vec::new();
-        for (rsid, rssd) in content.iter_details_mut() {
-            // Get first route since they all should resolve
-            let Some((pk, rsd)) = rssd.route_set.first_key_value() else {
-                dead_ids.push(rsid.clone());
-                continue;
-            };
-            // Go through first route 
-            for h in &rsd.hops {
-                let Some(nr) = routing_table.lookup_node_ref(TypedKey::new(rsd.crypto_kind, *h)) else {
-                    dead_ids.push(rsid.clone());
-                    break;
-                };
-                rssd.hop_node_refs.push(nr);
-            }
-        }
-        for id in dead_ids {
-            log_rtab!(debug "no entry, killing off private route: {}", id);
-            content.remove_detail(&id);
-        }
-
-        // Load secrets from pstore
-        let pstore = routing_table.network_manager().protected_store();
-        let secret_key_map: HashMap<PublicKey, SecretKey> = pstore
-            .load_user_secret_rkyv("RouteSpecStore")
-            .await?
-            .unwrap_or_default();
-
-        // Ensure we got secret keys for all the public keys
-        let mut got_secret_key_ids = HashSet::new();
-        for (rsid, rssd) in content.iter_details_mut() {
-            let mut found_all = true;
-            for (pk, rsd) in &mut rssd.route_set {
-                if let Some(sk) = secret_key_map.get(pk) {
-                    rsd.secret_key = *sk;
-                } else {
-                    found_all = false;
-                    break;
-                }
-            }
-            if found_all {
-                got_secret_key_ids.insert(rsid.clone());
-            }
-        }
-    
-        // If we missed any, nuke those route ids
-        let dead_ids:Vec<String> = content.keys().filter_map(|id| {
-            if !got_secret_key_ids.contains(id) {
-                Some(id.clone())
-            } else {
-                None
-            }
-        }).collect();
-        for id in dead_ids {
-            log_rtab!(debug "missing secret key, killing off private route: {}", id);
-            content.remove_detail(&id);
-        }
 
         let mut inner = RouteSpecStoreInner {
             content,
@@ -230,31 +100,8 @@ impl RouteSpecStore {
             inner.content.clone()
         };
 
-        // Save all the fields we care about to the frozen blob in table storage
+        // Save our content
-        // This skips #[with(Skip)] saving the secret keys, we save them in the protected store instead
+        content.save(self.unlocked_inner.routing_table.clone()).await?;
-        let table_store = self
-            .unlocked_inner
-            .routing_table
-            .network_manager()
-            .table_store();
-        let rsstdb = table_store.open("RouteSpecStore", 1).await?;
-        rsstdb.store_rkyv(0, b"content", &content).await?;
-
-        // // Keep secrets in protected store as well
-        let pstore = self
-            .unlocked_inner
-            .routing_table
-            .network_manager()
-            .protected_store();
-
-        let mut out: HashMap<PublicKey, SecretKey> = HashMap::new();
-        for (rsid, rssd) in content.iter_details() {
-            for (pk, rsd) in &rssd.route_set {
-                out.insert(*pk, rsd.secret_key);
-            }
-        }
-
-        let _ = pstore.save_user_secret_rkyv("RouteSpecStore", &out).await?; // ignore if this previously existed or not
 
         Ok(())
     }
@@ -263,7 +110,11 @@ impl RouteSpecStore {
     pub fn send_route_update(&self) {
         let (dead_routes, dead_remote_routes) = {
             let mut inner = self.inner.lock();
-            inner.cache.take_dead_routes()
+            let Some(dr) = inner.cache.take_dead_routes() else {
+                // Nothing to do
+                return;
+            };
+            dr
         };
 
         let update = VeilidUpdate::Route(VeilidStateRoute {
@@ -275,8 +126,6 @@ impl RouteSpecStore {
         update_callback(update);
     }
 
-
-
     /// Purge the route spec store
     pub async fn purge(&self) -> EyreResult<()> {
         {
@@ -301,7 +150,7 @@ impl RouteSpecStore {
         hop_count: usize,
         directions: DirectionSet,
         avoid_nodes: &[TypedKey],
-    ) -> EyreResult<Option<String>> {
+    ) -> EyreResult<Option<RouteId>> {
         let inner = &mut *self.inner.lock();
         let routing_table = self.unlocked_inner.routing_table.clone();
         let rti = &mut *routing_table.inner.write();
@@ -329,7 +178,7 @@ impl RouteSpecStore {
         hop_count: usize,
         directions: DirectionSet,
         avoid_nodes: &[TypedKey],
-    ) -> EyreResult<Option<String>> {
+    ) -> EyreResult<Option<RouteId>> {
         use core::cmp::Ordering;
 
         if hop_count < 1 {
@@ -625,7 +474,7 @@ impl RouteSpecStore {
 
         for start in 0..(nodes.len() - hop_count) {
             // Try the permutations available starting with 'start'
-            if let Some((rn, ck, cds)) = with_route_permutations(hop_count, start, &perm_func) {
+            if let Some((rn, cds)) = with_route_permutations(hop_count, start, &perm_func) {
                 route_nodes = rn;
                 can_do_sequenced = cds;
                 break;
@@ -666,11 +515,14 @@ impl RouteSpecStore {
 
         drop(perm_func);
 
+        // make id
+        let id = self.generate_allocated_route_id(&rssd)?;
+
         // Add to cache
         inner.cache.add_to_cache(&rssd);
 
         // Keep route in spec store
-        let id = inner.content.add_detail(rssd);
+        inner.content.add_detail(id.clone(), rssd);
 
         Ok(Some(id))
     }
@@ -737,23 +589,18 @@ impl RouteSpecStore {
     }
 
     #[instrument(level = "trace", skip(self), ret, err)]
-    async fn test_allocated_route(&self, id: &String) -> EyreResult<bool> {
+    async fn test_allocated_route(&self, private_route_id: RouteId) -> EyreResult<bool> {
         // Make loopback route to test with
         let dest = {
 
-            // Get best route from set
             // Match the private route's hop length for safety route length
-            let (key, hop_count) = {
+            let hop_count = {
                 let inner = &mut *self.inner.lock();
-                let Some(rssd) = inner.content.get_detail(id) else {
+                let Some(rssd) = inner.content.get_detail(&private_route_id) else {
                     bail!("route id not allocated");
                 };
-                let Some(tkey) = rssd.get_route_set_keys().best() else {
+                rssd.hop_count()
-                    bail!("route does not have best key");
-                };
-                (tkey.key, rssd.hop_count())
             };
-            let private_route = self.assemble_private_route(&key, None)?;
 
             // Always test routes with safety routes that are more likely to succeed
             let stability = Stability::Reliable;
@@ -761,7 +608,7 @@ impl RouteSpecStore {
             let sequencing = Sequencing::NoPreference;
 
             let safety_spec = SafetySpec {
-                preferred_route: Some(key.clone()),
+                preferred_route: Some(private_route_id),
                 hop_count,
                 stability,
                 sequencing,
@@ -769,7 +616,7 @@ impl RouteSpecStore {
             let safety_selection = SafetySelection::Safe(safety_spec);
 
             Destination::PrivateRoute {
-                private_route,
+                private_route_id,
                 safety_selection,
             }
         };
@@ -788,18 +635,10 @@ impl RouteSpecStore {
     }
 
     #[instrument(level = "trace", skip(self), ret, err)]
-    async fn test_remote_route(&self, id: &String) -> EyreResult<bool> {
+    async fn test_remote_route(&self, private_route_id: RouteId) -> EyreResult<bool> {
         
         // Make private route test
         let dest = {
-            // Get best remote route from imported set
-
-            // Get the route to test
-            let private_route = match self.peek_remote_private_route(id) {
-                Some(pr) => pr,
-                None => return Ok(false),
-            };
-
             // Get a safety route that is good enough
             let safety_spec = SafetySpec {
                 preferred_route: None,
@@ -811,7 +650,7 @@ impl RouteSpecStore {
             let safety_selection = SafetySelection::Safe(safety_spec);
 
             Destination::PrivateRoute {
-                private_route,
+                private_route_id,
                 safety_selection,
             }
         };
@@ -829,52 +668,44 @@ impl RouteSpecStore {
         Ok(true)
     }
 
-    /// Test an allocated route for continuity
-    #[instrument(level = "trace", skip(self), ret, err)]
-    pub async fn test_route(&self, key: &[TypedKey]) -> EyreResult<bool> {
-        let is_remote = {
-            let inner = &mut *self.inner.lock();
-            let cur_ts = get_aligned_timestamp();
-            Self::with_peek_remote_private_route(inner, cur_ts, key, |_| {}).is_some()
-        };
-        if is_remote {
-            self.test_remote_route(key).await
-        } else {
-            self.test_allocated_route(key).await
-        }
-    }
-
     /// Release an allocated route that is no longer in use
     #[instrument(level = "trace", skip(self), ret)]
-    fn release_allocated_route(&self, id: &String) -> bool {
+    fn release_allocated_route(&self, id: RouteId) -> bool {
         let mut inner = self.inner.lock();
-        let Some(rssd) = inner.content.remove_detail(id) else {
+        let Some(rssd) = inner.content.remove_detail(&id) else {
             return false;
         };
 
         // Remove from hop cache
-        if !inner.cache.remove_from_cache(&rssd) {
+        if !inner.cache.remove_from_cache(id, &rssd) {
             panic!("hop cache should have contained cache key");
         }
 
         true
     }
 
+    /// Check if a route id is remote or not
+    fn is_route_id_remote(&self, id: &RouteId) -> bool {
+        let inner = &mut *self.inner.lock();
+        let cur_ts = get_aligned_timestamp();
+        inner.cache.peek_remote_private_route_mut(cur_ts, &id).is_some()
+    }
+
+    /// Test an allocated route for continuity
+    #[instrument(level = "trace", skip(self), ret, err)]
+    pub async fn test_route(&self, id: RouteId) -> EyreResult<bool> {
+        let is_remote = self.is_route_id_remote(&id);
+        if is_remote {
+            self.test_remote_route(id).await
+        } else {
+            self.test_allocated_route(id).await
+        }
+    }
+
     /// Release an allocated or remote route that is no longer in use
     #[instrument(level = "trace", skip(self), ret)]
-    pub fn release_route(&self, id: &String) -> bool {
+    pub fn release_route(&self, id: RouteId) -> bool {
-
+        let is_remote = self.is_route_id_remote(&id);
-        let is_remote = {
-            let inner = &mut *self.inner.lock();
-
-            // Release from compiled route cache if it's used there
-            self.invalidate_compiled_route_cache(inner, id);
-
-            // Check to see if this is a remote route
-            let cur_ts = get_aligned_timestamp();
-            Self::with_peek_remote_private_route(inner, cur_ts, id, |_| {}).is_some()
-        };
-
         if is_remote {
             self.release_remote_private_route(id)
         } else {
@@ -943,11 +774,11 @@ impl RouteSpecStore {
     /// List all allocated routes
     pub fn list_allocated_routes<F, R>(&self, mut filter: F) -> Vec<R>
     where
-        F: FnMut(&PublicKey, &RouteSetSpecDetail) -> Option<R>,
+        F: FnMut(&RouteId, &RouteSetSpecDetail) -> Option<R>,
     {
         let inner = self.inner.lock();
-        let mut out = Vec::with_capacity(inner.content.details.len());
+        let mut out = Vec::with_capacity(inner.content.get_detail_count());
-        for detail in &inner.content.details {
+        for detail in inner.content.iter_details() {
             if let Some(x) = filter(detail.0, detail.1) {
                 out.push(x);
             }
@@ -958,11 +789,11 @@ impl RouteSpecStore {
     /// List all allocated routes
     pub fn list_remote_routes<F, R>(&self, mut filter: F) -> Vec<R>
     where
-        F: FnMut(&PublicKey, &RemotePrivateRouteInfo) -> Option<R>,
+        F: FnMut(&RouteId, &RemotePrivateRouteInfo) -> Option<R>,
     {
         let inner = self.inner.lock();
-        let mut out = Vec::with_capacity(inner.cache.remote_private_route_cache.len());
+        let mut out = Vec::with_capacity(inner.cache.get_remote_private_route_count());
-        for info in &inner.cache.remote_private_route_cache {
+        for info in inner.cache.iter_remote_private_routes() {
             if let Some(x) = filter(info.0, info.1) {
                 out.push(x);
             }
@@ -971,56 +802,20 @@ impl RouteSpecStore {
     }
 
     /// Get the debug description of a route
-    pub fn debug_route(&self, key: &PublicKey) -> Option<String> {
+    pub fn debug_route(&self, id: &RouteId) -> Option<String> {
         let inner = &mut *self.inner.lock();
         let cur_ts = get_aligned_timestamp();
-        // If this is a remote route, print it
+        if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &id) {
-        if let Some(s) =
+            return Some(format!("{:#?}", rpri));
-            Self::with_peek_remote_private_route(inner, cur_ts, key, |rpi| format!("{:#?}", rpi))
-        {
-            return Some(s);
         }
-        // Otherwise check allocated routes
+        if let Some(rssd) = inner.content.get_detail(id) {
-        Self::detail(inner, key).map(|rsd| format!("{:#?}", rsd))
+            return Some(format!("{:#?}", rssd));
+        }
+        None
     }
 
     //////////////////////////////////////////////////////////////////////
 
-    // Route cache
-    fn add_to_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, pr_pubkey: PublicKey, safety_route: SafetyRoute)
-    {
-        let key = CompiledRouteCacheKey {
-            sr_pubkey: safety_route.public_key,
-            pr_pubkey,
-        };
-
-        if let Some(v) = inner.cache.compiled_route_cache.insert(key, safety_route) {
-            log_rtab!(error "route cache already contained key: sr_pubkey={:?}, pr_pubkey={:?}", v.public_key, pr_pubkey);
-        }
-    }
-
-    fn lookup_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, sr_pubkey: PublicKey, pr_pubkey: PublicKey) -> Option<SafetyRoute> {
-
-        let key = CompiledRouteCacheKey {
-            sr_pubkey,
-            pr_pubkey,
-        };
-
-        inner.cache.compiled_route_cache.get(&key).cloned()
-    }
-
-    fn invalidate_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, dead_key: &PublicKey) {
-        let mut dead_entries = Vec::new();
-        for (k, _v) in inner.cache.compiled_route_cache.iter() {
-            if k.sr_pubkey == *dead_key || k.pr_pubkey == *dead_key {
-                dead_entries.push(k.clone());
-            }
-        }
-        for d in dead_entries {
-            inner.cache.compiled_route_cache.remove(&d);
-        }
-    }
-
     /// Compiles a safety route to the private route, with caching
     /// Returns an Err() if the parameters are wrong
     /// Returns Ok(None) if no allocation could happen at this time (not an error)
@@ -1441,16 +1236,20 @@ impl RouteSpecStore {
     }
 
     /// Import a remote private route for compilation
-    /// returns a route set id
+    /// It is safe to import the same route more than once and it will return the same route id
+    /// Returns a route set id
     #[instrument(level = "trace", skip(self, blob), ret, err)]
-    pub fn import_remote_private_route(&self, blob: Vec<u8>) -> EyreResult<String> { 
+    pub fn import_remote_private_route(&self, blob: Vec<u8>) -> EyreResult<RouteId> { 
+        let cur_ts = get_aligned_timestamp();
         
         // decode the pr blob
         let private_routes = RouteSpecStore::blob_to_private_routes(self.unlocked_inner.routing_table.crypto(), blob)?;
 
-        let inner = &mut *self.inner.lock();
+        // make the route id
-        
+        let id = self.generate_remote_route_id(&private_routes)?;
+
         // validate the private routes
+        let inner = &mut *self.inner.lock();
         for private_route in private_routes {
 
             // ensure private route has first hop
@@ -1464,62 +1263,63 @@ impl RouteSpecStore {
             }
         }
 
-        let cur_ts = get_aligned_timestamp();
+        inner.cache.cache_remote_private_route(cur_ts, id, private_routes);
-        let id = inner.cache.import_remote_private_route(cur_ts, private_routes);
 
         Ok(id)
     }
 
     /// Release a remote private route that is no longer in use
     #[instrument(level = "trace", skip(self), ret)]
-    fn release_remote_private_route(&self, id: &String) -> bool {
+    pub fn release_remote_private_route(&self, id: RouteId) -> bool {
         let inner = &mut *self.inner.lock();
         inner.cache.remove_remote_private_route(id)
     }
 
-    /// Retrieve an imported remote private route by its public key
+    /// Check if a remote private route id is valid
-    pub fn get_remote_private_route(&self, id: &String) -> Option<PrivateRoute> {
+    #[instrument(level = "trace", skip(self), ret)]
+    pub fn is_valid_remote_private_route(&self, id: &RouteId) -> bool {
         let inner = &mut *self.inner.lock();
         let cur_ts = get_aligned_timestamp();
-        Self::with_get_remote_private_route(inner, cur_ts, key, |r| {
+        inner.cache.peek_remote_private_route_mut(cur_ts, id).is_some()
-            r.private_route.as_ref().unwrap().clone()
-        })
     }
 
-    /// Retrieve an imported remote private route by its public key but don't 'touch' it
+    // /// Retrieve an imported remote private route by its public key
-    pub fn peek_remote_private_route(&self, id: &String) -> Option<PrivateRoute> {
+    // pub fn get_remote_private_route(&self, id: &String) -> Option<PrivateRoute> {
-        xx fix these
+    //     let inner = &mut *self.inner.lock();
-        let inner = &mut *self.inner.lock();
+    //     let cur_ts = get_aligned_timestamp();
-        let cur_ts = get_aligned_timestamp();
+    //     Self::with_get_remote_private_route(inner, cur_ts, key, |r| {
-        Self::with_peek_remote_private_route(inner, cur_ts, key, |r| {
+    //         r.private_route.as_ref().unwrap().clone()
-            r.private_route.as_ref().unwrap().clone()
+    //     })
-        })
+    // }
-    }
+
+    // /// Retrieve an imported remote private route by its public key but don't 'touch' it
+    // fn peek_remote_private_route(&self, id: &String) -> Option<PrivateRoute> {
+    //     let inner = &mut *self.inner.lock();
+    //     let cur_ts = get_aligned_timestamp();
+    //     inner.cache.with_peek_remote_private_route(cur_ts, id, f)
+    //     Self::with_peek_remote_private_route(inner, cur_ts, key, |r| {
+    //         r.private_route.as_ref().unwrap().clone()
+    //     })
+    // }
 
     /// Check to see if this remote (not ours) private route has seen our current node info yet
     /// This happens when you communicate with a private route without a safety route
-    pub fn has_remote_private_route_seen_our_node_info(&self, key: &PublicKey) -> bool {
+    pub fn has_remote_private_route_seen_our_node_info(&self, id: &RouteId) -> bool {
         let our_node_info_ts = {
             let rti = &*self.unlocked_inner.routing_table.inner.read();
             let Some(ts) = rti.get_own_node_info_ts(RoutingDomain::PublicInternet) else {
+                // Node info is invalid, skip this
                 return false;
             };
             ts
         };
 
-        let opt_rpr_node_info_ts = {
+        let inner = &mut *self.inner.lock();
-            let inner = &mut *self.inner.lock();
+        let cur_ts = get_aligned_timestamp();
-            let cur_ts = get_aligned_timestamp();
+        let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &id) else {
-            Self::with_peek_remote_private_route(inner, cur_ts, key, |rpr| {
-                rpr.last_seen_our_node_info_ts
-            })
-        };
-
-        let Some(rpr_node_info_ts) = opt_rpr_node_info_ts else {
             return false;
         };
-
+        rpri.has_seen_our_node_info_ts(our_node_info_ts)
-        our_node_info_ts == rpr_node_info_ts
     }
 
     /// Mark a remote private route as having seen our current node info
@@ -1543,20 +1343,23 @@ impl RouteSpecStore {
         };
 
         let inner = &mut *self.inner.lock();
+
         // Check for local route. If this is not a remote private route
         // then we just skip the recording. We may be running a test and using
         // our own local route as the destination private route.
-        if let Some(_) = Self::detail_mut(inner, key) {
+        if let Some(_) = inner.content.get_id_by_key(key) {
             return Ok(());
         }
-        if Self::with_get_remote_private_route(inner, cur_ts, key, |rpr| {
+
-            rpr.last_seen_our_node_info_ts = our_node_info_ts;
+        if let Some(rrid) = inner.cache.get_remote_private_route_id_by_key(key) {
-        })
+            if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &rrid)
-        .is_none()
+            {
-        {
+                rpri.set_last_seen_our_node_info_ts(our_node_info_ts);
-            bail!("private route is missing from store: {}", key);
+                return Ok(());
+            }
         }
-        Ok(())
+
+        bail!("private route is missing from store: {}", key);
     }
 
     /// Get the route statistics for any route we know about, local or remote
@@ -1570,15 +1373,20 @@ impl RouteSpecStore {
         if self.unlocked_inner.routing_table.matches_own_node_id_key(key) {
             return None;
         }
+
         // Check for local route
-        if let Some(rsd) = Self::detail_mut(inner, key) {
+        if let Some(rsid) = inner.content.get_id_by_key(key) {
-            return Some(f(&mut rsd.stats));
+            if let Some(rsd) = inner.content.get_detail_mut(&rsid) {
+                return Some(f(rsd.get_stats_mut()));
+            }
         }
+
         // Check for remote route
-        if let Some(res) =
+        if let Some(rrid) = inner.cache.get_remote_private_route_id_by_key(key) {
-            Self::with_peek_remote_private_route(inner, cur_ts, key, |rpr| f(&mut rpr.stats))
+            if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &rrid)
-        {
+            {
-            return Some(res);
+                return Some(f(rpri.get_stats_mut()));
+            }
         }
 
         None
@@ -1613,13 +1421,10 @@ impl RouteSpecStore {
         let inner = &mut *self.inner.lock();
 
         // Roll transfers for locally allocated routes
-        for rssd in inner.content.details.values_mut() {
+        inner.content.roll_transfers(last_ts, cur_ts);
-            rssd.stats.roll_transfers(last_ts, cur_ts);
+
-        }
         // Roll transfers for remote private routes
-        for (_k, v) in inner.cache.remote_private_route_cache.iter_mut() {
+        inner.cache.roll_transfers(last_ts, cur_ts);
-            v.stats.roll_transfers(last_ts, cur_ts);
-        }
     }
 
     /// Convert private route list to binary blob
@@ -1681,6 +1486,54 @@ impl RouteSpecStore {
             out.push(private_route);
         }
         
+        // Don't trust the order of the blob
+        out.sort_by(|a,b| {
+            a.public_key.cmp(&b.public_key)
+        });
+
         Ok(out)
     }
+
+    /// Generate RouteId from typed key set of route public keys
+    fn generate_allocated_route_id(&self, rssd: &RouteSetSpecDetail) -> EyreResult<RouteId> {
+        let route_set_keys = rssd.get_route_set_keys();
+        let crypto = self.unlocked_inner.routing_table.crypto();
+        
+        let mut idbytes = Vec::with_capacity(PUBLIC_KEY_LENGTH * route_set_keys.len());
+        let mut best_kind : Option<CryptoKind> = None;
+        for tk in route_set_keys.iter() {
+            if best_kind.is_none() || compare_crypto_kind(&tk.kind, best_kind.as_ref().unwrap()) == cmp::Ordering::Less {
+                best_kind = Some(tk.kind);
+            } 
+            idbytes.extend_from_slice(&tk.key.bytes);
+        }
+        let Some(best_kind) = best_kind else {
+            bail!("no compatible crypto kinds in route");
+        };
+        let vcrypto = crypto.get(best_kind).unwrap();
+
+        Ok(RouteId::new(vcrypto.generate_hash(&idbytes).bytes))
+
+    }
+
+    /// Generate RouteId from set of private routes    
+    fn generate_remote_route_id(&self, private_routes: &[PrivateRoute]) -> EyreResult<RouteId> {
+        let crypto = self.unlocked_inner.routing_table.crypto();
+        
+        let mut idbytes = Vec::with_capacity(PUBLIC_KEY_LENGTH * private_routes.len());
+        let mut best_kind : Option<CryptoKind> = None;
+        for private_route in private_routes {
+            if best_kind.is_none() || compare_crypto_kind(&private_route.public_key.kind, best_kind.as_ref().unwrap()) == cmp::Ordering::Less {
+                best_kind = Some(private_route.public_key.kind);
+            } 
+            idbytes.extend_from_slice(&private_route.public_key.key.bytes);
+        }
+        let Some(best_kind) = best_kind else {
+            bail!("no compatible crypto kinds in route");
+        };
+        let vcrypto = crypto.get(best_kind).unwrap();
+
+        Ok(RouteId::new(vcrypto.generate_hash(&idbytes).bytes))
+    }
+
 }

veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs

@@ -28,15 +28,15 @@ pub struct RouteSpecStoreCache {
     /// Route spec hop cache, used to quickly disqualify routes
     hop_cache: HashSet<Vec<u8>>,
     /// Remote private routes we've imported and statistics
-    remote_private_route_set_cache: LruCache<RemotePrivateRouteId, RemotePrivateRouteInfo>,
+    remote_private_route_set_cache: LruCache<RouteId, RemotePrivateRouteInfo>,
     /// Remote private routes indexed by public key
-    remote_private_routes_by_key: HashMap<PublicKey, RemotePrivateRouteId>,
+    remote_private_routes_by_key: HashMap<PublicKey, RouteId>,
     /// Compiled route cache
     compiled_route_cache: LruCache<CompiledRouteCacheKey, SafetyRoute>,
     /// List of dead allocated routes
-    dead_routes: Vec<RouteSetSpecId>,
+    dead_routes: Vec<RouteId>,
     /// List of dead remote routes
-    dead_remote_routes: Vec<RemotePrivateRouteId>,
+    dead_remote_routes: Vec<RouteId>,
 }
 
 impl RouteSpecStoreCache {
@@ -66,7 +66,7 @@ impl RouteSpecStoreCache {
     }
 
     /// removes an allocated route set from our cache
-    pub fn remove_from_cache(&mut self, rssd: &RouteSetSpecDetail) -> bool {
+    pub fn remove_from_cache(&mut self, id: RouteId, rssd: &RouteSetSpecDetail) -> bool {
         let cache_key = rssd.make_cache_key();
 
         // Remove from hop cache
@@ -100,10 +100,13 @@ impl RouteSpecStoreCache {
                     panic!("used_end_nodes cache should have contained hop");
                 }
             }
+
+            // Invalidate compiled route cache
+            self.invalidate_compiled_route_cache(pk);
         }
 
         // Mark it as dead for the update
-        self.dead_routes.push(rssd.make_id());
+        self.dead_routes.push(id);
 
         true
     }
@@ -122,43 +125,27 @@ impl RouteSpecStoreCache {
         })
     }
 
-    /// generate unique remote private route set id for a remote private route set
-    fn make_remote_private_route_id(private_routes: &[PrivateRoute]) -> String {
-        let mut idbytes = [0u8; 16];
-        for (pk, _) in &rprinfo.private_routes {
-            for (i, x) in pk.bytes.iter().enumerate() {
-                idbytes[i % 16] ^= *x;
-            }
-        }
-        let id = format!(
-            "{:08x}-{:04x}-{:04x}-{:04x}-{:08x}{:04x}",
-            u32::from_be_bytes(idbytes[0..4].try_into().expect("32 bits")),
-            u16::from_be_bytes(idbytes[4..6].try_into().expect("16 bits")),
-            u16::from_be_bytes(idbytes[6..8].try_into().expect("16 bits")),
-            u16::from_be_bytes(idbytes[8..10].try_into().expect("16 bits")),
-            u32::from_be_bytes(idbytes[10..14].try_into().expect("32 bits")),
-            u16::from_be_bytes(idbytes[14..16].try_into().expect("16 bits"))
-        );
-        id
-    }
-
     /// add remote private route to caches
     /// returns a remote private route set id
     fn add_remote_private_route(
         &mut self,
+        id: RouteId,
         rprinfo: RemotePrivateRouteInfo,
-    ) -> RemotePrivateRouteId {
+    ) -> RouteId {
-        let id = Self::make_remote_private_route_id(rprinfo.get_private_routes());
-
         // also store in id by key table
-        for (pk, _) in rprinfo.get_private_routes() {
+        for private_route in rprinfo.get_private_routes() {
-            self.remote_private_routes_by_key.insert(*pk, id.clone());
+            self.remote_private_routes_by_key
+                .insert(private_route.public_key.key, id.clone());
         }
         self.remote_private_route_set_cache
-            .insert(id.clone(), rprinfo, |dead_id, dead_rpri| {
+            .insert(id, rprinfo, |dead_id, dead_rpri| {
                 // If anything LRUs out, remove from the by-key table
-                for (dead_pk, _) in dead_rpri.get_private_routes() {
+                // Follow the same logic as 'remove_remote_private_route' here
-                    self.remote_private_routes_by_key.remove(&dead_pk).unwrap();
+                for dead_private_route in dead_rpri.get_private_routes() {
+                    self.remote_private_routes_by_key
+                        .remove(&dead_private_route.public_key.key)
+                        .unwrap();
+                    self.invalidate_compiled_route_cache(&dead_private_route.public_key.key);
                 }
                 self.dead_remote_routes.push(dead_id);
             });
@@ -166,33 +153,68 @@ impl RouteSpecStoreCache {
         id
     }
 
+    /// get count of remote private routes in cache
+    pub fn get_remote_private_route_count(&self) -> usize {
+        self.remote_private_route_set_cache.len()
+    }
+
+    /// iterate all of the remote private routes we have in the cache
+    pub fn iter_remote_private_routes(
+        &self,
+    ) -> hashlink::linked_hash_map::Iter<RouteId, RemotePrivateRouteInfo> {
+        self.remote_private_route_set_cache.iter()
+    }
+
     /// remote private route cache accessor
-    fn get_remote_private_route(
+    /// will LRU entries and may expire entries and not return them if they are stale
+    pub fn get_remote_private_route(
         &mut self,
-        id: &RemotePrivateRouteId,
+        cur_ts: Timestamp,
+        id: &RouteId,
     ) -> Option<&RemotePrivateRouteInfo> {
-        self.remote_private_route_set_cache.get(id)
+        if let Some(rpri) = self.remote_private_route_set_cache.get(id) {
+            if !rpri.did_expire(cur_ts) {
+                rpri.touch(cur_ts);
+                return Some(rpri);
+            }
+        }
+        None
     }
+
     /// mutable remote private route cache accessor
-    fn get_remote_private_route_mut(
+    /// will LRU entries and may expire entries and not return them if they are stale
+    pub fn get_remote_private_route_mut(
         &mut self,
-        id: &RemotePrivateRouteId,
+        cur_ts: Timestamp,
+        id: &RouteId,
     ) -> Option<&mut RemotePrivateRouteInfo> {
-        self.remote_private_route_set_cache.get_mut(id)
+        if let Some(rpri) = self.remote_private_route_set_cache.get_mut(id) {
+            if !rpri.did_expire(cur_ts) {
+                rpri.touch(cur_ts);
+                return Some(rpri);
+            }
+        }
+        None
     }
+
     /// mutable remote private route cache accessor without lru action
-    fn peek_remote_private_route_mut(
+    /// will not LRU entries but may expire entries and not return them if they are stale
+    pub fn peek_remote_private_route_mut(
         &mut self,
-        id: &RemotePrivateRouteId,
+        cur_ts: Timestamp,
+        id: &RouteId,
     ) -> Option<&mut RemotePrivateRouteInfo> {
-        self.remote_private_route_set_cache.peek_mut(id)
+        if let Some(rpri) = self.remote_private_route_set_cache.peek_mut(id) {
+            if !rpri.did_expire(cur_ts) {
+                rpri.touch(cur_ts);
+                return Some(rpri);
+            }
+        }
+        None
     }
 
     /// look up a remote private route id by one of the route public keys
-    pub fn get_remote_private_route_id_by_key(
+    pub fn get_remote_private_route_id_by_key(&self, key: &PublicKey) -> Option<RouteId> {
-        &self,
-        key: &PublicKey,
-    ) -> Option<RemotePrivateRouteId> {
         self.remote_private_routes_by_key.get(key).cloned()
     }
 
@@ -200,14 +222,14 @@ impl RouteSpecStoreCache {
     /// may LRU and/or expire other cache entries to make room for the new one
     /// or update an existing entry with the same private route set
     /// returns the route set id
-    pub fn import_remote_private_route(
+    pub fn cache_remote_private_route(
         &mut self,
         cur_ts: Timestamp,
+        id: RouteId,
         private_routes: Vec<PrivateRoute>,
-    ) -> RemotePrivateRouteId {
+    ) {
         // get id for this route set
-        let id = RouteSpecStoreCache::make_remote_private_route_id(&private_routes);
+        if let Some(rpri) = self.get_remote_private_route_mut(cur_ts, &id) {
-        let rpri = if let Some(rpri) = self.get_remote_private_route_mut(&id) {
             if rpri.did_expire(cur_ts) {
                 // Start fresh if this had expired
                 rpri.unexpire(cur_ts);
@@ -223,88 +245,95 @@ impl RouteSpecStoreCache {
                 last_touched_ts: cur_ts,
                 stats: RouteStats::new(cur_ts),
             };
-            let new_id = self.add_remote_private_route(rpri);
+            self.add_remote_private_route(id, rpri);
-            assert_eq!(id, new_id);
+            if self.peek_remote_private_route_mut(cur_ts, &id).is_none() {
-            if self.get_remote_private_route_mut(&id).is_none() {
+                panic!("remote private route should exist");
-                bail!("remote private route should exist");
             };
         };
-        id
     }
 
     /// remove a remote private route from the cache
-    pub fn remove_remote_private_route(&mut self, id: &RemotePrivateRouteId) -> bool {
+    pub fn remove_remote_private_route(&mut self, id: RouteId) -> bool {
-        let Some(rprinfo) = self.remote_private_route_set_cache.remove(id) else {
+        let Some(rprinfo) = self.remote_private_route_set_cache.remove(&id) else {
             return false;
         };
-        for (pk, _) in rprinfo.get_private_routes() {
+        for private_route in rprinfo.get_private_routes() {
-            self.remote_private_routes_by_key.remove(&pk).unwrap();
+            self.remote_private_routes_by_key
+                .remove(&private_route.public_key.key)
+                .unwrap();
+            self.invalidate_compiled_route_cache(&private_route.public_key.key);
         }
-        self.dead_remote_routes.push(id.clone());
+        self.dead_remote_routes.push(id);
         true
     }
 
-    /// get an existing remote private route cache entry
+    /// Stores a compiled 'safety + private' route so we don't have to compile it again later
-    /// will LRU entries and may expire entries and not return them if they are stale
+    pub fn add_to_compiled_route_cache(&self, pr_pubkey: PublicKey, safety_route: SafetyRoute) {
-    /// calls a callback with the remote private route info if returned
+        let key = CompiledRouteCacheKey {
-    pub fn with_get_remote_private_route<F, R>(
+            sr_pubkey: safety_route.public_key.key,
-        &mut self,
+            pr_pubkey,
-        cur_ts: Timestamp,
+        };
-        id: &RemotePrivateRouteId,
+
-        f: F,
+        if let Some(v) = self
-    ) -> Option<R>
+            .compiled_route_cache
-    where
+            .insert(key, safety_route, |_k, _v| {
-        F: FnOnce(&mut RemotePrivateRouteInfo) -> R,
+                // Do nothing on LRU evict
-    {
+            })
-        if let Some(rpri) = self.get_remote_private_route_mut(&id) {
+        {
-            if !rpri.did_expire(cur_ts) {
+            log_rtab!(error "route cache already contained key: sr_pubkey={:?}, pr_pubkey={:?}", v.public_key, pr_pubkey);
-                rpri.touch(cur_ts);
-                return Some(f(rpri));
-            }
         }
-        self.remove_remote_private_route(&id);
-        None
     }
 
-    // peek a remote private route cache entry
+    /// Looks up an existing compiled route from the safety and private route components
-    // will not LRU entries but may expire entries and not return them if they are stale
+    pub fn lookup_compiled_route_cache(
-    /// calls a callback with the remote private route info if returned
+        &self,
-    pub fn with_peek_remote_private_route<F, R>(
+        sr_pubkey: PublicKey,
-        &mut self,
+        pr_pubkey: PublicKey,
-        cur_ts: Timestamp,
+    ) -> Option<SafetyRoute> {
-        id: &RemotePrivateRouteId,
+        let key = CompiledRouteCacheKey {
-        f: F,
+            sr_pubkey,
-    ) -> Option<R>
+            pr_pubkey,
-    where
+        };
-        F: FnOnce(&mut RemotePrivateRouteInfo) -> R,
+        self.compiled_route_cache.get(&key).cloned()
-    {
+    }
-        if let Some(rpri) = self.peek_remote_private_route_mut(&id) {
+
-            if !rpri.did_expire(cur_ts) {
+    /// When routes are dropped, they should be removed from the compiled route cache
-                rpri.touch(cur_ts);
+    fn invalidate_compiled_route_cache(&self, dead_key: &PublicKey) {
-                return Some(f(rpri));
+        let mut dead_entries = Vec::new();
+        for (k, _v) in self.compiled_route_cache.iter() {
+            if k.sr_pubkey == *dead_key || k.pr_pubkey == *dead_key {
+                dead_entries.push(k.clone());
             }
         }
-        self.remove_remote_private_route(&id);
+        for d in dead_entries {
-        None
+            self.compiled_route_cache.remove(&d);
+        }
     }
 
     /// Take the dead local and remote routes so we can update clients
-    pub fn take_dead_routes(&mut self) -> (Vec<RouteSetSpecId>, Vec<RemotePrivateRouteId>) {
+    pub fn take_dead_routes(&mut self) -> Option<(Vec<RouteId>, Vec<RouteId>)> {
         if self.dead_routes.is_empty() && self.dead_remote_routes.is_empty() {
             // Nothing to do
-            return;
+            return None;
         }
         let dead_routes = core::mem::take(&mut self.dead_routes);
         let dead_remote_routes = core::mem::take(&mut self.dead_remote_routes);
-        (dead_routes, dead_remote_routes)
+        Some((dead_routes, dead_remote_routes))
     }
 
     /// Clean up imported remote routes
     /// Resets statistics for when our node info changes
-    pub fn reset_details(&mut self) {
+    pub fn reset_remote_private_routes(&mut self) {
-        for (_k, v) in self.remote_private_route_cache {
+        // Restart stats for routes so we test the route again
-            // Restart stats for routes so we test the route again
+        for (_k, v) in self.remote_private_route_set_cache {
-            v.stats.reset();
+            v.get_stats_mut().reset();
+        }
+    }
+
+    /// Roll transfer statistics
+    pub fn roll_transfers(&mut self, last_ts: Timestamp, cur_ts: Timestamp) {
+        for (_k, v) in self.remote_private_route_set_cache {
+            v.get_stats_mut().roll_transfers(last_ts, cur_ts);
         }
     }
 }

veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs

@@ -5,15 +5,116 @@ use super::*;
 #[archive_attr(repr(C, align(8)), derive(CheckBytes))]
 pub struct RouteSpecStoreContent {
     /// All of the route sets we have allocated so far indexed by key
-    id_by_key: HashMap<PublicKey, RouteSetSpecId>,
+    id_by_key: HashMap<PublicKey, RouteId>,
     /// All of the route sets we have allocated so far
-    details: HashMap<RouteSetSpecId, RouteSetSpecDetail>,
+    details: HashMap<RouteId, RouteSetSpecDetail>,
 }
 
 impl RouteSpecStoreContent {
-    pub fn add_detail(&mut self, detail: RouteSetSpecDetail) -> RouteSetSpecId {
+    pub async fn load(routing_table: RoutingTable) -> EyreResult<RouteSpecStoreContent> {
-        // generate unique key string
+        // Deserialize what we can
-        let id = detail.make_id();
+        let table_store = routing_table.network_manager().table_store();
+        let rsstdb = table_store.open("RouteSpecStore", 1).await?;
+        let mut content: RouteSpecStoreContent =
+            rsstdb.load_rkyv(0, b"content")?.unwrap_or_default();
+
+        // Look up all route hop noderefs since we can't serialize those
+        let mut dead_ids = Vec::new();
+        for (rsid, rssd) in content.details.iter_mut() {
+            // Get best route since they all should resolve
+            let Some(pk) = rssd.get_best_route_set_key() else {
+                dead_ids.push(rsid.clone());
+                continue;
+            };
+            let Some(rsd) = rssd.get_route_by_key(&pk) else {
+                dead_ids.push(rsid.clone());
+                continue;
+            };
+            // Go through best route and resolve noderefs
+            let mut hop_node_refs = Vec::with_capacity(rsd.hops.len());
+            for h in &rsd.hops {
+                let Some(nr) = routing_table.lookup_node_ref(TypedKey::new(rsd.crypto_kind, *h)) else {
+                    dead_ids.push(rsid.clone());
+                    break;
+                };
+                hop_node_refs.push(nr);
+            }
+
+            // Apply noderefs
+            rssd.set_hop_node_refs(hop_node_refs);
+        }
+        for id in dead_ids {
+            log_rtab!(debug "no entry, killing off private route: {}", id);
+            content.remove_detail(&id);
+        }
+
+        // Load secrets from pstore
+        let pstore = routing_table.network_manager().protected_store();
+        let secret_key_map: HashMap<PublicKey, SecretKey> = pstore
+            .load_user_secret_rkyv("RouteSpecStore")
+            .await?
+            .unwrap_or_default();
+
+        // Ensure we got secret keys for all the public keys
+        let mut got_secret_key_ids = HashSet::new();
+        for (rsid, rssd) in content.details.iter_mut() {
+            let mut found_all = true;
+            for (pk, rsd) in rssd.iter_route_set_mut() {
+                if let Some(sk) = secret_key_map.get(pk) {
+                    rsd.secret_key = *sk;
+                } else {
+                    found_all = false;
+                    break;
+                }
+            }
+            if found_all {
+                got_secret_key_ids.insert(rsid.clone());
+            }
+        }
+
+        // If we missed any, nuke those route ids
+        let dead_ids: Vec<RouteId> = content
+            .details
+            .keys()
+            .filter_map(|id| {
+                if !got_secret_key_ids.contains(id) {
+                    Some(*id)
+                } else {
+                    None
+                }
+            })
+            .collect();
+        for id in dead_ids {
+            log_rtab!(debug "missing secret key, killing off private route: {}", id);
+            content.remove_detail(&id);
+        }
+
+        Ok(content)
+    }
+
+    pub async fn save(&self, routing_table: RoutingTable) -> EyreResult<()> {
+        // Save all the fields we care about to the frozen blob in table storage
+        // This skips #[with(Skip)] saving the secret keys, we save them in the protected store instead
+        let table_store = routing_table.network_manager().table_store();
+        let rsstdb = table_store.open("RouteSpecStore", 1).await?;
+        rsstdb.store_rkyv(0, b"content", self).await?;
+
+        // // Keep secrets in protected store as well
+        let pstore = routing_table.network_manager().protected_store();
+
+        let mut out: HashMap<PublicKey, SecretKey> = HashMap::new();
+        for (rsid, rssd) in self.details.iter() {
+            for (pk, rsd) in rssd.iter_route_set() {
+                out.insert(*pk, rsd.secret_key);
+            }
+        }
+
+        let _ = pstore.save_user_secret_rkyv("RouteSpecStore", &out).await?; // ignore if this previously existed or not
+
+        Ok(())
+    }
+
+    pub fn add_detail(&mut self, id: RouteId, detail: RouteSetSpecDetail) {
         assert!(!self.details.contains_key(&id));
 
         // also store in id by key table
@@ -21,38 +122,32 @@ impl RouteSpecStoreContent {
             self.id_by_key.insert(*pk, id.clone());
         }
         self.details.insert(id.clone(), detail);
-
-        id
     }
-    pub fn remove_detail(&mut self, id: &RouteSetSpecId) -> Option<RouteSetSpecDetail> {
+    pub fn remove_detail(&mut self, id: &RouteId) -> Option<RouteSetSpecDetail> {
         let detail = self.details.remove(id)?;
         for (pk, _) in detail.iter_route_set() {
             self.id_by_key.remove(&pk).unwrap();
         }
         Some(detail)
     }
-    pub fn get_detail(&self, id: &RouteSetSpecId) -> Option<&RouteSetSpecDetail> {
+    pub fn get_detail_count(&self) -> usize {
+        self.details.len()
+    }
+    pub fn get_detail(&self, id: &RouteId) -> Option<&RouteSetSpecDetail> {
         self.details.get(id)
     }
-    pub fn get_detail_mut(&mut self, id: &RouteSetSpecId) -> Option<&mut RouteSetSpecDetail> {
+    pub fn get_detail_mut(&mut self, id: &RouteId) -> Option<&mut RouteSetSpecDetail> {
         self.details.get_mut(id)
     }
-    pub fn get_id_by_key(&self, key: &PublicKey) -> Option<RouteSetSpecId> {
+    pub fn get_id_by_key(&self, key: &PublicKey) -> Option<RouteId> {
         self.id_by_key.get(key).cloned()
     }
-    pub fn iter_ids(&self) -> std::collections::hash_map::Keys<RouteSetSpecId, RouteSetSpecDetail> {
+    pub fn iter_ids(&self) -> std::collections::hash_map::Keys<RouteId, RouteSetSpecDetail> {
         self.details.keys()
     }
-    pub fn iter_details(
+    pub fn iter_details(&self) -> std::collections::hash_map::Iter<RouteId, RouteSetSpecDetail> {
-        &self,
-    ) -> std::collections::hash_map::Iter<RouteSetSpecId, RouteSetSpecDetail> {
         self.details.iter()
     }
-    pub fn iter_details_mut(
-        &mut self,
-    ) -> std::collections::hash_map::IterMut<RouteSetSpecId, RouteSetSpecDetail> {
-        self.details.iter_mut()
-    }
 
     /// Clean up local allocated routes
     /// Resets publication status and statistics for when our node info changes
@@ -65,4 +160,11 @@ impl RouteSpecStoreContent {
             v.get_stats_mut().reset();
         }
     }
+
+    /// Roll transfer statistics
+    pub fn roll_transfers(&mut self, last_ts: Timestamp, cur_ts: Timestamp) {
+        for rssd in self.details.values_mut() {
+            rssd.get_stats_mut().roll_transfers(last_ts, cur_ts);
+        }
+    }
 }

veilid-core/src/routing_table/tasks/private_route_management.rs

@@ -8,7 +8,7 @@ const BACKGROUND_SAFETY_ROUTE_COUNT: usize = 2;
 
 impl RoutingTable {
     /// Fastest routes sort
-    fn route_sort_latency_fn(a: &(TypedKey, u64), b: &(TypedKey, u64)) -> cmp::Ordering {
+    fn route_sort_latency_fn(a: &(RouteId, u64), b: &(RouteId, u64)) -> cmp::Ordering {
         let mut al = a.1;
         let mut bl = b.1;
         // Treat zero latency as uncalculated
@@ -35,14 +35,14 @@ impl RoutingTable {
     ///
     /// If a route doesn't 'need_testing', then we neither test nor drop it
     #[instrument(level = "trace", skip(self))]
-    fn get_allocated_routes_to_test(&self, cur_ts: Timestamp) -> Vec<TypedKey> {
+    fn get_allocated_routes_to_test(&self, cur_ts: Timestamp) -> Vec<RouteId> {
         let default_route_hop_count =
             self.with_config(|c| c.network.rpc.default_route_hop_count as usize);
 
         let rss = self.route_spec_store();
-        let mut must_test_routes = Vec::<TypedKey>::new();
+        let mut must_test_routes = Vec::<RouteId>::new();
-        let mut unpublished_routes = Vec::<(TypedKey, u64)>::new();
+        let mut unpublished_routes = Vec::<(RouteId, u64)>::new();
-        let mut expired_routes = Vec::<TypedKey>::new();
+        let mut expired_routes = Vec::<RouteId>::new();
         rss.list_allocated_routes(|k, v| {
             let stats = v.get_stats();
             // Ignore nodes that don't need testing
@@ -81,7 +81,7 @@ impl RoutingTable {
         }
 
         // Process dead routes
-        for r in &expired_routes {
+        for r in expired_routes {
             log_rtab!(debug "Expired route: {}", r);
             rss.release_route(r);
         }
@@ -95,7 +95,7 @@ impl RoutingTable {
     async fn test_route_set(
         &self,
         stop_token: StopToken,
-        routes_needing_testing: Vec<TypedKey>,
+        routes_needing_testing: Vec<RouteId>,
     ) -> EyreResult<()> {
         if routes_needing_testing.is_empty() {
             return Ok(());
@@ -107,43 +107,45 @@ impl RoutingTable {
         #[derive(Default, Debug)]
         struct TestRouteContext {
             failed: bool,
-            dead_routes: Vec<TypedKey>,
+            dead_routes: Vec<RouteId>,
         }
 
-        let mut unord = FuturesUnordered::new();
         let ctx = Arc::new(Mutex::new(TestRouteContext::default()));
-        for r in routes_needing_testing {
+        {
-            let rss = rss.clone();
+            let mut unord = FuturesUnordered::new();
-            let ctx = ctx.clone();
+            for r in routes_needing_testing {
-            unord.push(
+                let rss = rss.clone();
-                async move {
+                let ctx = ctx.clone();
-                    let success = match rss.test_route(&r).await {
+                unord.push(
-                        Ok(v) => v,
+                    async move {
-                        Err(e) => {
+                        let success = match rss.test_route(r).await {
-                            log_rtab!(error "Test route failed: {}", e);
+                            Ok(v) => v,
-                            ctx.lock().failed = true;
+                            Err(e) => {
+                                log_rtab!(error "Test route failed: {}", e);
+                                ctx.lock().failed = true;
+                                return;
+                            }
+                        };
+                        if success {
+                            // Route is okay, leave it alone
                             return;
                         }
-                    };
+                        // Route test failed
-                    if success {
+                        ctx.lock().dead_routes.push(r);
-                        // Route is okay, leave it alone
-                        return;
                     }
-                    // Route test failed
+                    .instrument(Span::current())
-                    ctx.lock().dead_routes.push(r);
+                    .boxed(),
-                }
+                );
-                .instrument(Span::current())
+            }
-                .boxed(),
+
-            );
+            // Wait for test_route futures to complete in parallel
+            while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {}
         }
 
-        // Wait for test_route futures to complete in parallel
-        while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {}
-
         // Process failed routes
-        let ctx = &mut *ctx.lock();
+        let ctx = Arc::try_unwrap(ctx).unwrap().into_inner();
-        for r in &ctx.dead_routes {
+        for r in ctx.dead_routes {
-            log_rtab!(debug "Dead route failed to test: {}", &r);
+            log_rtab!(debug "Dead route failed to test: {}", r);
             rss.release_route(r);
         }
 
@@ -176,13 +178,16 @@ impl RoutingTable {
                 .await?;
         }
 
-        // Ensure we have a minimum of N allocated local, unpublished routes with the default number of hops
+        // Ensure we have a minimum of N allocated local, unpublished routes with the default number of hops and all our supported crypto kinds
         let default_route_hop_count =
             self.with_config(|c| c.network.rpc.default_route_hop_count as usize);
         let mut local_unpublished_route_count = 0usize;
         let rss = self.route_spec_store();
         rss.list_allocated_routes(|_k, v| {
-            if !v.is_published() && v.hop_count() == default_route_hop_count {
+            if !v.is_published()
+                && v.hop_count() == default_route_hop_count
+                && v.get_route_set_keys().kinds() == VALID_CRYPTO_KINDS
+            {
                 local_unpublished_route_count += 1;
             }
             Option::<()>::None
@@ -196,6 +201,7 @@ impl RoutingTable {
                 // Parameters here must be the default safety route spec
                 // These will be used by test_remote_route as well
                 if let Some(k) = rss.allocate_route(
+                    &VALID_CRYPTO_KINDS,
                     Stability::default(),
                     Sequencing::default(),
                     default_route_hop_count,

veilid-core/src/rpc_processor/destination.rs

@@ -22,7 +22,7 @@ pub enum Destination {
     /// Send to private route (privateroute)
     PrivateRoute {
         /// A private route set id to send to
-        private_route: String,
+        private_route_id: RouteId,
         /// Require safety route or not
         safety_selection: SafetySelection,
     },
@@ -44,9 +44,9 @@ impl Destination {
             safety_selection: SafetySelection::Unsafe(sequencing),
         }
     }
-    pub fn private_route(private_route: PrivateRoute, safety_selection: SafetySelection) -> Self {
+    pub fn private_route(private_route_id: RouteId, safety_selection: SafetySelection) -> Self {
         Self::PrivateRoute {
-            private_route,
+            private_route_id,
             safety_selection,
         }
     }
@@ -70,10 +70,10 @@ impl Destination {
                 safety_selection,
             },
             Destination::PrivateRoute {
-                private_route,
+                private_route_id,
                 safety_selection: _,
             } => Self::PrivateRoute {
-                private_route,
+                private_route_id,
                 safety_selection,
             },
         }
@@ -91,7 +91,7 @@ impl Destination {
                 safety_selection,
             } => safety_selection,
             Destination::PrivateRoute {
-                private_route: _,
+                private_route_id: _,
                 safety_selection,
             } => safety_selection,
         }
@@ -127,7 +127,7 @@ impl fmt::Display for Destination {
                 write!(f, "{}@{}{}", target, relay, sr)
             }
             Destination::PrivateRoute {
-                private_route,
+                private_route_id,
                 safety_selection,
             } => {
                 let sr = if matches!(safety_selection, SafetySelection::Safe(_)) {
@@ -136,7 +136,7 @@ impl fmt::Display for Destination {
                     ""
                 };
 
-                write!(f, "{}{}", private_route, sr)
+                write!(f, "{}{}", private_route_id, sr)
             }
         }
     }
@@ -207,7 +207,7 @@ impl RPCProcessor {
                 }
             },
             Destination::PrivateRoute {
-                private_route,
+                private_route_id,
                 safety_selection,
             } => {
                 let Some(avoid_node_id) = private_route.first_hop_node_id() else {

veilid-core/src/rpc_processor/mod.rs

@@ -674,7 +674,7 @@ impl RPCProcessor {
                 };
             }
             Destination::PrivateRoute {
-                private_route,
+                private_route_id,
                 safety_selection,
             } => {
                 // Send to private route
@@ -726,7 +726,7 @@ impl RPCProcessor {
                 }
             }
             Destination::PrivateRoute {
-                private_route: _,
+                private_route_id: _,
                 safety_selection: _,
             } => {
                 return Ok(SenderPeerInfo::default());

veilid-core/src/rpc_processor/rpc_find_node.rs

@@ -17,7 +17,7 @@ impl RPCProcessor {
         if matches!(
             dest,
             Destination::PrivateRoute {
-                private_route: _,
+                private_route_id: _,
                 safety_selection: _
             }
         ) {

veilid-core/src/rpc_processor/rpc_signal.rs

@@ -13,7 +13,7 @@ impl RPCProcessor {
         if matches!(
             dest,
             Destination::PrivateRoute {
-                private_route: _,
+                private_route_id: _,
                 safety_selection: _
             }
         ) {

veilid-core/src/rpc_processor/rpc_status.rs

@@ -53,7 +53,7 @@ impl RPCProcessor {
                         (Some(target.clone()), routing_domain)
                     }
                     Destination::PrivateRoute {
-                        private_route: _,
+                        private_route_id: _,
                         safety_selection: _,
                     } => (None, RoutingDomain::PublicInternet),
                 };
@@ -169,7 +169,7 @@ impl RPCProcessor {
                 safety_selection: _,
             }
             | Destination::PrivateRoute {
-                private_route: _,
+                private_route_id: _,
                 safety_selection: _,
             } => {
                 // sender info is irrelevant over relays and routes

veilid-core/src/veilid_api/routing_context.rs

@@ -4,8 +4,8 @@ use super::*;
 
 #[derive(Clone, Debug)]
 pub enum Target {
-    NodeId(PublicKey),          // Node by any of its public keys
+    NodeId(PublicKey),                        // Node by any of its public keys
-    PrivateRoute(String),       // Private route by its route set id
+    PrivateRoute(RouteId),       // Remote private route by its id
 }
 
 pub struct RoutingContextInner {}
@@ -121,14 +121,13 @@ impl RoutingContext {
             Target::PrivateRoute(rsid) => {
                 // Get remote private route
                 let rss = self.api.routing_table()?.route_spec_store();
-                let Some(private_route) = rss
+                if !rss.is_valid_remote_private_route(&rsid) 
-                    .get_remote_private_route(&rsid) 
                     else {
                         apibail_key_not_found!(pr);
                     };
 
                 Ok(rpc_processor::Destination::PrivateRoute {
-                    private_route: rsid,
+                    private_route_id: rsid,
                     safety_selection: self.unlocked_inner.safety_selection,
                 })
             }

veilid-core/src/veilid_api/types.rs

@@ -279,8 +279,8 @@ pub struct VeilidStateNetwork {
 )]
 #[archive_attr(repr(C), derive(CheckBytes))]
 pub struct VeilidStateRoute {
-    pub dead_routes: Vec<PublicKey>,
+    pub dead_routes: Vec<RouteId>,
-    pub dead_remote_routes: Vec<PublicKey>,
+    pub dead_remote_routes: Vec<RouteId>,
 }
 
 #[derive(
@@ -513,7 +513,7 @@ impl SafetySelection {
 #[archive_attr(repr(C), derive(CheckBytes))]
 pub struct SafetySpec {
     /// preferred safety route set id if it still exists
-    pub preferred_route: Option<String>,
+    pub preferred_route: Option<RouteId>,
     /// must be greater than 0
     pub hop_count: usize,
     /// prefer reliability over speed