route work

veilid-core/proto/veilid.capnp

@@ -126,9 +126,11 @@ struct SignalInfoReverseConnect {
 struct RouteHopData {         
     nonce                   @0  :Nonce;                 # nonce for encrypted blob
     blob                    @1  :Data;                  # encrypted blob with ENC(nonce,DH(PK,SK))
-                                                        #   can be one of: 
-                                                        #     if more hops remain in this route: RouteHop (0 byte appended as key)
-                                                        #     if end of safety route and starting private route: PrivateRoute (1 byte appended as key)
+                                                        # if this is a safety route RouteHopData, there is a single byte tag appended to the end of the encrypted blob
+                                                        # it can be one of: 
+                                                        #     if more hops remain in this route: RouteHop (0 byte appended as tag)
+                                                        #     if end of safety route and starting private route: PrivateRoute (1 byte appended as tag)
+                                                        # if this is a private route RouteHopData, only can decode to RouteHop, no tag is appended
 }
 
 struct RouteHop {
@@ -136,7 +138,8 @@ struct RouteHop {
         nodeId              @0  :NodeID;                # node id only for established routes
         peerInfo            @1  :PeerInfo;              # full peer info for this hop to establish the route
     }
-    nextHop                 @2  :RouteHopData;          # Next hop in encrypted blob
+    nextHop                 @2  :RouteHopData;          # Optional: if the private route is a stub, it contains no route hop data, just the target node for the routed operation.
+                                                        # if this is a safety route routehop, this field is not optional and must exist
 }
 
 struct PrivateRoute {
@@ -151,7 +154,6 @@ struct SafetyRoute {
     hops :union {
         data                @2  :RouteHopData;          # safety route has more hops
         private             @3  :PrivateRoute;          # safety route has ended and private route follows
-        xxx find better representation for privateroute stub (going straight to node)
     }
 }
 

veilid-core/src/network_manager/mod.rs

@@ -1569,12 +1569,12 @@ impl NetworkManager {
         // xxx: deal with spoofing and flooding here?
 
         // Pass message to RPC system
-        rpc.enqueue_message(
+        rpc.enqueue_direct_message(
             envelope,
-            body,
             source_noderef,
             connection_descriptor,
             routing_domain,
+            body,
         )?;
 
         // Inform caller that we dealt with the envelope locally

veilid-core/src/routing_table/route_spec_store.rs

@@ -17,9 +17,9 @@ pub struct CompiledRoute {
 struct RouteSpecDetail {
     /// Secret key
     #[serde(skip)]
-    secret_key: DHTKeySecret,
+    pub secret_key: DHTKeySecret,
     /// Route hops
-    hops: Vec<DHTKey>,
+    pub hops: Vec<DHTKey>,
     /// Route noderefs
     #[serde(skip)]
     hop_node_refs: Vec<NodeRef>,
@@ -273,6 +273,9 @@ impl RouteSpecStore {
         }
     }
 
+    fn detail(&self, public_key: &DHTKey) -> Option<&RouteSpecDetail> {
+        self.content.details.get(&public_key)
+    }
     fn detail_mut(&mut self, public_key: &DHTKey) -> Option<&mut RouteSpecDetail> {
         self.content.details.get_mut(&public_key)
     }
@@ -534,6 +537,13 @@ impl RouteSpecStore {
         Ok(Some(public_key))
     }
 
+    pub fn with_route_spec_detail<F, R>(&self, public_key: &DHTKey, f: F) -> Option<R>
+    where
+        F: FnOnce(&RouteSpecDetail) -> R,
+    {
+        self.detail(&public_key).map(|rsd| f(rsd))
+    }
+
     pub fn release_route(&mut self, public_key: DHTKey) {
         if let Some(detail) = self.content.details.remove(&public_key) {
             // Remove from hop cache
@@ -770,7 +780,7 @@ impl RouteSpecStore {
                                 RouteNode::PeerInfo(pi.unwrap())
                             }
                         },
-                        next_hop: route_hop_data,
+                        next_hop: Some(route_hop_data),
                     };
 
                     // Make next blob from route hop

veilid-core/src/rpc_processor/coders/private_safety_route.rs

@@ -60,10 +60,10 @@ pub fn encode_route_hop(
             encode_peer_info(&pi, &mut pi_builder)?;
         }
     }
-
+    if let Some(rhd) = &route_hop.next_hop {
         let mut rhd_builder = builder.reborrow().init_next_hop();
-    encode_route_hop_data(&route_hop.next_hop, &mut rhd_builder)?;
-
+        encode_route_hop_data(rhd, &mut rhd_builder)?;
+    }
     Ok(())
 }
 
@@ -83,10 +83,14 @@ pub fn decode_route_hop(reader: &veilid_capnp::route_hop::Reader) -> Result<Rout
         }
     };
 
+    let next_hop = if reader.has_next_hop() {
         let rhd_reader = reader
             .get_next_hop()
             .map_err(RPCError::map_protocol("invalid next hop in route hop"))?;
-    let next_hop = decode_route_hop_data(&rhd_reader)?;
+        Some(decode_route_hop_data(&rhd_reader)?)
+    } else {
+        None
+    };
 
     Ok(RouteHop { node, next_hop })
 }

veilid-core/src/rpc_processor/mod.rs

@@ -40,23 +40,48 @@ use stop_token::future::FutureExt;
 
 type OperationId = u64;
 
-/// The decoded header of an RPC message
 #[derive(Debug, Clone)]
-struct RPCMessageHeader {
-    /// Time the message was received, not sent
-    timestamp: u64,
+struct RPCMessageHeaderDetailDirect {
     /// The decoded header of the envelope
     envelope: Envelope,
-    /// The length in bytes of the rpc message body
-    body_len: u64,
     /// The noderef of the peer that sent the message (not the original sender). Ensures node doesn't get evicted from routing table until we're done with it
     peer_noderef: NodeRef,
     /// The connection from the peer sent the message (not the original sender)
     connection_descriptor: ConnectionDescriptor,
     /// The routing domain the message was sent through
     routing_domain: RoutingDomain,
-    // The private route the message was received through
-    //private_route: Option<DHTKey>,
+}
+
+#[derive(Debug, Clone)]
+struct RPCMessageHeaderDetailPrivateRoute {
+    
+    /// The private route we received the rpc over
+    private_route: DHTKey,
+    // The safety selection for replying to this private routed rpc
+    safety_selection: SafetySelection,
+}
+
+#[derive(Debug, Clone)]
+enum RPCMessageHeaderDetail {
+    Direct(RPCMessageHeaderDetailDirect),
+    PrivateRoute(RPCMessageHeaderDetailPrivateRoute),
+}
+
+/// The decoded header of an RPC message
+#[derive(Debug, Clone)]
+struct RPCMessageHeader {
+    
+    version: u8,
+    min_version: u8,
+    max_version: u8,
+    timestamp: u64,???? do we need a header for private routed messages? write process_rpc_message
+
+    /// Time the message was received, not sent
+    timestamp: u64,
+    /// The length in bytes of the rpc message body
+    body_len: u64,
+    /// The header detail depending on which way the message was received
+    detail: RPCMessageHeaderDetail,
 }
 
 #[derive(Debug)]
@@ -532,7 +557,17 @@ impl RPCProcessor {
                     SafetySelection::Safe(_) => {
                         // No private route was specified for the request
                         // but we are using a safety route, so we must create an empty private route
-                        let private_route = PrivateRoute::new_stub(node_id);
+                        let peer_info = match node_ref.make_peer_info(RoutingDomain::PublicInternet)
+                        {
+                            None => {
+                                return Ok(NetworkResult::no_connection_other(
+                                    "No PublicInternet peer info for stub private route",
+                                ))
+                            }
+                            Some(pi) => pi,
+                        };
+                        let private_route =
+                            PrivateRoute::new_stub(node_id, RouteNode::PeerInfo(peer_info));
 
                         // Wrap with safety route
                         out = self.wrap_with_route(safety_selection, private_route, message)?;
@@ -719,7 +754,7 @@ impl RPCProcessor {
     }
 
     // Convert the 'RespondTo' into a 'Destination' for a response
-    fn get_respond_to_destination(&self, request: &RPCMessage) -> Destination {
+    fn get_respond_to_destination(&self, request: &RPCMessage) -> NetworkResult<Destination> {
         // Get the question 'respond to'
         let respond_to = match request.operation.kind() {
             RPCOperationKind::Question(q) => q.respond_to(),
@@ -731,29 +766,47 @@ impl RPCProcessor {
         // To where should we respond?
         match respond_to {
             RespondTo::Sender => {
+                // Parse out the header detail from the question
+                let detail = match &request.header.detail {
+                    RPCMessageHeaderDetail::Direct(detail) => detail,
+                    RPCMessageHeaderDetail::PrivateRoute(_) => {
+                        // If this was sent via a private route, we don't know what the sender was, so drop this
+                        return NetworkResult::invalid_message(
+                            "not responding directly to question from private route",
+                        );
+                    }
+                };
+
                 // Reply directly to the request's source
-                let sender_id = request.header.envelope.get_sender_id();
+                let sender_id = detail.envelope.get_sender_id();
 
                 // This may be a different node's reference than the 'sender' in the case of a relay
-                let peer_noderef = request.header.peer_noderef.clone();
+                let peer_noderef = detail.peer_noderef.clone();
 
                 // If the sender_id is that of the peer, then this is a direct reply
                 // else it is a relayed reply through the peer
                 if peer_noderef.node_id() == sender_id {
-                    Destination::direct(peer_noderef)
+                    NetworkResult::value(Destination::direct(peer_noderef))
                 } else {
-                    Destination::relay(peer_noderef, sender_id)
+                    NetworkResult::value(Destination::relay(peer_noderef, sender_id))
                 }
             }
-            //xxx needs to know what route the request came in on in order to reply over that same route as the preferred safety route
-            RespondTo::PrivateRoute(pr) => Destination::private_route(
+            RespondTo::PrivateRoute(pr) => {
+                let detail = match &request.header.detail {
+                    RPCMessageHeaderDetail::Direct(_) => {
+                        // If this was sent directly, don't respond to a private route as this could give away this node's safety routes
+                        return NetworkResult::invalid_message(
+                            "not responding to private route from direct question",
+                        );
+                    }
+                    RPCMessageHeaderDetail::PrivateRoute(detail) => detail,
+                };
+
+                NetworkResult::value(Destination::private_route(
                     pr.clone(),
-                request
-                    .header
-                    .connection_descriptor
-                    .protocol_type()
-                    .is_connection_oriented(),
-            ),
+                    detail.safety_selection.clone(),
+                ))
+            }
         }
     }
 
@@ -766,7 +819,7 @@ impl RPCProcessor {
         answer: RPCAnswer,
     ) -> Result<NetworkResult<()>, RPCError> {
         // Extract destination from respond_to
-        let dest = self.get_respond_to_destination(&request);
+        let dest = network_result_try!(self.get_respond_to_destination(&request));
 
         // Get sender info if we should send that
         let opt_sender_info = self.get_sender_signed_node_info(&dest);
@@ -956,22 +1009,57 @@ impl RPCProcessor {
     }
 
     #[instrument(level = "trace", skip(self, body), err)]
-    pub fn enqueue_message(
+    pub fn enqueue_direct_message(
         &self,
         envelope: Envelope,
-        body: Vec<u8>,
         peer_noderef: NodeRef,
         connection_descriptor: ConnectionDescriptor,
         routing_domain: RoutingDomain,
+        body: Vec<u8>,
     ) -> EyreResult<()> {
         let msg = RPCMessageEncoded {
             header: RPCMessageHeader {
-                timestamp: intf::get_timestamp(),
+                detail: RPCMessageHeaderDetail::Direct(RPCMessageHeaderDetailDirect {
                     envelope,
-                body_len: body.len() as u64,
                     peer_noderef,
                     connection_descriptor,
                     routing_domain,
+                }),
+                timestamp: intf::get_timestamp(),
+                body_len: body.len() as u64,
+            },
+            data: RPCMessageData { contents: body },
+        };
+        let send_channel = {
+            let inner = self.inner.lock();
+            inner.send_channel.as_ref().unwrap().clone()
+        };
+        let span_id = Span::current().id();
+        send_channel
+            .try_send((span_id, msg))
+            .wrap_err("failed to enqueue received RPC message")?;
+        Ok(())
+    }
+
+    #[instrument(level = "trace", skip(self, body), err)]
+    pub fn enqueue_private_route_message(
+        &self,
+
+        xx need rpc version somewhere! the rpc version to decode should be packaged in with the body, and the allocator should ensure the version is compatible at the end node. can append to body, and then pop the last u8.
+        xx try to write the whole process_rpc_message pipeline first
+
+        private_route: DHTKey,
+        safety_selection: SafetySelection,
+        body: Vec<u8>,
+    ) -> EyreResult<()> {
+        let msg = RPCMessageEncoded {
+            header: RPCMessageHeader {
+                detail: RPCMessageHeaderDetail::PrivateRoute(RPCMessageHeaderDetailPrivateRoute {
+                    private_route,
+                    safety_selection,
+                }),
+                timestamp: intf::get_timestamp(),
+                body_len: body.len() as u64,
             },
             data: RPCMessageData { contents: body },
         };

veilid-core/src/rpc_processor/rpc_route.rs

@@ -1,11 +1,27 @@
 use super::*;
 
 impl RPCProcessor {
+    #[instrument(level = "trace", skip_all, err)]
     async fn process_route_safety_route_hop(
         &self,
-        route: &RPCOperationRoute,
+        route: RPCOperationRoute,
         route_hop: RouteHop,
     ) -> Result<(), RPCError> {
+        // Make sure hop count makes sense
+        if route.safety_route.hop_count as usize > self.unlocked_inner.max_route_hop_count {
+            return Err(RPCError::protocol(
+                "Safety route hop count too high to process",
+            ));
+        }
+        if route.safety_route.hop_count == 0 {
+            return Err(RPCError::protocol(
+                "Safety route hop count should not be zero if there are more hops",
+            ));
+        }
+        if route_hop.next_hop.is_none() {
+            return Err(RPCError::protocol("Safety route hop must have next hop"));
+        }
+
         // Get next hop node ref
         let next_hop_nr = match route_hop.node {
             RouteNode::NodeId(id) => {
@@ -37,65 +53,214 @@ impl RPCProcessor {
             safety_route: SafetyRoute {
                 public_key: route.safety_route.public_key,
                 hop_count: route.safety_route.hop_count - 1,
-                hops: SafetyRouteHops::Data(route_hop.next_hop),
+                hops: SafetyRouteHops::Data(route_hop.next_hop.unwrap()),
             },
             operation: route.operation,
         };
         let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route));
 
         // Send the next route statement
-        network_result_try!(
+        network_result_value_or_log!(debug
             self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt)
-                .await?
+                .await? => {
+                    return Err(RPCError::network("unable to send route statement for next safety route hop"));
+                }
         );
-
         Ok(())
     }
 
-    async fn process_route_safety_route_private_route_hop(
+    #[instrument(level = "trace", skip_all, err)]
+    async fn process_route_private_route_hop(
         &self,
-        route: &RPCOperationRoute,
-        private_route: &PrivateRoute,
+        route: RPCOperationRoute,
+        private_route: PrivateRoute,
     ) -> Result<(), RPCError> {
+        // Make sure hop count makes sense
+        if route.safety_route.hop_count != 0 {
+            return Err(RPCError::protocol(
+                "Safety hop count should be zero if switched to private route",
+            ));
+        }
+        if private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count {
+            return Err(RPCError::protocol(
+                "Private route hop count too high to process",
+            ));
+        }
+        if private_route.hop_count == 0 {
+            return Err(RPCError::protocol(
+                "Private route hop count should not be zero if there are more hops",
+            ));
+        }
+
+        // Get private route first hop (this is validated to not be None before calling this function)
+        let first_hop = private_route.first_hop.as_ref().unwrap();
+
+        // Get next hop node ref
+        let next_hop_nr = match &first_hop.node {
+            RouteNode::NodeId(id) => {
                 //
-        let route_hop = private_route.first_hop.unwrap();
+                self.routing_table
+                    .lookup_node_ref(id.key)
+                    .ok_or_else(|| RPCError::network(format!("node hop {} not found", id.key)))
+            }
+            RouteNode::PeerInfo(pi) => {
+                //
+                self.routing_table
+                    .register_node_with_signed_node_info(
+                        RoutingDomain::PublicInternet,
+                        pi.node_id.key,
+                        pi.signed_node_info.clone(),
+                        false,
+                    )
+                    .ok_or_else(|| {
+                        RPCError::network(format!(
+                            "node hop {} could not be registered",
+                            pi.node_id.key
+                        ))
+                    })
+            }
+        }?;
 
         // Pass along the route
         let next_hop_route = RPCOperationRoute {
             safety_route: SafetyRoute {
                 public_key: route.safety_route.public_key,
                 hop_count: 0,
-                hops: SafetyRouteHops::PrivateRoute(Private),
+                hops: SafetyRouteHops::Private(private_route),
             },
             operation: route.operation,
         };
         let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route));
 
         // Send the next route statement
-        network_result_try!(
+        network_result_value_or_log!(debug
             self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt)
-                .await?
+                .await? => {
+                    return Err(RPCError::network("unable to send route statement for private route hop"));
+                }
         );
 
         Ok(())
     }
+
+    #[instrument(level = "trace", skip_all, err)]
     async fn process_routed_operation(
         &self,
-        route: &RPCOperationRoute,
+        sender_id: DHTKey,
+        route: RPCOperationRoute,
         private_route: &PrivateRoute,
     ) -> Result<(), RPCError> {
-        //
+        // Make sure hop count makes sense
+        if route.safety_route.hop_count != 0 {
+            return Err(RPCError::protocol(
+                "Safety hop count should be zero if switched to private route",
+            ));
+        }
+        if private_route.hop_count != 0 {
+            return Err(RPCError::protocol(
+                "Private route hop count should be zero if we are at the end",
+            ));
+        }
+
+        let routed_operation = &route.operation;
+
+        // Get sequencing preference
+        if route.
+
+        // If the private route public key is our node id, then this was sent via safety route to our node directly
+        // so there will be no signatures to validate
+        let opt_pr_info = if private_route.public_key == self.routing_table.node_id() {
+            // the private route was a stub to our own node's secret
+            // return our secret key
+            Some((
+                self.routing_table.node_id_secret(), // Figure out how we'd reply to this if it were a question
+                SafetySelection::Unsafe(sequencing),
+            ))
+        } else {
+            // Look up the private route and ensure it's one in our spec store
+            let opt_signatures_valid = self.routing_table.with_route_spec_store(|rss, rti| {
+                rss.with_route_spec_detail(&private_route.public_key, |rsd| {
+                    // Ensure we have the right number of signatures
+                    if routed_operation.signatures.len() != rsd.hops.len() - 1 {
+                        // Wrong number of signatures
+                        log_rpc!(debug "wrong number of signatures ({} should be {}) for routed operation on private route {}", routed_operation.signatures.len(), rsd.hops.len() - 1, private_route.public_key);    
+                        return None;
+                    }
+                    // Validate signatures to ensure the route was handled by the nodes and not messed with
+                    for (hop_n, hop_public_key) in rsd.hops.iter().enumerate() {
+                        // The last hop is not signed, as the whole packet is signed
+                        if hop_n == routed_operation.signatures.len() {
+                            // Verify the node we received the routed operation from is the last hop in our route
+                            if *hop_public_key != sender_id {
+                                log_rpc!(debug "received routed operation from the wrong hop ({} should be {}) on private route {}", hop_public_key.encode(), sender_id.encode(), private_route.public_key);    
+                                return None;
+                            }
+                        } else {
+                            // Verify a signature for a hop node along the route
+                            if let Err(e) = verify(
+                                hop_public_key,
+                                &routed_operation.data,
+                                &routed_operation.signatures[hop_n],
+                            ) {
+                                log_rpc!(debug "failed to verify signature for hop {} at {} on private route {}", hop_n, hop_public_key, private_route.public_key);
+                                return None;
+                            }
+                        }
+                    }
+                    // Correct signatures
+                    Some((
+                        rsd.secret_key,
+                        SafetySelection::Safe(SafetySpec { preferred_route: todo!(), hop_count: todo!(), stability: todo!(), sequencing: todo!() })
+                    ))
+                })
+            });
+            opt_signatures_valid.ok_or_else(|| {
+                RPCError::protocol("routed operation received on unallocated private route")
+            })?
+        };
+        if opt_pr_info.is_none() {
+            return Err(RPCError::protocol(
+                "signatures did not validate for private route",
+            ));
+        }
+        let (secret_key, safety_selection) = opt_pr_info.unwrap();
+
+        // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret)
+        // xxx: punish nodes that send messages that fail to decrypt eventually
+        let dh_secret = self
+            .crypto
+            .cached_dh(&route.safety_route.public_key, &secret_key)
+            .map_err(RPCError::protocol)?;
+        let body = Crypto::decrypt_aead(
+            &routed_operation.data,
+            &routed_operation.nonce,
+            &dh_secret,
+            None,
+        )
+        .map_err(RPCError::map_internal(
+            "decryption of routed operation failed",
+        ))?;
+
+        // Pass message to RPC system
+        self.enqueue_private_route_message(private_route.public_key, safety_selection, body)
+            .map_err(RPCError::internal)?;
+
         Ok(())
     }
 
-    #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)]
+    #[instrument(level = "trace", skip(self, msg), err)]
     pub(crate) async fn process_route(&self, msg: RPCMessage) -> Result<(), RPCError> {
         // xxx do not process latency for routed messages
-        // tracing::Span::current().record("res", &tracing::field::display(res));
+        
+        // Get header detail, must be direct and not inside a route itself
+        let (envelope, peer_noderef, connection_descriptor, routing_domain) = match msg.header.detail {
+            RPCMessageHeaderDetail::Direct { envelope, peer_noderef, connection_descriptor, routing_domain } => (envelope, peer_noderef, connection_descriptor, routing_domain),
+            RPCMessageHeaderDetail::PrivateRoute { private_route, safety_selection } => { return Err(RPCError::protocol("route operation can not be inside route")) },
+        };
 
         // Get the statement
-        let route = match msg.operation.kind() {
-            RPCOperationKind::Statement(s) => match s.detail() {
+        let route = match msg.operation.into_kind() {
+            RPCOperationKind::Statement(s) => match s.into_detail() {
                 RPCStatementDetail::Route(s) => s,
                 _ => panic!("not a route statement"),
             },
@@ -103,9 +268,9 @@ impl RPCProcessor {
         };
 
         // See what kind of safety route we have going on here
-        match &route.safety_route.hops {
+        match route.safety_route.hops {
             // There is a safety route hop
-            SafetyRouteHops::Data(d) => {
+            SafetyRouteHops::Data(ref d) => {
                 // See if this is last hop in safety route, if so, we're decoding a PrivateRoute not a RouteHop
                 let (blob_tag, blob_data) = if let Some(b) = d.blob.last() {
                     (*b, &d.blob[0..d.blob.len() - 1])
@@ -120,7 +285,9 @@ impl RPCProcessor {
                     .cached_dh(&route.safety_route.public_key, &node_id_secret)
                     .map_err(RPCError::protocol)?;
                 let dec_blob_data = Crypto::decrypt_aead(blob_data, &d.nonce, &dh_secret, None)
-                    .map_err(RPCError::map_internal("encryption failed"))?;
+                    .map_err(RPCError::map_internal(
+                        "decryption of safety route hop failed",
+                    ))?;
                 let dec_blob_reader = capnp::message::Reader::new(
                     RPCMessageData {
                         contents: dec_blob_data,
@@ -129,7 +296,7 @@ impl RPCProcessor {
                 );
 
                 // Decode the blob appropriately
-                if blob_tag == 0 {
+                if blob_tag == 1 {
                     // PrivateRoute
                     let private_route = {
                         let pr_reader = dec_blob_reader
@@ -138,44 +305,21 @@ impl RPCProcessor {
                         decode_private_route(&pr_reader)?
                     };
 
-                    // Make sure hop count makes sense
-                    if route.safety_route.hop_count as usize != 0 {
-                        return Err(RPCError::protocol(
-                            "Safety hop count should be zero if switched to private route",
-                        ));
-                    }
-
                     // Get the next hop node ref
                     if private_route.first_hop.is_some() {
-                        // Make sure hop count makes sense
-                        if private_route.hop_count as usize
-                            > self.unlocked_inner.max_route_hop_count
-                        {
-                            return Err(RPCError::protocol(
-                                "Private route hop count too high to process",
-                            ));
-                        }
-                        if private_route.hop_count == 0 {
-                            return Err(RPCError::protocol(
-                                "Private route hop count should not be zero if there are more hops",
-                            ));
-                        }
-
                         // Switching to private route from safety route
-                        self.process_route_safety_route_private_route_hop(route, &private_route)
+                        self.process_route_private_route_hop(route, private_route)
                             .await?;
                     } else {
-                        // Make sure hop count makes sense
-                        if private_route.hop_count != 0 {
-                            return Err(RPCError::protocol(
-                                "Private route hop count should be zero if we are at the end",
-                            ));
+                        // Private route is empty, process routed operation
+                        self.process_routed_operation(
+                            envelope.get_sender_id(),
+                            route,
+                            &private_route,
+                        )
+                        .await?;
                     }
-
-                        // Private route was a stub, process routed operation
-                        self.process_routed_operation(route, &private_route).await?;
-                    }
-                } else if blob_tag == 1 {
+                } else if blob_tag == 0 {
                     // RouteHop
                     let route_hop = {
                         let rh_reader = dec_blob_reader
@@ -184,54 +328,66 @@ impl RPCProcessor {
                         decode_route_hop(&rh_reader)?
                     };
 
-                    // Make sure hop count makes sense
-                    if route.safety_route.hop_count as usize
-                        > self.unlocked_inner.max_route_hop_count
-                    {
-                        return Err(RPCError::protocol(
-                            "Safety route hop count too high to process",
-                        ));
-                    }
-                    if route.safety_route.hop_count == 0 {
-                        return Err(RPCError::protocol(
-                            "Safety route hop count should not be zero if there are more hops",
-                        ));
-                    }
-
                     self.process_route_safety_route_hop(route, route_hop)
                         .await?;
                 } else {
                     return Err(RPCError::protocol("invalid blob tag"));
                 }
             }
-            // Safety route has ended, now do private route
-            SafetyRouteHops::Private(private_route) => {
-                if private_route.first_hop.is_some() {
-                    // Make sure hop count makes sense
-                    if private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count {
-                        return Err(RPCError::protocol(
-                            "Private route hop count too high to process",
-                        ));
-                    }
-                    if private_route.hop_count == 0 {
-                        return Err(RPCError::protocol(
-                            "Private route hop count should not be zero if there are more hops",
-                        ));
-                    }
+            // No safety route left, now doing private route
+            SafetyRouteHops::Private(ref private_route) => {
+                if let Some(first_hop) = &private_route.first_hop {
+                    // See if we have a next hop to send to
+                    let opt_next_first_hop = if let Some(next_hop) = &first_hop.next_hop {
+                        // Decrypt the blob with DEC(nonce, DH(the PR's public key, this hop's secret)
+                        let node_id_secret = self.routing_table.node_id_secret();
+                        let dh_secret = self
+                            .crypto
+                            .cached_dh(&private_route.public_key, &node_id_secret)
+                            .map_err(RPCError::protocol)?;
+                        let dec_blob_data =
+                            Crypto::decrypt_aead(&next_hop.blob, &next_hop.nonce, &dh_secret, None)
+                                .map_err(RPCError::map_internal(
+                                    "decryption of private route hop failed",
+                                ))?;
+                        let dec_blob_reader = capnp::message::Reader::new(
+                            RPCMessageData {
+                                contents: dec_blob_data,
+                            },
+                            Default::default(),
+                        );
 
-                    // There are some hops left
-                    self.process_route_safety_route_private_route_hop(route, private_route)
+                        // Decode next RouteHop
+                        let route_hop = {
+                            let rh_reader = dec_blob_reader
+                                .get_root::<veilid_capnp::route_hop::Reader>()
+                                .map_err(RPCError::protocol)?;
+                            decode_route_hop(&rh_reader)?
+                        };
+                        Some(route_hop)
+                    } else {
+                        // If the first hop has no RouteHopData, then this is a stub private route
+                        // and we should just pass the operation to its final destination with
+                        // an empty safety and private route
+                        None
+                    };
+
+                    // Make next PrivateRoute and pass it on
+                    let private_route = PrivateRoute {
+                        public_key: private_route.public_key,
+                        hop_count: private_route.hop_count - 1,
+                        first_hop: opt_next_first_hop,
+                    };
+                    self.process_route_private_route_hop(route, private_route)
                         .await?;
                 } else {
-                    // Make sure hop count makes sense
-                    if private_route.hop_count != 0 {
-                        return Err(RPCError::protocol(
-                            "Private route hop count should be zero if we are at the end",
-                        ));
-                    }
-
                     // No hops left, time to process the routed operation
-                    self.process_routed_operation(route, private_route).await?;
+                    self.process_routed_operation(
+                        msg.header.envelope.get_sender_id(),
+                        route,
+                        private_route,
+                    )
+                    .await?;
                 }
             }
         }

veilid-core/src/veilid_api/privacy.rs

@@ -30,7 +30,7 @@ impl fmt::Display for RouteNode {
 #[derive(Clone, Debug)]
 pub struct RouteHop {
     pub node: RouteNode,
-    pub next_hop: RouteHopData,
+    pub next_hop: Option<RouteHopData>,
 }
 
 #[derive(Clone, Debug)]
@@ -41,13 +41,29 @@ pub struct PrivateRoute {
 }
 
 impl PrivateRoute {
-    pub fn new_stub(public_key: DHTKey) -> Self {
+    /// Empty private route is the form used when receiving the last hop
+    pub fn new_empty(public_key: DHTKey) -> Self {
         Self {
             public_key,
             hop_count: 0,
             first_hop: None,
         }
     }
+    /// Stub route is the form used when no privacy is required, but you need to specify the destination for a safety route
+    pub fn new_stub(public_key: DHTKey, node: RouteNode) -> Self {
+        Self {
+            public_key,
+            hop_count: 1,
+            first_hop: Some(RouteHop {
+                node,
+                next_hop: None,
+            }),
+        }
+    }
+    /// Switch from full node info to simple node id
+    /// Only simplified single hop, primarily useful for stubs
+    /// Published routes with >= 1 hops should be in NodeId form already, as they have
+    /// already been connectivity-verified by the time the route is published
     pub fn simplify(self) -> Self {
         Self {
             public_key: self.public_key,