fixes

veilid-core/src/routing_table/privacy.rs

@@ -21,6 +21,43 @@ pub enum RouteNode {
     PeerInfo(PeerInfo),
 }
 
+impl RouteNode {
+    pub fn node_ref(
+        &self,
+        routing_table: RoutingTable,
+        crypto_kind: CryptoKind,
+    ) -> Option<NodeRef> {
+        match self {
+            RouteNode::NodeId(id) => {
+                //
+                routing_table.lookup_node_ref(TypedKey::new(crypto_kind, id))
+            }
+            RouteNode::PeerInfo(pi) => {
+                //
+                routing_table.register_node_with_peer_info(
+                    RoutingDomain::PublicInternet,
+                    pi.clone(),
+                    false,
+                )
+            }
+        }
+    }
+
+    pub fn describe(&self, crypto_kind: CryptoKind) -> String {
+        match self {
+            RouteNode::NodeId(id) => {
+                format!("{}", TypedKey::new(crypto_kind, id))
+            }
+            RouteNode::PeerInfo(pi) => match pi.node_ids.get(crypto_kind) {
+                Some(id) => format!("{}", TypedKey::new(crypto_kind, id)),
+                None => {
+                    format!("({})?{}", crypto_kind, pi.node_ids)
+                }
+            },
+        }
+    }
+}
+
 /// An unencrypted private/safety route hop
 #[derive(Clone, Debug)]
 pub struct RouteHop {

veilid-core/src/routing_table/route_spec_store/route_spec_store.rs

@@ -870,7 +870,7 @@ impl RouteSpecStore {
                 };
 
                 let opt_first_hop = match pr_first_hop_node {
-                    RouteNode::NodeId(id) => rti.lookup_any_node_ref(routing_table.clone(), id),
+                    RouteNode::NodeId(id) => rti.lookup_node_ref(routing_table.clone(), TypedKey::new(crypto_kind, id)),
                     RouteNode::PeerInfo(pi) => rti.register_node_with_peer_info(
                         routing_table.clone(),
                         RoutingDomain::PublicInternet,

veilid-core/src/rpc_processor/mod.rs

@@ -52,6 +52,8 @@ struct RPCMessageHeaderDetailDirect {
 /// Header details for rpc messages received over only a safety route but not a private route
 #[derive(Debug, Clone)]
 struct RPCMessageHeaderDetailSafetyRouted {
+    /// Direct header
+    direct: RPCMessageHeaderDetailDirect,
     /// Remote safety route used
     remote_safety_route: PublicKey,
     /// The sequencing used for this route
@@ -61,6 +63,8 @@ struct RPCMessageHeaderDetailSafetyRouted {
 /// Header details for rpc messages received over a private route
 #[derive(Debug, Clone)]
 struct RPCMessageHeaderDetailPrivateRouted {
+    /// Direct header
+    direct: RPCMessageHeaderDetailDirect,
     /// Remote safety route used (or possibly node id the case of no safety route)
     remote_safety_route: PublicKey,
     /// The private route we received the rpc over
@@ -92,8 +96,8 @@ impl RPCMessageHeader {
     pub fn crypto_kind(&self) -> CryptoKind {
         match &self.detail {
             RPCMessageHeaderDetail::Direct(d) => d.envelope.get_crypto_kind(),
-            RPCMessageHeaderDetail::SafetyRouted(s) => s.remote_safety_route.,
+            RPCMessageHeaderDetail::SafetyRouted(s) => s.direct.envelope.get_crypto_kind(),
-            RPCMessageHeaderDetail::PrivateRouted(p) => todo!(),
+            RPCMessageHeaderDetail::PrivateRouted(p) => p.direct.envelope.get_crypto_kind(),
         }
     }
 }
@@ -528,7 +532,8 @@ impl RPCProcessor {
         let dh_secret = vcrypto
             .cached_dh(&pr_pubkey, &compiled_route.secret)
             .map_err(RPCError::map_internal("dh failed"))?;
-        let enc_msg_data = vcrypto.encrypt_aead(&message_data, &nonce, &dh_secret, None)
+        let enc_msg_data = vcrypto
+            .encrypt_aead(&message_data, &nonce, &dh_secret, None)
             .map_err(RPCError::map_internal("encryption failed"))?;
 
         // Make the routed operation
@@ -541,8 +546,8 @@ impl RPCProcessor {
             safety_route: compiled_route.safety_route,
             operation,
         };
-        let ssni_route = self
+        let ssni_route =
-            .get_sender_peer_info(&Destination::direct(compiled_route.first_hop.clone()))?;
+            self.get_sender_peer_info(&Destination::direct(compiled_route.first_hop.clone()))?;
         let operation = RPCOperation::new_statement(
             RPCStatement::new(RPCStatementDetail::Route(route_operation)),
             ssni_route,
@@ -653,7 +658,8 @@ impl RPCProcessor {
                         // No private route was specified for the request
                         // but we are using a safety route, so we must create an empty private route
                         // Destination relay is ignored for safety routed operations
-                        let peer_info = match destination_node_ref.make_peer_info(RoutingDomain::PublicInternet)
+                        let peer_info = match destination_node_ref
+                            .make_peer_info(RoutingDomain::PublicInternet)
                         {
                             None => {
                                 return Ok(NetworkResult::no_connection_other(
@@ -662,8 +668,10 @@ impl RPCProcessor {
                             }
                             Some(pi) => pi,
                         };
-                        let private_route =
+                        let private_route = PrivateRoute::new_stub(
-                            PrivateRoute::new_stub(destination_node_ref.best_node_id(), RouteNode::PeerInfo(peer_info));
+                            destination_node_ref.best_node_id(),
+                            RouteNode::PeerInfo(peer_info),
+                        );
 
                         // Wrap with safety route
                         out = self.wrap_with_route(
@@ -749,10 +757,7 @@ impl RPCProcessor {
             return Ok(SenderPeerInfo::new_no_peer_info(target_node_info_ts));
         }
 
-        Ok(SenderPeerInfo::new(
+        Ok(SenderPeerInfo::new(own_peer_info, target_node_info_ts))
-            own_peer_info,
-            target_node_info_ts,
-        ))
     }
 
     /// Record failure to send to node or route
@@ -1194,7 +1199,10 @@ impl RPCProcessor {
                 let routing_domain = detail.routing_domain;
 
                 // Decode the operation
-                let sender_node_id = TypedKey::new(detail.envelope.get_crypto_kind(), detail.envelope.get_sender_id());
+                let sender_node_id = TypedKey::new(
+                    detail.envelope.get_crypto_kind(),
+                    detail.envelope.get_sender_id(),
+                );
 
                 // Decode the RPC message
                 let operation = {
@@ -1386,6 +1394,7 @@ impl RPCProcessor {
     #[instrument(level = "trace", skip(self, body), err)]
     pub fn enqueue_safety_routed_message(
         &self,
+        direct: RPCMessageHeaderDetailDirect,
         remote_safety_route: PublicKey,
         sequencing: Sequencing,
         body: Vec<u8>,
@@ -1393,6 +1402,7 @@ impl RPCProcessor {
         let msg = RPCMessageEncoded {
             header: RPCMessageHeader {
                 detail: RPCMessageHeaderDetail::SafetyRouted(RPCMessageHeaderDetailSafetyRouted {
+                    direct,
                     remote_safety_route,
                     sequencing,
                 }),
@@ -1415,6 +1425,7 @@ impl RPCProcessor {
     #[instrument(level = "trace", skip(self, body), err)]
     pub fn enqueue_private_routed_message(
         &self,
+        direct: RPCMessageHeaderDetailDirect,
         remote_safety_route: PublicKey,
         private_route: PublicKey,
         safety_spec: SafetySpec,
@@ -1424,6 +1435,7 @@ impl RPCProcessor {
             header: RPCMessageHeader {
                 detail: RPCMessageHeaderDetail::PrivateRouted(
                     RPCMessageHeaderDetailPrivateRouted {
+                        direct,
                         remote_safety_route,
                         private_route,
                         safety_spec,

veilid-core/src/rpc_processor/rpc_route.rs

@@ -26,32 +26,12 @@ impl RPCProcessor {
         }
 
         // Get next hop node ref
-        let mut next_hop_nr = match route_hop.node {
+        let Some(mut next_hop_nr) = route_hop.node.node_ref(self.routing_table.clone(), safety_route.public_key.kind) else {
-            RouteNode::NodeId(id) => {
+            return Err(RPCError::network(format!(
-                //
+                "could not get route node hop ref: {}",
-                let Some(nr) = self.routing_table.lookup_node_ref(id.key) else {
+                route_hop.node.describe(safety_route.public_key.kind)
-                    return Ok(NetworkResult::invalid_message(format!("node hop {} not found", id.key)));
-                };
-                nr
-            }
-            RouteNode::PeerInfo(pi) => {
-                //
-                let Some(nr) = self.routing_table
-                    .register_node_with_peer_info(
-                        RoutingDomain::PublicInternet,
-                        pi.node_id.key,
-                        pi.signed_node_info,
-                        false,
-                    ) else 
-                    {
-                        return Ok(NetworkResult::invalid_message(format!(
-                            "node hop {} could not be registered",
-                            pi.node_id.key
             )));
         };
-                nr
-            }
-        };
 
         // Apply sequencing preference
         next_hop_nr.set_sequencing(routed_operation.sequencing);
@@ -88,30 +68,12 @@ impl RPCProcessor {
         }
 
         // Get next hop node ref
-        let mut next_hop_nr = match &next_route_node {
+        let Some(mut next_hop_nr) = next_route_node.node_ref(self.routing_table.clone(), safety_route_public_key.kind) else {
-            RouteNode::NodeId(id) => {
+            return Err(RPCError::network(format!(
-                //
+                "could not get route node hop ref: {}",
-                self.routing_table
+                next_route_node.describe(safety_route_public_key.kind)
-                    .lookup_node_ref(id.key)
+            )));
-                    .ok_or_else(|| RPCError::network(format!("node hop {} not found", id.key)))
+        };
-            }
-            RouteNode::PeerInfo(pi) => {
-                //
-                self.routing_table
-                    .register_node_with_peer_info(
-                        RoutingDomain::PublicInternet,
-                        pi.node_id.key,
-                        pi.signed_node_info.clone(),
-                        false,
-                    )
-                    .ok_or_else(|| {
-                        RPCError::network(format!(
-                            "node hop {} could not be registered",
-                            pi.node_id.key
-                        ))
-                    })
-            }
-        }?;
 
         // Apply sequencing preference
         next_hop_nr.set_sequencing(routed_operation.sequencing);
@@ -140,19 +102,18 @@ impl RPCProcessor {
     #[instrument(level = "trace", skip_all, err)]
     fn process_safety_routed_operation(
         &self,
-        _detail: RPCMessageHeaderDetailDirect,
+        detail: RPCMessageHeaderDetailDirect,
+        vcrypto: CryptoSystemVersion,
         routed_operation: RoutedOperation,
         remote_sr_pubkey: TypedKey,
     ) -> Result<NetworkResult<()>, RPCError> {
-
         // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret)
         // xxx: punish nodes that send messages that fail to decrypt eventually? How to do this for safety routes?
-        let node_id_secret = self.routing_table.node_id_secret();
+        let node_id_secret = self.routing_table.node_id_secret(remote_sr_pubkey.kind);
-        let dh_secret = self
+        let dh_secret = vcrypto
-            .crypto
+            .cached_dh(&remote_sr_pubkey.key, &node_id_secret)
-            .cached_dh(&remote_sr_pubkey, &node_id_secret)
             .map_err(RPCError::protocol)?;
-        let body = match Crypto::decrypt_aead(
+        let body = match vcrypto.decrypt_aead(
             &routed_operation.data,
             &routed_operation.nonce,
             &dh_secret,
@@ -160,12 +121,20 @@ impl RPCProcessor {
         ) {
             Ok(v) => v,
             Err(e) => {
-                return Ok(NetworkResult::invalid_message(format!("decryption of routed operation failed: {}", e)));
+                return Ok(NetworkResult::invalid_message(format!(
+                    "decryption of routed operation failed: {}",
+                    e
+                )));
             }
         };
 
         // Pass message to RPC system
-        self.enqueue_safety_routed_message(remote_sr_pubkey, routed_operation.sequencing, body)
+        self.enqueue_safety_routed_message(
+            detail,
+            remote_sr_pubkey.key,
+            routed_operation.sequencing,
+            body,
+        )
         .map_err(RPCError::internal)?;
 
         Ok(NetworkResult::value(()))
@@ -176,6 +145,7 @@ impl RPCProcessor {
     fn process_private_routed_operation(
         &self,
         detail: RPCMessageHeaderDetailDirect,
+        vcrypto: CryptoSystemVersion,
         routed_operation: RoutedOperation,
         remote_sr_pubkey: TypedKey,
         pr_pubkey: TypedKey,
@@ -196,7 +166,7 @@ impl RPCProcessor {
                     (
                         rsd.secret_key,
                         SafetySpec {
-                            preferred_route: rss.get_route_id_for_key(&pr_pubkey),
+                            preferred_route: rss.get_route_id_for_key(&pr_pubkey.key),
                             hop_count: rssd.hop_count(),
                             stability: rssd.get_stability(),
                             sequencing: routed_operation.sequencing,
@@ -208,14 +178,13 @@ impl RPCProcessor {
                 return Ok(NetworkResult::invalid_message("signatures did not validate for private route"));
             };
 
-
         // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret)
         // xxx: punish nodes that send messages that fail to decrypt eventually. How to do this for private routes?
-        let dh_secret = self
+        let dh_secret = vcrypto
-            .crypto
+            .cached_dh(&remote_sr_pubkey.key, &secret_key)
-            .cached_dh(&remote_sr_pubkey, &secret_key)
             .map_err(RPCError::protocol)?;
-        let body = Crypto::decrypt_aead(
+        let body = vcrypto
+            .decrypt_aead(
                 &routed_operation.data,
                 &routed_operation.nonce,
                 &dh_secret,
@@ -226,7 +195,13 @@ impl RPCProcessor {
             ))?;
 
         // Pass message to RPC system
-        self.enqueue_private_routed_message(remote_sr_pubkey, pr_pubkey, safety_spec, body)
+        self.enqueue_private_routed_message(
+            detail,
+            remote_sr_pubkey.key,
+            pr_pubkey.key,
+            safety_spec,
+            body,
+        )
         .map_err(RPCError::internal)?;
 
         Ok(NetworkResult::value(()))
@@ -236,20 +211,26 @@ impl RPCProcessor {
     fn process_routed_operation(
         &self,
         detail: RPCMessageHeaderDetailDirect,
+        vcrypto: CryptoSystemVersion,
         routed_operation: RoutedOperation,
         remote_sr_pubkey: TypedKey,
         pr_pubkey: TypedKey,
     ) -> Result<NetworkResult<()>, RPCError> {
-
         // If the private route public key is our node id, then this was sent via safety route to our node directly
         // so there will be no signatures to validate
         if self.routing_table.node_ids().contains(&pr_pubkey) {
             // The private route was a stub
-            self.process_safety_routed_operation(detail, routed_operation, remote_sr_pubkey)
+            self.process_safety_routed_operation(
+                detail,
+                vcrypto,
+                routed_operation,
+                remote_sr_pubkey,
+            )
         } else {
             // Both safety and private routes used, should reply with a safety route
             self.process_private_routed_operation(
                 detail,
+                vcrypto,
                 routed_operation,
                 remote_sr_pubkey,
                 pr_pubkey,
@@ -275,7 +256,11 @@ impl RPCProcessor {
             };
 
             // Decrypt route hop data
-            let route_hop = network_result_try!(self.decrypt_private_route_hop_data(&route_hop_data, &private_route.public_key, &mut routed_operation)?);
+            let route_hop = network_result_try!(self.decrypt_private_route_hop_data(
+                &route_hop_data,
+                &private_route.public_key,
+                &mut routed_operation
+            )?);
 
             // Ensure hop count > 0
             if private_route.hop_count == 0 {
@@ -285,7 +270,8 @@ impl RPCProcessor {
             }
 
             // Make next PrivateRoute and pass it on
-            return self.process_route_private_route_hop(
+            return self
+                .process_route_private_route_hop(
                     routed_operation,
                     route_hop.node,
                     sr_pubkey,
@@ -312,8 +298,12 @@ impl RPCProcessor {
     }
 
     /// Decrypt route hop data and sign routed operation
-    pub(crate) fn decrypt_private_route_hop_data(&self, route_hop_data: &RouteHopData, pr_pubkey: &TypedKey, route_operation: &mut RoutedOperation) -> Result<NetworkResult<RouteHop>, RPCError>
+    pub(crate) fn decrypt_private_route_hop_data(
-    {
+        &self,
+        route_hop_data: &RouteHopData,
+        pr_pubkey: &TypedKey,
+        route_operation: &mut RoutedOperation,
+    ) -> Result<NetworkResult<RouteHop>, RPCError> {
         // Get crypto kind
         let crypto_kind = pr_pubkey.kind;
         let Some(vcrypto) = self.crypto.get(crypto_kind) else {
@@ -335,7 +325,10 @@ impl RPCProcessor {
         ) {
             Ok(v) => v,
             Err(e) => {
-                return Ok(NetworkResult::invalid_message(format!("unable to decrypt private route hop data: {}", e)));
+                return Ok(NetworkResult::invalid_message(format!(
+                    "unable to decrypt private route hop data: {}",
+                    e
+                )));
             }
         };
         let dec_blob_reader = RPCMessageData::new(dec_blob_data).get_reader()?;
@@ -353,7 +346,8 @@ impl RPCProcessor {
         if route_hop.next_hop.is_some() {
             let node_id = self.routing_table.node_id(crypto_kind);
             let node_id_secret = self.routing_table.node_id_secret(crypto_kind);
-            let sig = vcrypto.sign(&node_id.key, &node_id_secret, &route_operation.data)
+            let sig = vcrypto
+                .sign(&node_id.key, &node_id_secret, &route_operation.data)
                 .map_err(RPCError::internal)?;
             route_operation.signatures.push(sig);
         }
@@ -402,7 +396,13 @@ impl RPCProcessor {
                 let dh_secret = vcrypto
                     .cached_dh(&route.safety_route.public_key.key, &node_id_secret)
                     .map_err(RPCError::protocol)?;
-                let mut dec_blob_data = vcrypto.decrypt_aead(&route_hop_data.blob, &route_hop_data.nonce, &dh_secret, None)
+                let mut dec_blob_data = vcrypto
+                    .decrypt_aead(
+                        &route_hop_data.blob,
+                        &route_hop_data.nonce,
+                        &dh_secret,
+                        None,
+                    )
                     .map_err(RPCError::protocol)?;
 
                 // See if this is last hop in safety route, if so, we're decoding a PrivateRoute not a RouteHop
@@ -423,12 +423,14 @@ impl RPCProcessor {
                     };
 
                     // Switching from full safety route to private route first hop
-                    network_result_try!(self.process_private_route_first_hop(
+                    network_result_try!(
+                        self.process_private_route_first_hop(
                             route.operation,
                             route.safety_route.public_key,
                             private_route,
                         )
-                    .await?);
+                        .await?
+                    );
                 } else if dec_blob_tag == 0 {
                     // RouteHop
                     let route_hop = {
@@ -439,8 +441,14 @@ impl RPCProcessor {
                     };
 
                     // Continue the full safety route with another hop
-                    network_result_try!(self.process_route_safety_route_hop(route.operation, route_hop, route.safety_route)
+                    network_result_try!(
-                        .await?);
+                        self.process_route_safety_route_hop(
+                            route.operation,
+                            route_hop,
+                            route.safety_route
+                        )
+                        .await?
+                    );
                 } else {
                     return Ok(NetworkResult::invalid_message("invalid blob tag"));
                 }
@@ -451,17 +459,22 @@ impl RPCProcessor {
                 match private_route.hops {
                     PrivateRouteHops::FirstHop(_) => {
                         // Safety route was a stub, start with the beginning of the private route
-                        network_result_try!(self.process_private_route_first_hop(
+                        network_result_try!(
+                            self.process_private_route_first_hop(
                                 route.operation,
                                 route.safety_route.public_key,
                                 private_route,
                             )
-                        .await?);
+                            .await?
+                        );
                     }
                     PrivateRouteHops::Data(route_hop_data) => {
-                        
                         // Decrypt route hop data
-                        let route_hop = network_result_try!(self.decrypt_private_route_hop_data(&route_hop_data, &private_route.public_key, &mut route.operation)?);
+                        let route_hop = network_result_try!(self.decrypt_private_route_hop_data(
+                            &route_hop_data,
+                            &private_route.public_key,
+                            &mut route.operation
+                        )?);
 
                         // Ensure hop count > 0
                         if private_route.hop_count == 0 {
@@ -471,7 +484,8 @@ impl RPCProcessor {
                         }
 
                         // Make next PrivateRoute and pass it on
-                        network_result_try!(self.process_route_private_route_hop(
+                        network_result_try!(
+                            self.process_route_private_route_hop(
                                 route.operation,
                                 route_hop.node,
                                 route.safety_route.public_key,
@@ -484,7 +498,8 @@ impl RPCProcessor {
                                         .unwrap_or(PrivateRouteHops::Empty),
                                 },
                             )
-                        .await?);
+                            .await?
+                        );
                     }
                     PrivateRouteHops::Empty => {
                         // Ensure hop count == 0
@@ -502,6 +517,7 @@ impl RPCProcessor {
                         // No hops left, time to process the routed operation
                         network_result_try!(self.process_routed_operation(
                             detail,
+                            vcrypto,
                             route.operation,
                             route.safety_route.public_key,
                             private_route.public_key,