refactor for routing domains

veilid-core/src/routing_table/bucket_entry.rs

@@ -44,11 +44,10 @@ struct LastConnectionKey(PeerScope, ProtocolType, AddressType);
 #[derive(Debug)]
 pub struct BucketEntryInner {
     min_max_version: Option<(u8, u8)>,
-    seen_our_node_info: bool,
     updated_since_last_network_change: bool,
     last_connections: BTreeMap<LastConnectionKey, (ConnectionDescriptor, u64)>,
-    opt_signed_node_info: Option<SignedNodeInfo>,
-    opt_local_node_info: Option<LocalNodeInfo>,
+    signed_node_info: [Option<Box<SignedNodeInfo>>; RoutingDomain::count()],
+    seen_our_node_info: [bool; RoutingDomain::count()],
     peer_stats: PeerStats,
     latency_stats_accounting: LatencyStatsAccounting,
     transfer_stats_accounting: TransferStatsAccounting,
@@ -118,6 +117,7 @@ impl BucketEntryInner {
     // Retuns true if the node info changed
     pub fn update_signed_node_info(
         &mut self,
+        routing_domain: RoutingDomain,
         signed_node_info: SignedNodeInfo,
         allow_invalid_signature: bool,
     ) {
@@ -128,7 +128,7 @@ impl BucketEntryInner {
         }
 
         // See if we have an existing signed_node_info to update or not
-        if let Some(current_sni) = &self.opt_signed_node_info {
+        if let Some(current_sni) = &self.signed_node_info[routing_domain as usize] {
             // If the timestamp hasn't changed or is less, ignore this update
             if signed_node_info.timestamp <= current_sni.timestamp {
                 // If we received a node update with the same timestamp
@@ -137,7 +137,7 @@ impl BucketEntryInner {
                     && signed_node_info.timestamp == current_sni.timestamp
                 {
                     // No need to update the signednodeinfo though since the timestamp is the same
-                    // Just return true so we can make the node not dead
+                    // Touch the node and let it try to live again
                     self.updated_since_last_network_change = true;
                     self.touch_last_seen(intf::get_timestamp());
                 }
@@ -152,43 +152,57 @@ impl BucketEntryInner {
         ));
 
         // Update the signed node info
-        self.opt_signed_node_info = Some(signed_node_info);
+        self.signed_node_info[routing_domain as usize] = Some(Box::new(signed_node_info));
         self.updated_since_last_network_change = true;
         self.touch_last_seen(intf::get_timestamp());
     }
-    pub fn update_local_node_info(&mut self, local_node_info: LocalNodeInfo) {
-        self.opt_local_node_info = Some(local_node_info)
-    }
 
-    pub fn has_node_info(&self) -> bool {
-        self.opt_signed_node_info.is_some()
-    }
-
-    pub fn has_valid_signed_node_info(&self) -> bool {
-        if let Some(sni) = &self.opt_signed_node_info {
-            sni.is_valid()
+    pub fn has_node_info(&self, opt_routing_domain: Option<RoutingDomain>) -> bool {
+        if let Some(rd) = opt_routing_domain {
+            self.signed_node_info[rd as usize].is_some()
         } else {
+            for rd in RoutingDomain::all() {
+                if self.signed_node_info[rd as usize].is_some() {
+                    return true;
+                }
+            }
             false
         }
     }
 
-    pub fn has_local_node_info(&self) -> bool {
-        self.opt_local_node_info.is_some()
+    pub fn has_valid_signed_node_info(&self, opt_routing_domain: Option<RoutingDomain>) -> bool {
+        if let Some(rd) = opt_routing_domain {
+            if let Some(sni) = &self.signed_node_info[rd as usize] {
+                if sni.is_valid() {
+                    return true;
+                }
+            }
+            false
+        } else {
+            for rd in RoutingDomain::all() {
+                if let Some(sni) = &self.signed_node_info[rd as usize] {
+                    if sni.is_valid() {
+                        return true;
+                    }
+                }
+            }
+            false
+        }
     }
 
-    pub fn node_info(&self) -> Option<NodeInfo> {
-        self.opt_signed_node_info
+    pub fn node_info(&self, routing_domain: RoutingDomain) -> Option<NodeInfo> {
+        self.signed_node_info[routing_domain as usize]
             .as_ref()
             .map(|s| s.node_info.clone())
     }
-    pub fn local_node_info(&self) -> Option<LocalNodeInfo> {
-        self.opt_local_node_info.clone()
-    }
-    pub fn peer_info(&self, key: DHTKey) -> Option<PeerInfo> {
-        self.opt_signed_node_info.as_ref().map(|s| PeerInfo {
-            node_id: NodeId::new(key),
-            signed_node_info: s.clone(),
-        })
+
+    pub fn peer_info(&self, key: DHTKey, routing_domain: RoutingDomain) -> Option<PeerInfo> {
+        self.signed_node_info[routing_domain as usize]
+            .as_ref()
+            .map(|s| PeerInfo {
+                node_id: NodeId::new(key),
+                signed_node_info: *s.clone(),
+            })
     }
 
     fn descriptor_to_key(last_connection: ConnectionDescriptor) -> LastConnectionKey {
@@ -256,12 +270,12 @@ impl BucketEntryInner {
         self.peer_stats.status = Some(status);
     }
 
-    pub fn set_seen_our_node_info(&mut self, seen: bool) {
-        self.seen_our_node_info = seen;
+    pub fn set_seen_our_node_info(&mut self, routing_domain: RoutingDomain, seen: bool) {
+        self.seen_our_node_info[routing_domain as usize] = seen;
     }
 
-    pub fn has_seen_our_node_info(&self) -> bool {
-        self.seen_our_node_info
+    pub fn has_seen_our_node_info(&self, routing_domain: RoutingDomain) -> bool {
+        self.seen_our_node_info[routing_domain as usize]
     }
 
     pub fn set_updated_since_last_network_change(&mut self, updated: bool) {
@@ -337,20 +351,14 @@ impl BucketEntryInner {
     }
 
     // Check if this node needs a ping right now to validate it is still reachable
-    pub(super) fn needs_ping(
-        &self,
-        node_id: &DHTKey,
-        cur_ts: u64,
-        relay_node_id: Option<DHTKey>,
-    ) -> bool {
+    pub(super) fn needs_ping(&self, node_id: &DHTKey, cur_ts: u64, needs_keepalive: bool) -> bool {
         // See which ping pattern we are to use
         let state = self.state(cur_ts);
 
-        // If this entry is our relay node, then we should ping it regularly to keep our association alive
-        if let Some(relay_node_id) = relay_node_id {
-            if relay_node_id == *node_id {
-                return self.needs_constant_ping(cur_ts, KEEPALIVE_PING_INTERVAL_SECS as u64);
-            }
+        // If this entry needs a keepalive (like a relay node),
+        // then we should ping it regularly to keep our association alive
+        if needs_keepalive {
+            return self.needs_constant_ping(cur_ts, KEEPALIVE_PING_INTERVAL_SECS as u64);
         }
 
         match state {
@@ -494,11 +502,10 @@ impl BucketEntry {
             ref_count: AtomicU32::new(0),
             inner: RwLock::new(BucketEntryInner {
                 min_max_version: None,
-                seen_our_node_info: false,
+                seen_our_node_info: [false, false],
                 updated_since_last_network_change: false,
                 last_connections: BTreeMap::new(),
-                opt_signed_node_info: None,
-                opt_local_node_info: None,
+                signed_node_info: [None, None],
                 peer_stats: PeerStats {
                     time_added: now,
                     rpc_stats: RPCStats::default(),
@@ -547,7 +554,7 @@ impl Drop for BucketEntry {
 
             panic!(
                 "bucket entry dropped with non-zero refcount: {:#?}",
-                self.inner.read().node_info()
+                &*self.inner.read()
             )
         }
     }

veilid-core/src/routing_table/debug.rs

@@ -85,8 +85,17 @@ impl RoutingTable {
             out += &format!("  {:>2}: {:?}\n", n, gdi);
         }
 
-        out += "Own PeerInfo:\n";
-        out += &format!("  {:#?}\n", self.get_own_peer_info());
+        out += "LocalNetwork PeerInfo:\n";
+        out += &format!(
+            "  {:#?}\n",
+            self.get_own_peer_info(RoutingDomain::LocalNetwork)
+        );
+
+        out += "PublicInternet PeerInfo:\n";
+        out += &format!(
+            "  {:#?}\n",
+            self.get_own_peer_info(RoutingDomain::PublicInternet)
+        );
 
         out
     }
@@ -142,7 +151,7 @@ impl RoutingTable {
         let mut out = String::new();
         out += &format!("Entry {:?}:\n", node_id);
         if let Some(nr) = self.lookup_node_ref(node_id) {
-            out += &nr.operate(|e| format!("{:#?}\n", e));
+            out += &nr.operate(|_rt, e| format!("{:#?}\n", e));
         } else {
             out += "Entry not found\n";
         }

veilid-core/src/routing_table/find_nodes.rs

@@ -19,12 +19,19 @@ impl RoutingTable {
     ) -> impl FnMut(&BucketEntryInner) -> bool {
         // does it have matching public dial info?
         move |e| {
-            e.node_info()
-                .map(|n| {
-                    n.first_filtered_dial_info_detail(|did| did.matches_filter(&dial_info_filter))
+            for rd in RoutingDomain::all() {
+                if let Some(ni) = e.node_info(rd) {
+                    if ni
+                        .first_filtered_dial_info_detail(|did| {
+                            did.matches_filter(&dial_info_filter)
+                        })
                         .is_some()
-                })
-                .unwrap_or(false)
+                    {
+                        return true;
+                    }
+                }
+            }
+            false
         }
     }
 
@@ -34,14 +41,17 @@ impl RoutingTable {
     ) -> impl FnMut(&BucketEntryInner) -> bool {
         // does the node's outbound capabilities match the dialinfo?
         move |e| {
-            e.node_info()
-                .map(|n| {
+            for rd in RoutingDomain::all() {
+                if let Some(ni) = e.node_info(rd) {
                     let mut dif = DialInfoFilter::all();
-                    dif = dif.with_protocol_type_set(n.outbound_protocols);
-                    dif = dif.with_address_type_set(n.address_types);
-                    dial_info.matches_filter(&dif)
-                })
-                .unwrap_or(false)
+                    dif = dif.with_protocol_type_set(ni.outbound_protocols);
+                    dif = dif.with_address_type_set(ni.address_types);
+                    if dial_info.matches_filter(&dif) {
+                        return true;
+                    }
+                }
+            }
+            false
         }
     }
 
@@ -75,14 +85,17 @@ impl RoutingTable {
             // count
             node_count,
             // filter
-            Some(move |_k: DHTKey, v: Option<Arc<BucketEntry>>| {
+            Some(|_k: DHTKey, v: Option<Arc<BucketEntry>>| {
                 let entry = v.unwrap();
                 entry.with(|e| {
-                    // skip nodes on our local network here
-                    if e.local_node_info().is_some() {
+                    // skip nodes on local network
+                    if e.node_info(RoutingDomain::LocalNetwork).is_some() {
+                        return false;
+                    }
+                    // skip nodes not on public internet
+                    if e.node_info(RoutingDomain::PublicInternet).is_none() {
                         return false;
                     }
-
                     // skip nodes that dont match entry filter
                     entry_filter(e)
                 })
@@ -113,7 +126,7 @@ impl RoutingTable {
                 let entry = v.unwrap();
                 entry.with(|e| {
                     // skip nodes on our local network here
-                    if e.local_node_info().is_some() {
+                    if e.node_info(RoutingDomain::LocalNetwork).is_some() {
                         return false;
                     }
 
@@ -147,7 +160,9 @@ impl RoutingTable {
                         keep
                     };
 
-                    e.node_info().map(filter).unwrap_or(false)
+                    e.node_info(RoutingDomain::PublicInternet)
+                        .map(filter)
+                        .unwrap_or(false)
                 })
             }),
             // transform
@@ -158,49 +173,61 @@ impl RoutingTable {
     }
 
     // Get our own node's peer info (public node info) so we can share it with other nodes
-    pub fn get_own_peer_info(&self) -> PeerInfo {
-        PeerInfo::new(NodeId::new(self.node_id()), self.get_own_signed_node_info())
+    pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo {
+        PeerInfo::new(
+            NodeId::new(self.node_id()),
+            self.get_own_signed_node_info(routing_domain),
+        )
     }
 
-    pub fn get_own_signed_node_info(&self) -> SignedNodeInfo {
+    pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedNodeInfo {
         let node_id = NodeId::new(self.node_id());
         let secret = self.node_id_secret();
-        SignedNodeInfo::with_secret(self.get_own_node_info(), node_id, &secret).unwrap()
+        SignedNodeInfo::with_secret(self.get_own_node_info(routing_domain), node_id, &secret)
+            .unwrap()
     }
 
-    pub fn get_own_node_info(&self) -> NodeInfo {
+    pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo {
         let netman = self.network_manager();
-        let relay_node = netman.relay_node();
+        let relay_node = self.relay_node(routing_domain);
         let pc = netman.get_protocol_config();
         NodeInfo {
-            network_class: netman.get_network_class().unwrap_or(NetworkClass::Invalid),
+            network_class: netman
+                .get_network_class(routing_domain)
+                .unwrap_or(NetworkClass::Invalid),
             outbound_protocols: pc.outbound,
             address_types: pc.family_global,
             min_version: MIN_VERSION,
             max_version: MAX_VERSION,
-            dial_info_detail_list: self.dial_info_details(RoutingDomain::PublicInternet),
-            relay_peer_info: relay_node.and_then(|rn| rn.peer_info().map(Box::new)),
+            dial_info_detail_list: self.dial_info_details(routing_domain),
+            relay_peer_info: relay_node.and_then(|rn| rn.peer_info(routing_domain).map(Box::new)),
         }
     }
 
     pub fn filter_has_valid_signed_node_info(
+        &self,
         v: Option<Arc<BucketEntry>>,
         own_peer_info_is_valid: bool,
+        opt_routing_domain: Option<RoutingDomain>,
     ) -> bool {
+        let routing_table = self.clone();
         match v {
             None => own_peer_info_is_valid,
-            Some(entry) => entry.with(|e| e.has_valid_signed_node_info()),
+            Some(entry) => entry.with(|e| e.has_valid_signed_node_info(opt_routing_domain)),
         }
     }
 
     pub fn transform_to_peer_info(
+        &self,
+        routing_domain: RoutingDomain,
         k: DHTKey,
         v: Option<Arc<BucketEntry>>,
         own_peer_info: &PeerInfo,
     ) -> PeerInfo {
+        let routing_table = self.clone();
         match v {
             None => own_peer_info.clone(),
-            Some(entry) => entry.with(|e| e.peer_info(k).unwrap()),
+            Some(entry) => entry.with(|e| e.peer_info(k, routing_domain).unwrap()),
         }
     }
 

veilid-core/src/routing_table/mod.rs

@@ -15,22 +15,39 @@ use bucket::*;
 pub use bucket_entry::*;
 pub use debug::*;
 pub use find_nodes::*;
+use hashlink::LruCache;
 pub use node_ref::*;
 pub use stats_accounting::*;
 
+const RECENT_PEERS_TABLE_SIZE: usize = 64;
+
 //////////////////////////////////////////////////////////////////////////
 
 #[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Ord, Eq)]
 pub enum RoutingDomain {
-    PublicInternet,
-    LocalNetwork,
+    PublicInternet = 0,
+    LocalNetwork = 1,
+}
+impl RoutingDomain {
+    pub const fn count() -> usize {
+        2
+    }
+    pub const fn all() -> [RoutingDomain; RoutingDomain::count()] {
+        [RoutingDomain::PublicInternet, RoutingDomain::LocalNetwork]
+    }
 }
 
 #[derive(Debug, Default)]
 pub struct RoutingDomainDetail {
+    relay_node: Option<NodeRef>,
     dial_info_details: Vec<DialInfoDetail>,
 }
 
+#[derive(Debug, Clone, Copy)]
+pub struct RecentPeersEntry {
+    last_connection: ConnectionDescriptor,
+}
+
 struct RoutingTableInner {
     network_manager: NetworkManager,
     node_id: DHTKey,              // The current node's public DHT key
@@ -46,6 +63,7 @@ struct RoutingTableInner {
     self_latency_stats_accounting: LatencyStatsAccounting, // Interim accounting mechanism for this node's RPC latency to any other node
     self_transfer_stats_accounting: TransferStatsAccounting, // Interim accounting mechanism for the total bandwidth to/from this node
     self_transfer_stats: TransferStatsDownUp, // Statistics about the total bandwidth to/from this node
+    recent_peers: LruCache<DHTKey, RecentPeersEntry>, // Peers we have recently communicated with
 }
 
 #[derive(Clone, Debug, Default)]
@@ -82,6 +100,7 @@ impl RoutingTable {
             self_latency_stats_accounting: LatencyStatsAccounting::new(),
             self_transfer_stats_accounting: TransferStatsAccounting::new(),
             self_transfer_stats: TransferStatsDownUp::default(),
+            recent_peers: LruCache::new(RECENT_PEERS_TABLE_SIZE),
         }
     }
     fn new_unlocked_inner(_config: VeilidConfig) -> RoutingTableUnlockedInner {
@@ -159,6 +178,16 @@ impl RoutingTable {
         }
     }
 
+    pub fn relay_node(&self, domain: RoutingDomain) -> Option<NodeRef> {
+        let inner = self.inner.read();
+        Self::with_routing_domain(&*inner, domain, |rd| rd.relay_node.clone())
+    }
+
+    pub fn set_relay_node(&self, domain: RoutingDomain, opt_relay_node: Option<NodeRef>) {
+        let inner = self.inner.write();
+        Self::with_routing_domain(&mut *inner, domain, |rd| rd.relay_node = opt_relay_node);
+    }
+
     pub fn has_dial_info(&self, domain: RoutingDomain) -> bool {
         let inner = self.inner.read();
         Self::with_routing_domain(&*inner, domain, |rd| !rd.dial_info_details.is_empty())
@@ -295,14 +324,14 @@ impl RoutingTable {
 
         // Public dial info changed, go through all nodes and reset their 'seen our node info' bit
         if matches!(domain, RoutingDomain::PublicInternet) {
-            Self::reset_all_seen_our_node_info(&*inner);
-            Self::reset_all_updated_since_last_network_change(&*inner);
+            Self::reset_all_seen_our_node_info(&mut *inner);
+            Self::reset_all_updated_since_last_network_change(&mut *inner);
         }
 
         Ok(())
     }
 
-    fn reset_all_seen_our_node_info(inner: &RoutingTableInner) {
+    fn reset_all_seen_our_node_info(inner: &mut RoutingTableInner) {
         let cur_ts = intf::get_timestamp();
         Self::with_entries(&*inner, cur_ts, BucketEntryState::Dead, |_, v| {
             v.with_mut(|e| e.set_seen_our_node_info(false));
@@ -310,7 +339,7 @@ impl RoutingTable {
         });
     }
 
-    fn reset_all_updated_since_last_network_change(inner: &RoutingTableInner) {
+    fn reset_all_updated_since_last_network_change(inner: &mut RoutingTableInner) {
         let cur_ts = intf::get_timestamp();
         Self::with_entries(&*inner, cur_ts, BucketEntryState::Dead, |_, v| {
             v.with_mut(|e| e.set_updated_since_last_network_change(false));
@@ -328,7 +357,7 @@ impl RoutingTable {
 
         // Public dial info changed, go through all nodes and reset their 'seen our node info' bit
         if matches!(domain, RoutingDomain::PublicInternet) {
-            Self::reset_all_seen_our_node_info(&*inner);
+            Self::reset_all_seen_our_node_info(&mut *inner);
         }
     }
 
@@ -479,12 +508,17 @@ impl RoutingTable {
         None
     }
 
-    pub fn get_nodes_needing_updates(&self, cur_ts: u64, all: bool) -> Vec<NodeRef> {
+    pub fn get_nodes_needing_updates(
+        &self,
+        routing_domain: RoutingDomain,
+        cur_ts: u64,
+        all: bool,
+    ) -> Vec<NodeRef> {
         let inner = self.inner.read();
         let mut node_refs = Vec::<NodeRef>::with_capacity(inner.bucket_entry_count);
         Self::with_entries(&*inner, cur_ts, BucketEntryState::Unreliable, |k, v| {
             // Only update nodes that haven't seen our node info yet
-            if all || !v.with(|e| e.has_seen_our_node_info()) {
+            if all || !v.with(|e| e.has_seen_our_node_info(routing_domain)) {
                 node_refs.push(NodeRef::new(self.clone(), k, v, None));
             }
             Option::<()>::None
@@ -492,16 +526,25 @@ impl RoutingTable {
         node_refs
     }
 
-    pub fn get_nodes_needing_ping(
-        &self,
-        cur_ts: u64,
-        relay_node_id: Option<DHTKey>,
-    ) -> Vec<NodeRef> {
+    pub fn get_nodes_needing_ping(&self, cur_ts: u64) -> Vec<NodeRef> {
         let inner = self.inner.read();
+
+        // Collect relay nodes
+        let mut relays: HashSet<DHTKey> = HashSet::new();
+        for rd in RoutingDomain::all() {
+            let opt_relay_id =
+                Self::with_routing_domain(&*inner, RoutingDomain::PublicInternet, |rd| {
+                    rd.relay_node.map(|rn| rn.node_id())
+                });
+            if let Some(relay_id) = opt_relay_id {
+                relays.insert(relay_id);
+            }
+        }
+
+        // Collect all entries that are 'needs_ping' and have some node info making them reachable somehow
         let mut node_refs = Vec::<NodeRef>::with_capacity(inner.bucket_entry_count);
         Self::with_entries(&*inner, cur_ts, BucketEntryState::Unreliable, |k, v| {
-            // Only update nodes that haven't seen our node info yet
-            if v.with(|e| e.needs_ping(&k, cur_ts, relay_node_id)) {
+            if v.with(|e| e.has_node_info(None) && e.needs_ping(&k, cur_ts, relays.contains(&k))) {
                 node_refs.push(NodeRef::new(self.clone(), k, v, None));
             }
             Option::<()>::None
@@ -600,6 +643,7 @@ impl RoutingTable {
     // and add the dial info we have for it, since that's pretty common
     pub fn register_node_with_signed_node_info(
         &self,
+        routing_domain: RoutingDomain,
         node_id: DHTKey,
         signed_node_info: SignedNodeInfo,
         allow_invalid_signature: bool,
@@ -617,7 +661,7 @@ impl RoutingTable {
         }
 
         self.create_node_ref(node_id, |e| {
-            e.update_signed_node_info(signed_node_info, allow_invalid_signature);
+            e.update_signed_node_info(routing_domain, signed_node_info, allow_invalid_signature);
         })
     }
 
@@ -653,64 +697,6 @@ impl RoutingTable {
         Ok(())
     }
 
-    //////////////////////////////////////////////////////////////////////
-    // Stats Accounting
-    pub fn stats_question_sent(
-        &self,
-        node_ref: NodeRef,
-        ts: u64,
-        bytes: u64,
-        expects_answer: bool,
-    ) {
-        self.inner
-            .write()
-            .self_transfer_stats_accounting
-            .add_up(bytes);
-        node_ref.operate_mut(|e| {
-            e.question_sent(ts, bytes, expects_answer);
-        })
-    }
-    pub fn stats_question_rcvd(&self, node_ref: NodeRef, ts: u64, bytes: u64) {
-        self.inner
-            .write()
-            .self_transfer_stats_accounting
-            .add_down(bytes);
-        node_ref.operate_mut(|e| {
-            e.question_rcvd(ts, bytes);
-        })
-    }
-    pub fn stats_answer_sent(&self, node_ref: NodeRef, bytes: u64) {
-        self.inner
-            .write()
-            .self_transfer_stats_accounting
-            .add_up(bytes);
-        node_ref.operate_mut(|e| {
-            e.answer_sent(bytes);
-        })
-    }
-    pub fn stats_answer_rcvd(&self, node_ref: NodeRef, send_ts: u64, recv_ts: u64, bytes: u64) {
-        {
-            let mut inner = self.inner.write();
-            inner.self_transfer_stats_accounting.add_down(bytes);
-            inner
-                .self_latency_stats_accounting
-                .record_latency(recv_ts - send_ts);
-        }
-        node_ref.operate_mut(|e| {
-            e.answer_rcvd(send_ts, recv_ts, bytes);
-        })
-    }
-    pub fn stats_question_lost(&self, node_ref: NodeRef) {
-        node_ref.operate_mut(|e| {
-            e.question_lost();
-        })
-    }
-    pub fn stats_failed_to_send(&self, node_ref: NodeRef, ts: u64, expects_answer: bool) {
-        node_ref.operate_mut(|e| {
-            e.failed_to_send(ts, expects_answer);
-        })
-    }
-
     //////////////////////////////////////////////////////////////////////
     // Routing Table Health Metrics
 
@@ -735,4 +721,23 @@ impl RoutingTable {
         }
         health
     }
+
+    pub fn get_recent_peers(&self) -> Vec<(DHTKey, RecentPeersEntry)> {
+        let inner = self.inner.read();
+        inner
+            .recent_peers
+            .iter()
+            .map(|(k, v)| (k.clone(), v.clone()))
+            .collect()
+    }
+
+    pub fn touch_recent_peer(
+        inner: &mut RoutingTableInner,
+        node_id: DHTKey,
+        last_connection: ConnectionDescriptor,
+    ) {
+        inner
+            .recent_peers
+            .insert(node_id, RecentPeersEntry { last_connection });
+    }
 }

veilid-core/src/routing_table/node_ref.rs

@@ -85,57 +85,82 @@ impl NodeRef {
     //         .unwrap_or(true)
     // }
 
-    pub fn operate<T, F>(&self, f: F) -> T
+    pub(super) fn operate<T, F>(&self, f: F) -> T
     where
-        F: FnOnce(&BucketEntryInner) -> T,
+        F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T,
     {
-        self.entry.with(f)
+        let inner = &*self.routing_table.inner.read();
+        self.entry.with(|e| f(inner, e))
     }
 
-    pub fn operate_mut<T, F>(&self, f: F) -> T
+    pub(super) fn operate_mut<T, F>(&self, f: F) -> T
     where
-        F: FnOnce(&mut BucketEntryInner) -> T,
+        F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T,
     {
-        self.entry.with_mut(f)
+        let inner = &mut *self.routing_table.inner.write();
+        self.entry.with_mut(|e| f(inner, e))
     }
 
-    pub fn peer_info(&self) -> Option<PeerInfo> {
-        self.operate(|e| e.peer_info(self.node_id()))
+    pub fn peer_info(&self, routing_domain: RoutingDomain) -> Option<PeerInfo> {
+        self.operate(|_rti, e| e.peer_info(self.node_id(), routing_domain))
     }
-    pub fn has_seen_our_node_info(&self) -> bool {
-        self.operate(|e| e.has_seen_our_node_info())
+    pub fn has_valid_signed_node_info(&self, opt_routing_domain: Option<RoutingDomain>) -> bool {
+        self.operate(|_rti, e| e.has_valid_signed_node_info(opt_routing_domain))
     }
-    pub fn set_seen_our_node_info(&self) {
-        self.operate_mut(|e| e.set_seen_our_node_info(true));
+    pub fn has_seen_our_node_info(&self, routing_domain: RoutingDomain) -> bool {
+        self.operate(|_rti, e| e.has_seen_our_node_info(routing_domain))
+    }
+    pub fn set_seen_our_node_info(&self, routing_domain: RoutingDomain) {
+        self.operate_mut(|_rti, e| e.set_seen_our_node_info(routing_domain, true));
     }
     pub fn has_updated_since_last_network_change(&self) -> bool {
-        self.operate(|e| e.has_updated_since_last_network_change())
+        self.operate(|_rti, e| e.has_updated_since_last_network_change())
     }
     pub fn set_updated_since_last_network_change(&self) {
-        self.operate_mut(|e| e.set_updated_since_last_network_change(true));
+        self.operate_mut(|_rti, e| e.set_updated_since_last_network_change(true));
     }
-    pub fn network_class(&self) -> Option<NetworkClass> {
-        self.operate(|e| e.node_info().map(|n| n.network_class))
+
+    pub fn update_node_status(&self, node_status: NodeStatus) {
+        self.operate_mut(|_rti, e| {
+            e.update_node_status(node_status);
+        });
     }
-    pub fn outbound_protocols(&self) -> Option<ProtocolTypeSet> {
-        self.operate(|e| e.node_info().map(|n| n.outbound_protocols))
+
+    pub fn min_max_version(&self) -> Option<(u8, u8)> {
+        self.operate(|_rti, e| e.min_max_version())
     }
-    pub fn address_types(&self) -> Option<AddressTypeSet> {
-        self.operate(|e| e.node_info().map(|n| n.address_types))
+
+    pub fn set_min_max_version(&self, min_max_version: (u8, u8)) {
+        self.operate_mut(|_rti, e| e.set_min_max_version(min_max_version))
     }
-    pub fn node_info_outbound_filter(&self) -> DialInfoFilter {
+
+    pub fn state(&self, cur_ts: u64) -> BucketEntryState {
+        self.operate(|_rti, e| e.state(cur_ts))
+    }
+
+    pub fn network_class(&self, routing_domain: RoutingDomain) -> Option<NetworkClass> {
+        self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.network_class))
+    }
+    pub fn outbound_protocols(&self, routing_domain: RoutingDomain) -> Option<ProtocolTypeSet> {
+        self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.outbound_protocols))
+    }
+    pub fn address_types(&self, routing_domain: RoutingDomain) -> Option<AddressTypeSet> {
+        self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.address_types))
+    }
+    pub fn node_info_outbound_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter {
         let mut dif = DialInfoFilter::all();
-        if let Some(outbound_protocols) = self.outbound_protocols() {
+        if let Some(outbound_protocols) = self.outbound_protocols(routing_domain) {
             dif = dif.with_protocol_type_set(outbound_protocols);
         }
-        if let Some(address_types) = self.address_types() {
+        if let Some(address_types) = self.address_types(routing_domain) {
             dif = dif.with_address_type_set(address_types);
         }
         dif
     }
 
-    pub fn relay(&self) -> Option<NodeRef> {
-        let target_rpi = self.operate(|e| e.node_info().map(|n| n.relay_peer_info))?;
+    pub fn relay(&self, routing_domain: RoutingDomain) -> Option<NodeRef> {
+        let target_rpi =
+            self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.relay_peer_info))?;
         target_rpi.and_then(|t| {
             // If relay is ourselves, then return None, because we can't relay through ourselves
             // and to contact this node we should have had an existing inbound connection
@@ -145,7 +170,12 @@ impl NodeRef {
 
             // Register relay node and return noderef
             self.routing_table
-                .register_node_with_signed_node_info(t.node_id.key, t.signed_node_info, false)
+                .register_node_with_signed_node_info(
+                    routing_domain,
+                    t.node_id.key,
+                    t.signed_node_info,
+                    false,
+                )
                 .map(|mut nr| {
                     nr.set_filter(self.filter_ref().cloned());
                     nr
@@ -156,28 +186,24 @@ impl NodeRef {
         &self,
         routing_domain: Option<RoutingDomain>,
     ) -> Option<DialInfoDetail> {
-        self.operate(|e| {
+        self.operate(|_rt, e| {
             // Prefer local dial info first unless it is filtered out
             if routing_domain == None || routing_domain == Some(RoutingDomain::LocalNetwork) {
-                e.local_node_info().and_then(|l| {
-                    l.first_filtered_dial_info(|di| {
+                e.node_info(RoutingDomain::LocalNetwork).and_then(|l| {
+                    l.first_filtered_dial_info_detail(|did| {
                         if let Some(filter) = self.filter.as_ref() {
-                            di.matches_filter(filter)
+                            did.matches_filter(filter)
                         } else {
                             true
                         }
                     })
-                    .map(|di| DialInfoDetail {
-                        class: DialInfoClass::Direct,
-                        dial_info: di,
-                    })
                 })
             } else {
                 None
             }
             .or_else(|| {
                 if routing_domain == None || routing_domain == Some(RoutingDomain::PublicInternet) {
-                    e.node_info().and_then(|n| {
+                    e.node_info(RoutingDomain::PublicInternet).and_then(|n| {
                         n.first_filtered_dial_info_detail(|did| {
                             if let Some(filter) = self.filter.as_ref() {
                                 did.matches_filter(filter)
@@ -198,26 +224,21 @@ impl NodeRef {
         routing_domain: Option<RoutingDomain>,
     ) -> Vec<DialInfoDetail> {
         let mut out = Vec::new();
-        self.operate(|e| {
+        self.operate(|_rt, e| {
             // Prefer local dial info first unless it is filtered out
             if routing_domain == None || routing_domain == Some(RoutingDomain::LocalNetwork) {
-                if let Some(lni) = e.local_node_info() {
-                    for di in lni.all_filtered_dial_info(|di| {
+                if let Some(ni) = e.node_info(RoutingDomain::LocalNetwork) {
+                    out.append(&mut ni.all_filtered_dial_info_details(|did| {
                         if let Some(filter) = self.filter.as_ref() {
-                            di.matches_filter(filter)
+                            did.matches_filter(filter)
                         } else {
                             true
                         }
-                    }) {
-                        out.push(DialInfoDetail {
-                            class: DialInfoClass::Direct,
-                            dial_info: di,
-                        });
-                    }
+                    }))
                 }
             }
             if routing_domain == None || routing_domain == Some(RoutingDomain::PublicInternet) {
-                if let Some(ni) = e.node_info() {
+                if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) {
                     out.append(&mut ni.all_filtered_dial_info_details(|did| {
                         if let Some(filter) = self.filter.as_ref() {
                             did.matches_filter(filter)
@@ -235,7 +256,7 @@ impl NodeRef {
     pub async fn last_connection(&self) -> Option<ConnectionDescriptor> {
         // Get the last connection and the last time we saw anything with this connection
         let (last_connection, last_seen) =
-            self.operate(|e| e.last_connection(self.filter.clone()))?;
+            self.operate(|_rti, e| e.last_connection(self.filter.clone()))?;
 
         // Should we check the connection table?
         if last_connection.protocol_type().is_connection_oriented() {
@@ -254,21 +275,60 @@ impl NodeRef {
     }
 
     pub fn clear_last_connections(&self) {
-        self.operate_mut(|e| e.clear_last_connections())
+        self.operate_mut(|_rti, e| e.clear_last_connections())
     }
 
     pub fn set_last_connection(&self, connection_descriptor: ConnectionDescriptor, ts: u64) {
-        self.operate_mut(|e| e.set_last_connection(connection_descriptor, ts))
+        self.operate_mut(|_rti, e| e.set_last_connection(connection_descriptor, ts))
     }
 
     pub fn has_any_dial_info(&self) -> bool {
-        self.operate(|e| {
-            e.node_info()
-                .map(|n| n.has_any_dial_info())
-                .unwrap_or(false)
-                || e.local_node_info()
-                    .map(|l| l.has_dial_info())
-                    .unwrap_or(false)
+        self.operate(|_rti, e| {
+            for rtd in RoutingDomain::all() {
+                if let Some(ni) = e.node_info(rtd) {
+                    if ni.has_any_dial_info() {
+                        return true;
+                    }
+                }
+            }
+            false
+        })
+    }
+
+    pub fn stats_question_sent(&self, ts: u64, bytes: u64, expects_answer: bool) {
+        self.operate_mut(|rti, e| {
+            rti.self_transfer_stats_accounting.add_up(bytes);
+            e.question_sent(ts, bytes, expects_answer);
+        })
+    }
+    pub fn stats_question_rcvd(&self, ts: u64, bytes: u64) {
+        self.operate_mut(|rti, e| {
+            rti.self_transfer_stats_accounting.add_down(bytes);
+            e.question_rcvd(ts, bytes);
+        })
+    }
+    pub fn stats_answer_sent(&self, bytes: u64) {
+        self.operate_mut(|rti, e| {
+            rti.self_transfer_stats_accounting.add_up(bytes);
+            e.answer_sent(bytes);
+        })
+    }
+    pub fn stats_answer_rcvd(&self, send_ts: u64, recv_ts: u64, bytes: u64) {
+        self.operate_mut(|rti, e| {
+            rti.self_transfer_stats_accounting.add_down(bytes);
+            rti.self_latency_stats_accounting
+                .record_latency(recv_ts - send_ts);
+            e.answer_rcvd(send_ts, recv_ts, bytes);
+        })
+    }
+    pub fn stats_question_lost(&self) {
+        self.operate_mut(|_rti, e| {
+            e.question_lost();
+        })
+    }
+    pub fn stats_failed_to_send(&self, ts: u64, expects_answer: bool) {
+        self.operate_mut(|_rti, e| {
+            e.failed_to_send(ts, expects_answer);
         })
     }
 }