Merge branch 'new-public-address-detection' into 'main'

New public address detection

Closes #297

See merge request veilid/veilid!174

veilid-core/src/network_manager/native/discovery_context.rs

@@ -0,0 +1,621 @@
+/// Context detection of public dial info for a single protocol and address type
+/// Also performs UPNP/IGD mapping if enabled and possible
+use super::*;
+use futures_util::stream::FuturesUnordered;
+
+const PORT_MAP_VALIDATE_TRY_COUNT: usize = 3;
+const PORT_MAP_VALIDATE_DELAY_MS: u32 = 500;
+const PORT_MAP_TRY_COUNT: usize = 3;
+
+// Detection result of dial info detection futures
+#[derive(Clone, Debug)]
+pub enum DetectedDialInfo {
+    SymmetricNAT,
+    Detected(DialInfoDetail),
+}
+
+// Result of checking external address
+#[derive(Clone, Debug)]
+struct ExternalInfo {
+    dial_info: DialInfo,
+    address: SocketAddress,
+    node: NodeRef,
+}
+
+struct DiscoveryContextInner {
+    // first node contacted
+    external_1: Option<ExternalInfo>,
+    // second node contacted
+    external_2: Option<ExternalInfo>,
+}
+
+struct DiscoveryContextUnlockedInner {
+    routing_table: RoutingTable,
+    net: Network,
+    // per-protocol
+    intf_addrs: Vec<SocketAddress>,
+    protocol_type: ProtocolType,
+    address_type: AddressType,
+}
+
+#[derive(Clone)]
+pub struct DiscoveryContext {
+    unlocked_inner: Arc<DiscoveryContextUnlockedInner>,
+    inner: Arc<Mutex<DiscoveryContextInner>>,
+}
+
+impl DiscoveryContext {
+    pub fn new(
+        routing_table: RoutingTable,
+        net: Network,
+        protocol_type: ProtocolType,
+        address_type: AddressType,
+    ) -> Self {
+        let intf_addrs =
+            Self::get_local_addresses(routing_table.clone(), protocol_type, address_type);
+
+        Self {
+            unlocked_inner: Arc::new(DiscoveryContextUnlockedInner {
+                routing_table,
+                net,
+                intf_addrs,
+                protocol_type,
+                address_type,
+            }),
+            inner: Arc::new(Mutex::new(DiscoveryContextInner {
+                external_1: None,
+                external_2: None,
+            })),
+        }
+    }
+
+    ///////
+    // Utilities
+
+    // This pulls the already-detected local interface dial info from the routing table
+    #[instrument(level = "trace", skip(routing_table), ret)]
+    fn get_local_addresses(
+        routing_table: RoutingTable,
+        protocol_type: ProtocolType,
+        address_type: AddressType,
+    ) -> Vec<SocketAddress> {
+        let filter = DialInfoFilter::all()
+            .with_protocol_type(protocol_type)
+            .with_address_type(address_type);
+        routing_table
+            .dial_info_details(RoutingDomain::LocalNetwork)
+            .iter()
+            .filter_map(|did| {
+                if did.dial_info.matches_filter(&filter) {
+                    Some(did.dial_info.socket_address())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
+    // Ask for a public address check from a particular noderef
+    // This is done over the normal port using RPC
+    #[instrument(level = "trace", skip(self), ret)]
+    async fn request_public_address(&self, node_ref: NodeRef) -> Option<SocketAddress> {
+        let rpc = self.unlocked_inner.routing_table.rpc_processor();
+
+        let res = network_result_value_or_log!(match rpc.rpc_call_status(Destination::direct(node_ref.clone())).await {
+                Ok(v) => v,
+                Err(e) => {
+                    log_net!(error
+                        "failed to get status answer from {:?}: {}",
+                        node_ref, e
+                    );
+                    return None;
+                }
+            } => [ format!(": node_ref={}", node_ref) ] {
+                return None;
+            }
+        );
+
+        log_net!(
+            "request_public_address {:?}: Value({:?})",
+            node_ref,
+            res.answer
+        );
+        res.answer.map(|si| si.socket_address)
+    }
+
+    // find fast peers with a particular address type, and ask them to tell us what our external address is
+    // This is done over the normal port using RPC
+    #[instrument(level = "trace", skip(self), ret)]
+    async fn discover_external_addresses(&self) -> bool {
+        let node_count = {
+            let config = self.unlocked_inner.routing_table.network_manager().config();
+            let c = config.get();
+            c.network.dht.max_find_node_count as usize
+        };
+        let routing_domain = RoutingDomain::PublicInternet;
+        let protocol_type = self.unlocked_inner.protocol_type;
+        let address_type = self.unlocked_inner.address_type;
+
+        // Build an filter that matches our protocol and address type
+        // and excludes relayed nodes so we can get an accurate external address
+        let dial_info_filter = DialInfoFilter::all()
+            .with_protocol_type(protocol_type)
+            .with_address_type(address_type);
+        let inbound_dial_info_entry_filter = RoutingTable::make_inbound_dial_info_entry_filter(
+            routing_domain,
+            dial_info_filter.clone(),
+        );
+        let disallow_relays_filter = Box::new(
+            move |rti: &RoutingTableInner, v: Option<Arc<BucketEntry>>| {
+                let v = v.unwrap();
+                v.with(rti, |_rti, e| {
+                    if let Some(n) = e.signed_node_info(routing_domain) {
+                        n.relay_ids().is_empty()
+                    } else {
+                        false
+                    }
+                })
+            },
+        ) as RoutingTableEntryFilter;
+        let will_validate_dial_info_filter = Box::new(
+            move |rti: &RoutingTableInner, v: Option<Arc<BucketEntry>>| {
+                let entry = v.unwrap();
+                entry.with(rti, move |_rti, e| {
+                    e.node_info(routing_domain)
+                        .map(|ni| {
+                            ni.has_capability(CAP_VALIDATE_DIAL_INFO)
+                                && ni.is_fully_direct_inbound()
+                        })
+                        .unwrap_or(false)
+                })
+            },
+        ) as RoutingTableEntryFilter;
+
+        let filters = VecDeque::from([
+            inbound_dial_info_entry_filter,
+            disallow_relays_filter,
+            will_validate_dial_info_filter,
+        ]);
+
+        // Find public nodes matching this filter
+        let nodes = self
+            .unlocked_inner
+            .routing_table
+            .find_fast_public_nodes_filtered(node_count, filters);
+        if nodes.is_empty() {
+            log_net!(debug
+                "no external address detection peers of type {:?}:{:?}",
+                protocol_type,
+                address_type
+            );
+            return false;
+        }
+
+        // For each peer, ask them for our public address, filtering on desired dial info
+        let mut unord = FuturesUnordered::new();
+
+        let get_public_address_func = |node: NodeRef| {
+            let this = self.clone();
+            let node = node.filtered_clone(
+                NodeRefFilter::new()
+                    .with_routing_domain(routing_domain)
+                    .with_dial_info_filter(dial_info_filter.clone()),
+            );
+            async move {
+                if let Some(address) = this.request_public_address(node.clone()).await {
+                    let dial_info = this
+                        .unlocked_inner
+                        .net
+                        .make_dial_info(address, this.unlocked_inner.protocol_type);
+                    return Some(ExternalInfo {
+                        dial_info,
+                        address,
+                        node,
+                    });
+                }
+                None
+            }
+        };
+
+        let mut external_address_infos = Vec::new();
+
+        for ni in 0..nodes.len() - 1 {
+            let node = nodes[ni].clone();
+
+            let gpa_future = get_public_address_func(node);
+            unord.push(gpa_future);
+
+            // Always process two at a time so we get both addresses in parallel if possible
+            if unord.len() == 2 {
+                // Process one
+                if let Some(Some(ei)) = unord.next().await {
+                    external_address_infos.push(ei);
+                    if external_address_infos.len() == 2 {
+                        break;
+                    }
+                }
+            }
+        }
+        // Finish whatever is left if we need to
+        if external_address_infos.len() < 2 {
+            while let Some(res) = unord.next().await {
+                if let Some(ei) = res {
+                    external_address_infos.push(ei);
+                    if external_address_infos.len() == 2 {
+                        break;
+                    }
+                }
+            }
+        }
+        if external_address_infos.len() < 2 {
+            log_net!(debug "not enough peers responded with an external address");
+            return false;
+        }
+
+        {
+            let mut inner = self.inner.lock();
+            inner.external_1 = Some(external_address_infos[0].clone());
+            log_net!(debug "external_1: {:?}", inner.external_1);
+            inner.external_2 = Some(external_address_infos[1].clone());
+            log_net!(debug "external_2: {:?}", inner.external_2);
+        }
+
+        true
+    }
+
+    #[instrument(level = "trace", skip(self), ret)]
+    async fn validate_dial_info(
+        &self,
+        node_ref: NodeRef,
+        dial_info: DialInfo,
+        redirect: bool,
+    ) -> bool {
+        let rpc_processor = self.unlocked_inner.routing_table.rpc_processor();
+
+        // asking for node validation doesn't have to use the dial info filter of the dial info we are validating
+        let mut node_ref = node_ref.clone();
+        node_ref.set_filter(None);
+
+        // ask the node to send us a dial info validation receipt
+        let out = rpc_processor
+            .rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect)
+            .await
+            .map_err(logthru_net!(
+                "failed to send validate_dial_info to {:?}",
+                node_ref
+            ))
+            .unwrap_or(false);
+        out
+    }
+
+    #[instrument(level = "trace", skip(self), ret)]
+    async fn try_upnp_port_mapping(&self) -> Option<DialInfo> {
+        let protocol_type = self.unlocked_inner.protocol_type;
+        let low_level_protocol_type = protocol_type.low_level_protocol_type();
+        let address_type = self.unlocked_inner.address_type;
+        let local_port = self
+            .unlocked_inner
+            .net
+            .get_local_port(protocol_type)
+            .unwrap();
+        let external_1 = self.inner.lock().external_1.as_ref().unwrap().clone();
+
+        let igd_manager = self.unlocked_inner.net.unlocked_inner.igd_manager.clone();
+        let mut tries = 0;
+        loop {
+            tries += 1;
+
+            // Attempt a port mapping. If this doesn't succeed, it's not going to
+            let Some(mapped_external_address) = igd_manager
+                .map_any_port(low_level_protocol_type, address_type, local_port, Some(external_1.address.to_ip_addr()))
+                .await else
+            {
+                return None;
+            };
+
+            // Make dial info from the port mapping
+            let external_mapped_dial_info = self.unlocked_inner.net.make_dial_info(
+                SocketAddress::from_socket_addr(mapped_external_address),
+                protocol_type,
+            );
+
+            // Attempt to validate the port mapping
+            let mut validate_tries = 0;
+            loop {
+                validate_tries += 1;
+
+                // Ensure people can reach us. If we're firewalled off, this is useless
+                if self
+                    .validate_dial_info(
+                        external_1.node.clone(),
+                        external_mapped_dial_info.clone(),
+                        false,
+                    )
+                    .await
+                {
+                    return Some(external_mapped_dial_info);
+                }
+
+                if validate_tries == PORT_MAP_VALIDATE_TRY_COUNT {
+                    log_net!(debug "UPNP port mapping succeeded but port {}/{} is still unreachable.\nretrying\n",
+                    local_port, match low_level_protocol_type {
+                        LowLevelProtocolType::UDP => "udp",
+                        LowLevelProtocolType::TCP => "tcp",
+                    });
+                    sleep(PORT_MAP_VALIDATE_DELAY_MS).await
+                } else {
+                    break;
+                }
+            }
+
+            // Release the mapping if we're still unreachable
+            let _ = igd_manager
+                .unmap_port(
+                    low_level_protocol_type,
+                    address_type,
+                    external_1.address.port(),
+                )
+                .await;
+
+            if tries == PORT_MAP_TRY_COUNT {
+                warn!("UPNP port mapping succeeded but port {}/{} is still unreachable.\nYou may need to add a local firewall allowed port on this machine.\n",
+                    local_port, match low_level_protocol_type {
+                        LowLevelProtocolType::UDP => "udp",
+                        LowLevelProtocolType::TCP => "tcp",
+                    }
+                );
+                break;
+            }
+        }
+        None
+    }
+
+    ///////
+    // Per-protocol discovery routines
+
+    // If we know we are not behind NAT, check our firewall status
+    #[instrument(level = "trace", skip(self), ret)]
+    async fn protocol_process_no_nat(
+        &self,
+        unord: &mut FuturesUnordered<SendPinBoxFuture<Option<DetectedDialInfo>>>,
+    ) {
+        let external_1 = self.inner.lock().external_1.as_ref().unwrap().clone();
+
+        let this = self.clone();
+        let do_no_nat_fut: SendPinBoxFuture<Option<DetectedDialInfo>> = Box::pin(async move {
+            // Do a validate_dial_info on the external address from a redirected node
+            if this
+                .validate_dial_info(external_1.node.clone(), external_1.dial_info.clone(), true)
+                .await
+            {
+                // Add public dial info with Direct dialinfo class
+                Some(DetectedDialInfo::Detected(DialInfoDetail {
+                    dial_info: external_1.dial_info.clone(),
+                    class: DialInfoClass::Direct,
+                }))
+            } else {
+                // Add public dial info with Blocked dialinfo class
+                Some(DetectedDialInfo::Detected(DialInfoDetail {
+                    dial_info: external_1.dial_info.clone(),
+                    class: DialInfoClass::Blocked,
+                }))
+            }
+        });
+        unord.push(do_no_nat_fut);
+    }
+
+    // If we know we are behind NAT check what kind
+    #[instrument(level = "trace", skip(self), ret)]
+    async fn protocol_process_nat(
+        &self,
+        unord: &mut FuturesUnordered<SendPinBoxFuture<Option<DetectedDialInfo>>>,
+    ) {
+        // Get the external dial info for our use here
+        let (external_1, external_2) = {
+            let inner = self.inner.lock();
+            (
+                inner.external_1.as_ref().unwrap().clone(),
+                inner.external_2.as_ref().unwrap().clone(),
+            )
+        };
+
+        // If we have two different external addresses, then this is a symmetric NAT
+        if external_2.address != external_1.address {
+            let do_symmetric_nat_fut: SendPinBoxFuture<Option<DetectedDialInfo>> =
+                Box::pin(async move { Some(DetectedDialInfo::SymmetricNAT) });
+            unord.push(do_symmetric_nat_fut);
+            return;
+        }
+
+        // Manual Mapping Detection
+        ///////////
+        let this = self.clone();
+        if let Some(local_port) = self
+            .unlocked_inner
+            .net
+            .get_local_port(self.unlocked_inner.protocol_type)
+        {
+            if external_1.dial_info.port() != local_port {
+                let c_external_1 = external_1.clone();
+                let do_manual_map_fut: SendPinBoxFuture<Option<DetectedDialInfo>> =
+                    Box::pin(async move {
+                        // Do a validate_dial_info on the external address, but with the same port as the local port of local interface, from a redirected node
+                        // This test is to see if a node had manual port forwarding done with the same port number as the local listener
+                        let mut external_1_dial_info_with_local_port =
+                            c_external_1.dial_info.clone();
+                        external_1_dial_info_with_local_port.set_port(local_port);
+
+                        if this
+                            .validate_dial_info(
+                                c_external_1.node.clone(),
+                                external_1_dial_info_with_local_port.clone(),
+                                true,
+                            )
+                            .await
+                        {
+                            // Add public dial info with Direct dialinfo class
+                            return Some(DetectedDialInfo::Detected(DialInfoDetail {
+                                dial_info: external_1_dial_info_with_local_port,
+                                class: DialInfoClass::Direct,
+                            }));
+                        }
+
+                        None
+                    });
+                unord.push(do_manual_map_fut);
+            }
+        }
+
+        // NAT Detection
+        ///////////
+
+        // Full Cone NAT Detection
+        ///////////
+        let this = self.clone();
+        let do_nat_detect_fut: SendPinBoxFuture<Option<DetectedDialInfo>> = Box::pin(async move {
+            let mut retry_count = {
+                let c = this.unlocked_inner.net.config.get();
+                c.network.restricted_nat_retries
+            };
+
+            // Loop for restricted NAT retries
+            loop {
+                let mut ord = FuturesOrdered::new();
+
+                let c_this = this.clone();
+                let c_external_1 = external_1.clone();
+                let do_full_cone_fut: SendPinBoxFuture<Option<DetectedDialInfo>> =
+                    Box::pin(async move {
+                        // Let's see what kind of NAT we have
+                        // Does a redirected dial info validation from a different address and a random port find us?
+                        if c_this
+                            .validate_dial_info(
+                                c_external_1.node.clone(),
+                                c_external_1.dial_info.clone(),
+                                true,
+                            )
+                            .await
+                        {
+                            // Yes, another machine can use the dial info directly, so Full Cone
+                            // Add public dial info with full cone NAT network class
+
+                            return Some(DetectedDialInfo::Detected(DialInfoDetail {
+                                dial_info: c_external_1.dial_info,
+                                class: DialInfoClass::FullConeNAT,
+                            }));
+                        }
+                        None
+                    });
+                ord.push_back(do_full_cone_fut);
+
+                let c_this = this.clone();
+                let c_external_1 = external_1.clone();
+                let c_external_2 = external_2.clone();
+                let do_restricted_cone_fut: SendPinBoxFuture<Option<DetectedDialInfo>> =
+                    Box::pin(async move {
+                        // We are restricted, determine what kind of restriction
+
+                        // If we're going to end up as a restricted NAT of some sort
+                        // Address is the same, so it's address or port restricted
+
+                        // Do a validate_dial_info on the external address from a random port
+                        if c_this
+                            .validate_dial_info(
+                                c_external_2.node.clone(),
+                                c_external_1.dial_info.clone(),
+                                false,
+                            )
+                            .await
+                        {
+                            // Got a reply from a non-default port, which means we're only address restricted
+                            return Some(DetectedDialInfo::Detected(DialInfoDetail {
+                                dial_info: c_external_1.dial_info.clone(),
+                                class: DialInfoClass::AddressRestrictedNAT,
+                            }));
+                        }
+                        // Didn't get a reply from a non-default port, which means we are also port restricted
+                        Some(DetectedDialInfo::Detected(DialInfoDetail {
+                            dial_info: c_external_1.dial_info.clone(),
+                            class: DialInfoClass::PortRestrictedNAT,
+                        }))
+                    });
+                ord.push_back(do_restricted_cone_fut);
+
+                // Return the first result we get
+                let mut some_ddi = None;
+                while let Some(res) = ord.next().await {
+                    if let Some(ddi) = res {
+                        some_ddi = Some(ddi);
+                        break;
+                    }
+                }
+
+                if let Some(ddi) = some_ddi {
+                    if let DetectedDialInfo::Detected(did) = &ddi {
+                        // If we got something better than restricted NAT or we're done retrying
+                        if did.class < DialInfoClass::AddressRestrictedNAT || retry_count == 0 {
+                            return Some(ddi);
+                        }
+                    }
+                }
+                if retry_count == 0 {
+                    break;
+                }
+                retry_count -= 1;
+            }
+
+            None
+        });
+        unord.push(do_nat_detect_fut);
+    }
+
+    /// Add discovery futures to an unordered set that may detect dialinfo when they complete
+    pub async fn discover(
+        &self,
+        unord: &mut FuturesUnordered<SendPinBoxFuture<Option<DetectedDialInfo>>>,
+    ) {
+        let enable_upnp = {
+            let c = self.unlocked_inner.net.config.get();
+            c.network.upnp
+        };
+
+        // Do this right away because it's fast and every detection is going to need it
+        // Get our external addresses from two fast nodes
+        if !self.discover_external_addresses().await {
+            // If we couldn't get an external address, then we should just try the whole network class detection again later
+            return;
+        }
+
+        // UPNP Automatic Mapping
+        ///////////
+        if enable_upnp {
+            let this = self.clone();
+            let do_mapped_fut: SendPinBoxFuture<Option<DetectedDialInfo>> = Box::pin(async move {
+                // Attempt a port mapping via all available and enabled mechanisms
+                // Try this before the direct mapping in the event that we are restarting
+                // and may not have recorded a mapping created the last time
+                if let Some(external_mapped_dial_info) = this.try_upnp_port_mapping().await {
+                    // Got a port mapping, let's use it
+                    return Some(DetectedDialInfo::Detected(DialInfoDetail {
+                        dial_info: external_mapped_dial_info.clone(),
+                        class: DialInfoClass::Mapped,
+                    }));
+                }
+                None
+            });
+            unord.push(do_mapped_fut);
+        }
+
+        // NAT Detection
+        ///////////
+
+        // If our local interface list contains external_1 then there is no NAT in place
+        let external_1 = self.inner.lock().external_1.as_ref().unwrap().clone();
+
+        if self.unlocked_inner.intf_addrs.contains(&external_1.address) {
+            self.protocol_process_no_nat(unord).await;
+        } else {
+            self.protocol_process_nat(unord).await;
+        }
+    }
+}

veilid-core/src/network_manager/native/mod.rs

@@ -1,3 +1,4 @@
+mod discovery_context;
 mod igd_manager;
 mod network_class_discovery;
 mod network_tcp;
@@ -8,6 +9,7 @@ mod start_protocols;
 use super::*;
 use crate::routing_table::*;
 use connection_manager::*;
+use discovery_context::*;
 use network_interfaces::*;
 use network_tcp::*;
 use protocol::tcp::RawTcpProtocolHandler;
@@ -645,7 +647,11 @@ impl Network {
                 let peer_socket_addr = dial_info.to_socket_addr();
                 let ph = match self.find_best_udp_protocol_handler(&peer_socket_addr, &None) {
                     Some(ph) => ph,
-                    None => bail!("no appropriate UDP protocol handler for dial_info"),
+                    None => {
+                        return Ok(NetworkResult::no_connection_other(
+                            "no appropriate UDP protocol handler for dial_info",
+                        ));
+                    }
                 };
                 connection_descriptor = network_result_try!(ph
                     .send_message(data, peer_socket_addr)
@@ -874,8 +880,8 @@ impl Network {
         }
 
         // commit routing table edits
-        editor_public_internet.commit();
+        editor_public_internet.commit(true).await;
-        editor_local_network.commit();
+        editor_local_network.commit(true).await;
 
         info!("network started");
         self.inner.lock().network_started = true;
@@ -927,17 +933,19 @@ impl Network {
 
         routing_table
             .edit_routing_domain(RoutingDomain::PublicInternet)
-            .clear_dial_info_details()
+            .clear_dial_info_details(None, None)
             .set_network_class(None)
             .clear_relay_node()
-            .commit();
+            .commit(true)
+            .await;
 
         routing_table
             .edit_routing_domain(RoutingDomain::LocalNetwork)
-            .clear_dial_info_details()
+            .clear_dial_info_details(None, None)
             .set_network_class(None)
             .clear_relay_node()
-            .commit();
+            .commit(true)
+            .await;
 
         // Reset state including network class
         *self.inner.lock() = Self::new_inner();

veilid-core/src/network_manager/tasks/public_address_check.rs

@@ -77,7 +77,7 @@ impl NetworkManager {
         });
 
         // Get the ip(block) this report is coming from
-        let ipblock = ip_to_ipblock(
+        let reporting_ipblock = ip_to_ipblock(
             ip6_prefix_size,
             connection_descriptor.remote_address().to_ip_addr(),
         );
@@ -91,6 +91,13 @@ impl NetworkManager {
             return;
         }
 
+        // If the socket address reported is the same as the reporter, then this is coming through a relay
+        // or it should be ignored due to local proximity (nodes on the same network block should not be trusted as
+        // public ip address reporters, only disinterested parties)
+        if reporting_ipblock == ip_to_ipblock(ip6_prefix_size, socket_address.to_ip_addr()) {
+            return;
+        }
+
         // Check if the public address report is coming from a node/block that gives an 'inconsistent' location
         // meaning that the node may be not useful for public address detection
         // This is done on a per address/protocol basis
@@ -105,7 +112,7 @@ impl NetworkManager {
         if inner
             .public_address_inconsistencies_table
             .get(&addr_proto_type_key)
-            .map(|pait| pait.contains_key(&ipblock))
+            .map(|pait| pait.contains_key(&reporting_ipblock))
             .unwrap_or(false)
         {
             return;
@@ -117,7 +124,7 @@ impl NetworkManager {
             .public_address_check_cache
             .entry(addr_proto_type_key)
             .or_insert_with(|| LruCache::new(PUBLIC_ADDRESS_CHECK_CACHE_SIZE));
-        pacc.insert(ipblock, socket_address);
+        pacc.insert(reporting_ipblock, socket_address);
 
         // Determine if our external address has likely changed
         let mut bad_public_address_detection_punishment: Option<

veilid-core/src/network_manager/wasm/mod.rs

@@ -387,7 +387,7 @@ impl Network {
         editor_public_internet.set_network_class(Some(NetworkClass::WebApp));
 
         // commit routing table edits
-        editor_public_internet.commit();
+        editor_public_internet.commit(true).await;
 
         self.inner.lock().network_started = true;
         Ok(())
@@ -414,10 +414,11 @@ impl Network {
         // Drop all dial info
         routing_table
             .edit_routing_domain(RoutingDomain::PublicInternet)
-            .clear_dial_info_details()
+            .clear_dial_info_details(None, None)
             .set_network_class(None)
             .clear_relay_node()
-            .commit();
+            .commit(true)
+            .await;
 
         // Cancels all async background tasks by dropping join handles
         *self.inner.lock() = Self::new_inner();

veilid-core/src/routing_table/routing_domain_editor.rs

@@ -1,7 +1,10 @@
 use super::*;
 
 enum RoutingDomainChange {
-    ClearDialInfoDetails,
+    ClearDialInfoDetails {
+        address_type: Option<AddressType>,
+        protocol_type: Option<ProtocolType>,
+    },
     ClearRelayNode,
     SetRelayNode {
         relay_node: NodeRef,
@@ -39,8 +42,16 @@ impl RoutingDomainEditor {
     }
 
     #[instrument(level = "debug", skip(self))]
-    pub fn clear_dial_info_details(&mut self) -> &mut Self {
+    pub fn clear_dial_info_details(
-        self.changes.push(RoutingDomainChange::ClearDialInfoDetails);
+        &mut self,
+        address_type: Option<AddressType>,
+        protocol_type: Option<ProtocolType>,
+    ) -> &mut Self {
+        self.changes
+            .push(RoutingDomainChange::ClearDialInfoDetails {
+                address_type,
+                protocol_type,
+            });
         self
     }
     #[instrument(level = "debug", skip(self))]
@@ -111,32 +122,54 @@ impl RoutingDomainEditor {
     }
 
     #[instrument(level = "debug", skip(self))]
-    pub fn commit(&mut self) {
+    pub async fn commit(&mut self, pause_tasks: bool) {
         // No locking if we have nothing to do
         if self.changes.is_empty() {
             return;
         }
 
+        // Briefly pause routing table ticker while changes are made
+        if pause_tasks {
+            self.routing_table.pause_tasks(true).await;
+        }
+
+        // Apply changes
         let mut changed = false;
         {
-            let node_ids = self.routing_table.node_ids();
-
             let mut inner = self.routing_table.inner.write();
             inner.with_routing_domain_mut(self.routing_domain, |detail| {
                 for change in self.changes.drain(..) {
                     match change {
-                        RoutingDomainChange::ClearDialInfoDetails => {
+                        RoutingDomainChange::ClearDialInfoDetails {
-                            debug!("[{:?}] cleared dial info details", self.routing_domain);
+                            address_type,
-                            detail.common_mut().clear_dial_info_details();
+                            protocol_type,
+                        } => {
+                            if address_type.is_some() || protocol_type.is_some() {
+                                info!(
+                                    "[{:?}] cleared dial info: {}:{}",
+                                    self.routing_domain,
+                                    address_type
+                                        .map(|at| format!("{:?}", at))
+                                        .unwrap_or("---".to_string()),
+                                    protocol_type
+                                        .map(|at| format!("{:?}", at))
+                                        .unwrap_or("---".to_string()),
+                                );
+                            } else {
+                                info!("[{:?}] cleared all dial info", self.routing_domain);
+                            }
+                            detail
+                                .common_mut()
+                                .clear_dial_info_details(address_type, protocol_type);
                             changed = true;
                         }
                         RoutingDomainChange::ClearRelayNode => {
-                            debug!("[{:?}] cleared relay node", self.routing_domain);
+                            info!("[{:?}] cleared relay node", self.routing_domain);
                             detail.common_mut().set_relay_node(None);
                             changed = true;
                         }
                         RoutingDomainChange::SetRelayNode { relay_node } => {
-                            debug!("[{:?}] set relay node: {}", self.routing_domain, relay_node);
+                            info!("[{:?}] set relay node: {}", self.routing_domain, relay_node);
                             detail.common_mut().set_relay_node(Some(relay_node.clone()));
                             changed = true;
                         }
@@ -146,18 +179,16 @@ impl RoutingDomainEditor {
                             changed = true;
                         }
                         RoutingDomainChange::AddDialInfoDetail { dial_info_detail } => {
-                            debug!(
+                            info!(
-                                "[{:?}] add dial info detail: {:?}",
+                                "[{:?}] dial info: {:?}:{}",
-                                self.routing_domain, dial_info_detail
+                                self.routing_domain,
+                                dial_info_detail.class,
+                                dial_info_detail.dial_info
                             );
                             detail
                                 .common_mut()
                                 .add_dial_info_detail(dial_info_detail.clone());
 
-                            info!(
-                                "{:?} Dial Info: {}@{}",
-                                self.routing_domain, node_ids, dial_info_detail.dial_info
-                            );
                             changed = true;
                         }
                         RoutingDomainChange::SetupNetwork {
@@ -176,22 +207,22 @@ impl RoutingDomainEditor {
                                 || old_address_types != address_types
                                 || old_capabilities != *capabilities;
 
-                            debug!(
-                                "[{:?}] setup network: {:?} {:?} {:?} {:?}",
-                                self.routing_domain,
-                                outbound_protocols,
-                                inbound_protocols,
-                                address_types,
-                                capabilities
-                            );
-
-                            detail.common_mut().setup_network(
-                                outbound_protocols,
-                                inbound_protocols,
-                                address_types,
-                                capabilities.clone(),
-                            );
                             if this_changed {
+                                info!(
+                                    "[{:?}] setup network: {:?} {:?} {:?} {:?}",
+                                    self.routing_domain,
+                                    outbound_protocols,
+                                    inbound_protocols,
+                                    address_types,
+                                    capabilities
+                                );
+
+                                detail.common_mut().setup_network(
+                                    outbound_protocols,
+                                    inbound_protocols,
+                                    address_types,
+                                    capabilities.clone(),
+                                );
                                 changed = true;
                             }
                         }
@@ -199,14 +230,16 @@ impl RoutingDomainEditor {
                             let old_network_class = detail.common().network_class();
 
                             let this_changed = old_network_class != network_class;
-
-                            debug!(
-                                "[{:?}] set network class: {:?}",
-                                self.routing_domain, network_class,
-                            );
-
-                            detail.common_mut().set_network_class(network_class);
                             if this_changed {
+                                if let Some(network_class) = network_class {
+                                    info!(
+                                        "[{:?}] set network class: {:?}",
+                                        self.routing_domain, network_class,
+                                    );
+                                } else {
+                                    info!("[{:?}] cleared network class", self.routing_domain,);
+                                }
+                                detail.common_mut().set_network_class(network_class);
                                 changed = true;
                             }
                         }
@@ -229,5 +262,8 @@ impl RoutingDomainEditor {
                 rss.reset();
             }
         }
+
+        // Unpause routing table ticker
+        self.routing_table.pause_tasks(false).await;
     }
 }

veilid-core/src/routing_table/routing_domains.rs

@@ -103,8 +103,21 @@ impl RoutingDomainDetailCommon {
     pub fn dial_info_details(&self) -> &Vec<DialInfoDetail> {
         &self.dial_info_details
     }
-    pub(super) fn clear_dial_info_details(&mut self) {
+    pub(super) fn clear_dial_info_details(&mut self, address_type: Option<AddressType>, protocol_type: Option<ProtocolType>) {
-        self.dial_info_details.clear();
+        self.dial_info_details.retain_mut(|e| {
+            let mut remove = true;
+            if let Some(pt) = protocol_type {
+                if pt != e.dial_info.protocol_type() {
+                    remove = false;
+                }
+            }
+            if let Some(at) = address_type {
+                if at != e.dial_info.address_type() {
+                    remove = false;
+                }
+            }
+            !remove
+        });
         self.clear_cache();
     }
     pub(super) fn add_dial_info_detail(&mut self, did: DialInfoDetail) {

veilid-core/src/routing_table/routing_table_inner.rs

@@ -34,6 +34,8 @@ pub struct RoutingTableInner {
     pub(super) recent_peers: LruCache<TypedKey, RecentPeersEntry>,
     /// Storage for private/safety RouteSpecs
     pub(super) route_spec_store: Option<RouteSpecStore>,
+    /// Tick paused or not
+    pub(super) tick_paused: bool,
 }
 
 impl RoutingTableInner {
@@ -50,6 +52,7 @@ impl RoutingTableInner {
             self_transfer_stats: TransferStatsDownUp::default(),
             recent_peers: LruCache::new(RECENT_PEERS_TABLE_SIZE),
             route_spec_store: None,
+            tick_paused: false,
         }
     }
 

veilid-core/src/routing_table/tasks/mod.rs

@@ -125,6 +125,11 @@ impl RoutingTable {
     /// Ticks about once per second
     /// to run tick tasks which may run at slower tick rates as configured
     pub async fn tick(&self) -> EyreResult<()> {
+        // Don't tick if paused
+        if self.inner.read().tick_paused {
+            return Ok(());
+        }
+
         // Do rolling transfers every ROLLING_TRANSFERS_INTERVAL_SECS secs
         self.unlocked_inner.rolling_transfers_task.tick().await?;
 
@@ -168,13 +173,33 @@ impl RoutingTable {
         self.unlocked_inner.relay_management_task.tick().await?;
 
         // Run the private route management task
-        self.unlocked_inner
+        // If we don't know our network class then don't do this yet
-            .private_route_management_task
+        if self.has_valid_network_class(RoutingDomain::PublicInternet) {
-            .tick()
+            self.unlocked_inner
-            .await?;
+                .private_route_management_task
+                .tick()
+                .await?;
+        }
 
         Ok(())
     }
+    pub(crate) async fn pause_tasks(&self, paused: bool) {
+        let cancel = {
+            let mut inner = self.inner.write();
+            if !inner.tick_paused && paused {
+                inner.tick_paused = true;
+                true
+            } else if inner.tick_paused && !paused {
+                inner.tick_paused = false;
+                false
+            } else {
+                false
+            }
+        };
+        if cancel {
+            self.cancel_tasks().await;
+        }
+    }
 
     pub(crate) async fn cancel_tasks(&self) {
         // Cancel all tasks being ticked

veilid-core/src/routing_table/tasks/ping_validator.rs

@@ -12,7 +12,7 @@ impl RoutingTable {
     // Ping each node in the routing table if they need to be pinged
     // to determine their reliability
     #[instrument(level = "trace", skip(self), err)]
-    fn relay_keepalive_public_internet(
+    async fn relay_keepalive_public_internet(
         &self,
         cur_ts: Timestamp,
         relay_nr: NodeRef,
@@ -41,7 +41,8 @@ impl RoutingTable {
         // Say we're doing this keepalive now
         self.edit_routing_domain(RoutingDomain::PublicInternet)
             .set_relay_node_keepalive(Some(cur_ts))
-            .commit();
+            .commit(false)
+            .await;
 
         // We need to keep-alive at one connection per ordering for relays
         // but also one per NAT mapping that we need to keep open for our inbound dial info
@@ -119,7 +120,7 @@ impl RoutingTable {
     // Ping each node in the routing table if they need to be pinged
     // to determine their reliability
     #[instrument(level = "trace", skip(self), err)]
-    fn ping_validator_public_internet(
+    async fn ping_validator_public_internet(
         &self,
         cur_ts: Timestamp,
         unord: &mut FuturesUnordered<
@@ -136,7 +137,8 @@ impl RoutingTable {
 
         // If this is our relay, let's check for NAT keepalives
         if let Some(relay_nr) = opt_relay_nr {
-            self.relay_keepalive_public_internet(cur_ts, relay_nr, unord)?;
+            self.relay_keepalive_public_internet(cur_ts, relay_nr, unord)
+                .await?;
         }
 
         // Just do a single ping with the best protocol for all the other nodes to check for liveness
@@ -156,7 +158,7 @@ impl RoutingTable {
     // Ping each node in the LocalNetwork routing domain if they
     // need to be pinged to determine their reliability
     #[instrument(level = "trace", skip(self), err)]
-    fn ping_validator_local_network(
+    async fn ping_validator_local_network(
         &self,
         cur_ts: Timestamp,
         unord: &mut FuturesUnordered<
@@ -195,10 +197,12 @@ impl RoutingTable {
         let mut unord = FuturesUnordered::new();
 
         // PublicInternet
-        self.ping_validator_public_internet(cur_ts, &mut unord)?;
+        self.ping_validator_public_internet(cur_ts, &mut unord)
+            .await?;
 
         // LocalNetwork
-        self.ping_validator_local_network(cur_ts, &mut unord)?;
+        self.ping_validator_local_network(cur_ts, &mut unord)
+            .await?;
 
         // Wait for ping futures to complete in parallel
         while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {}

veilid-core/src/routing_table/tasks/private_route_management.rs

@@ -169,11 +169,6 @@ impl RoutingTable {
         _last_ts: Timestamp,
         cur_ts: Timestamp,
     ) -> EyreResult<()> {
-        // If we don't know our network class then don't do this yet
-        if !self.has_valid_network_class(RoutingDomain::PublicInternet) {
-            return Ok(());
-        }
-
         // Test locally allocated routes first
         // This may remove dead routes
         let routes_needing_testing = self.get_allocated_routes_to_test(cur_ts);

veilid-core/src/routing_table/tasks/relay_management.rs

@@ -23,13 +23,13 @@ impl RoutingTable {
                 let state = relay_node.state(cur_ts);
                 // Relay node is dead or no longer needed
                 if matches!(state, BucketEntryState::Dead) {
-                    info!("Relay node died, dropping relay {}", relay_node);
+                    debug!("Relay node died, dropping relay {}", relay_node);
                     editor.clear_relay_node();
                     false
                 }
                 // Relay node no longer can relay
                 else if relay_node.operate(|_rti, e| !relay_node_filter(e)) {
-                    info!(
+                    debug!(
                         "Relay node can no longer relay, dropping relay {}",
                         relay_node
                     );
@@ -38,7 +38,7 @@ impl RoutingTable {
                 }
                 // Relay node is no longer required
                 else if !own_node_info.requires_relay() {
-                    info!(
+                    debug!(
                         "Relay node no longer required, dropping relay {}",
                         relay_node
                     );
@@ -47,7 +47,7 @@ impl RoutingTable {
                 }
                 // Should not have relay for invalid network class
                 else if !self.has_valid_network_class(RoutingDomain::PublicInternet) {
-                    info!(
+                    debug!(
                         "Invalid network class does not get a relay, dropping relay {}",
                         relay_node
                     );
@@ -75,7 +75,7 @@ impl RoutingTable {
                         false,
                     ) {
                         Ok(nr) => {
-                            info!("Outbound relay node selected: {}", nr);
+                            debug!("Outbound relay node selected: {}", nr);
                             editor.set_relay_node(nr);
                             got_outbound_relay = true;
                         }
@@ -84,20 +84,20 @@ impl RoutingTable {
                         }
                     }
                 } else {
-                    info!("Outbound relay desired but not available");
+                    debug!("Outbound relay desired but not available");
                 }
             }
             if !got_outbound_relay {
                 // Find a node in our routing table that is an acceptable inbound relay
                 if let Some(nr) = self.find_inbound_relay(RoutingDomain::PublicInternet, cur_ts) {
-                    info!("Inbound relay node selected: {}", nr);
+                    debug!("Inbound relay node selected: {}", nr);
                     editor.set_relay_node(nr);
                 }
             }
         }
 
         // Commit the changes
-        editor.commit();
+        editor.commit(false).await;
 
         Ok(())
     }

veilid-core/src/veilid_api/debug.rs

@@ -570,7 +570,8 @@ impl VeilidAPI {
         routing_table
             .edit_routing_domain(routing_domain)
             .set_relay_node(relay_node)
-            .commit();
+            .commit(true)
+            .await;
         Ok("Relay changed".to_owned())
     }
 
@@ -581,17 +582,24 @@ impl VeilidAPI {
     }
 
     async fn debug_config(&self, args: String) -> VeilidAPIResult<String> {
-        let config = self.config()?;
+        let mut args = args.as_str();
+        let mut config = self.config()?;
+        if !args.starts_with("insecure") {
+            config = config.safe_config();
+        } else {
+            args = &args[8..];
+        }
         let args = args.trim_start();
+
         if args.is_empty() {
-            return config.get_key_json("");
+            return config.get_key_json("", true);
         }
         let (arg, rest) = args.split_once(' ').unwrap_or((args, ""));
         let rest = rest.trim_start().to_owned();
 
         // One argument is 'config get'
         if rest.is_empty() {
-            return config.get_key_json(arg);
+            return config.get_key_json(arg, true);
         }
 
         // More than one argument is 'config set'
@@ -1371,7 +1379,7 @@ peerinfo [routingdomain]
 entries [dead|reliable]
 entry <node>
 nodeinfo
-config [configkey [new value]]
+config [insecure] [configkey [new value]]
 txtrecord
 keypair
 purge <buckets|connections|routes>

veilid-core/src/veilid_config.rs

@@ -576,7 +576,7 @@ impl VeilidConfig {
         self.inner.read()
     }
 
-    pub fn safe_config(&self) -> VeilidConfigInner {
+    fn safe_config_inner(&self) -> VeilidConfigInner {
         let mut safe_cfg = self.inner.read().clone();
 
         // Remove secrets
@@ -587,6 +587,20 @@ impl VeilidConfig {
         safe_cfg
     }
 
+    pub fn safe_config(&self) -> VeilidConfig {
+        let mut safe_cfg = self.inner.read().clone();
+
+        // Remove secrets
+        safe_cfg.network.routing_table.node_id_secret = TypedSecretGroup::new();
+        safe_cfg.protected_store.device_encryption_key_password = "".to_owned();
+        safe_cfg.protected_store.new_device_encryption_key_password = None;
+
+        VeilidConfig {
+            update_cb: self.update_cb.clone(),
+            inner: Arc::new(RwLock::new(safe_cfg)),
+        }
+    }
+
     pub fn with_mut<F, R>(&self, f: F) -> VeilidAPIResult<R>
     where
         F: FnOnce(&mut VeilidConfigInner) -> VeilidAPIResult<R>,
@@ -611,14 +625,14 @@ impl VeilidConfig {
 
         // Send configuration update to clients
         if let Some(update_cb) = &self.update_cb {
-            let safe_cfg = self.safe_config();
+            let safe_cfg = self.safe_config_inner();
             update_cb(VeilidUpdate::Config(VeilidStateConfig { config: safe_cfg }));
         }
 
         Ok(out)
     }
 
-    pub fn get_key_json(&self, key: &str) -> VeilidAPIResult<String> {
+    pub fn get_key_json(&self, key: &str, pretty: bool) -> VeilidAPIResult<String> {
         let c = self.get();
 
         // Generate json from whole config
@@ -627,7 +641,11 @@ impl VeilidConfig {
 
         // Find requested subkey
         if key.is_empty() {
-            Ok(jvc.to_string())
+            Ok(if pretty {
+                jvc.pretty(2)
+            } else {
+                jvc.to_string()
+            })
         } else {
             // Split key into path parts
             let keypath: Vec<&str> = key.split('.').collect();
@@ -638,7 +656,11 @@ impl VeilidConfig {
                 }
                 out = &out[k];
             }
-            Ok(out.to_string())
+            Ok(if pretty {
+                out.pretty(2)
+            } else {
+                out.to_string()
+            })
         }
     }
     pub fn set_key_json(&self, key: &str, value: &str) -> VeilidAPIResult<()> {

veilid-flutter/example/pubspec.lock

@@ -61,10 +61,10 @@ packages:
     dependency: transitive
     description:
       name: collection
-      sha256: "4a07be6cb69c84d677a6c3096fcf960cc3285a8330b4603e0d463d15d9bd934c"
+      sha256: f092b211a4319e98e5ff58223576de6c2803db36221657b46c82574721240687
       url: "https://pub.dev"
     source: hosted
-    version: "1.17.1"
+    version: "1.17.2"
   convert:
     dependency: transitive
     description:
@@ -168,14 +168,6 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.0.0"
-  js:
-    dependency: transitive
-    description:
-      name: js
-      sha256: f2c445dce49627136094980615a031419f7f3eb393237e4ecd97ac15dea343f3
-      url: "https://pub.dev"
-    source: hosted
-    version: "0.6.7"
   json_annotation:
     dependency: transitive
     description:
@@ -212,18 +204,18 @@ packages:
     dependency: transitive
     description:
       name: matcher
-      sha256: "6501fbd55da300384b768785b83e5ce66991266cec21af89ab9ae7f5ce1c4cbb"
+      sha256: "1803e76e6653768d64ed8ff2e1e67bea3ad4b923eb5c56a295c3e634bad5960e"
       url: "https://pub.dev"
     source: hosted
-    version: "0.12.15"
+    version: "0.12.16"
   material_color_utilities:
     dependency: transitive
     description:
       name: material_color_utilities
-      sha256: d92141dc6fe1dad30722f9aa826c7fbc896d021d792f80678280601aff8cf724
+      sha256: "9528f2f296073ff54cb9fee677df673ace1218163c3bc7628093e7eed5203d41"
       url: "https://pub.dev"
     source: hosted
-    version: "0.2.0"
+    version: "0.5.0"
   meta:
     dependency: transitive
     description:
@@ -329,10 +321,10 @@ packages:
     dependency: transitive
     description:
       name: source_span
-      sha256: dd904f795d4b4f3b870833847c461801f6750a9fa8e61ea5ac53f9422b31f250
+      sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c"
       url: "https://pub.dev"
     source: hosted
-    version: "1.9.1"
+    version: "1.10.0"
   stack_trace:
     dependency: transitive
     description:
@@ -385,10 +377,10 @@ packages:
     dependency: transitive
     description:
       name: test_api
-      sha256: eb6ac1540b26de412b3403a163d919ba86f6a973fe6cc50ae3541b80092fdcfb
+      sha256: "75760ffd7786fffdfb9597c35c5b27eaeec82be8edfb6d71d32651128ed7aab8"
       url: "https://pub.dev"
     source: hosted
-    version: "0.5.1"
+    version: "0.6.0"
   typed_data:
     dependency: transitive
     description:
@@ -411,7 +403,15 @@ packages:
       path: ".."
       relative: true
     source: path
-    version: "0.2.0"
+    version: "0.2.1"
+  web:
+    dependency: transitive
+    description:
+      name: web
+      sha256: dc8ccd225a2005c1be616fe02951e2e342092edf968cf0844220383757ef8f10
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.1.4-beta"
   win32:
     dependency: transitive
     description:
@@ -437,5 +437,5 @@ packages:
     source: hosted
     version: "3.5.0"
 sdks:
-  dart: ">=3.0.0 <4.0.0"
+  dart: ">=3.1.0-185.0.dev <4.0.0"
   flutter: ">=3.10.6"