add better dht debugging

veilid-core/src/routing_table/find_peers.rs

@@ -99,8 +99,8 @@ impl RoutingTable {
             },
         );
 
-        // xxx test
         // Validate peers returned are, in fact, closer to the key than the node we sent this to
+        // This same test is used on the other side so we vet things here
         let valid = match Self::verify_peers_closer(vcrypto2, own_node_id, key, &closest_nodes) {
             Ok(v) => v,
             Err(e) => {
@@ -108,13 +108,16 @@ impl RoutingTable {
             }
         };
         if !valid {
-            panic!("non-closer peers returned");
+            error!(
+                "non-closer peers returned: own_node_id={:#?} key={:#?} closest_nodes={:#?}",
+                own_node_id, key, closest_nodes
+            );
         }
 
         NetworkResult::value(closest_nodes)
     }
 
-    /// Determine if set of peers is closer to key_near than key_far
+    /// Determine if set of peers is closer to key_near than key_far is to key_near
     pub(crate) fn verify_peers_closer(
         vcrypto: CryptoSystemVersion,
         key_far: TypedKey,
@@ -128,14 +131,30 @@ impl RoutingTable {
         }
 
         let mut closer = true;
+        let d_far = vcrypto.distance(&key_far.value, &key_near.value);
         for peer in peers {
             let Some(key_peer) = peer.node_ids().get(kind) else {
                 bail!("peers need to have a key with the same cryptosystem");
             };
             let d_near = vcrypto.distance(&key_near.value, &key_peer.value);
-            let d_far = vcrypto.distance(&key_far.value, &key_peer.value);
             if d_far < d_near {
+                let warning = format!(
+                    r#"peer: {}
+near (key): {} 
+far (self): {} 
+    d_near: {}
+     d_far: {}
+       cmp: {:?}"#,
+                    key_peer.value,
+                    key_near.value,
+                    key_far.value,
+                    d_near,
+                    d_far,
+                    d_near.cmp(&d_far)
+                );
+                warn!("{}", warning);
                 closer = false;
+                break;
             }
         }
 

veilid-core/src/routing_table/tasks/private_route_management.rs

@@ -198,12 +198,12 @@ impl RoutingTable {
             // Newly allocated routes
             let mut newly_allocated_routes = Vec::new();
             for _n in 0..routes_to_allocate {
-                // Parameters here must be the default safety route spec
+                // Parameters here must be the most inclusive safety route spec
                 // These will be used by test_remote_route as well
                 if let Some(k) = rss.allocate_route(
                     &VALID_CRYPTO_KINDS,
                     Stability::default(),
-                    Sequencing::default(),
+                    Sequencing::EnsureOrdered,
                     default_route_hop_count,
                     DirectionSet::all(),
                     &[],