liz/veilid
liz/veilid
1
0
Fork 0
Code Pull Requests Packages Releases Activity

more refactor

Browse Source
This commit is contained in:
John Smith 2023-02-25 19:51:14 -05:00
parent 7962d3fe11
commit 4bc3f950df
17 changed files with 587 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
veilid-core/src/crypto/byte_array_types.rs
View File

@ -41,6 +41,9 @@ pub const SHARED_SECRET_LENGTH: usize = 32;
/// Length of a shared secret in bytes after encoding to base64url
#[allow(dead_code)]
pub const SHARED_SECRET_LENGTH_ENCODED: usize = 43;
/// Length of a route id in bytes
#[allow(dead_code)]
pub const ROUTE_ID_LENGTH: usize = 32;
//////////////////////////////////////////////////////////////////////
@ -260,3 +263,4 @@ byte_array_type!(Signature, SIGNATURE_LENGTH);
byte_array_type!(PublicKeyDistance, PUBLIC_KEY_LENGTH);
byte_array_type!(Nonce, NONCE_LENGTH);
byte_array_type!(SharedSecret, SHARED_SECRET_LENGTH);
byte_array_type!(RouteId, ROUTE_ID_LENGTH);

1
veilid-core/src/crypto/types.rs
View File

@ -11,6 +11,7 @@ use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as
pub type CryptoKind = FourCC;
/// Sort best crypto kinds first
/// Better crypto kinds are 'less', ordered toward the front of a list
pub fn compare_crypto_kind(a: &CryptoKind, b: &CryptoKind) -> cmp::Ordering {
let a_idx = VALID_CRYPTO_KINDS.iter().position(|k| k == a);
let b_idx = VALID_CRYPTO_KINDS.iter().position(|k| k == b);

8
veilid-core/src/routing_table/route_spec_store/mod.rs
View File

@ -1,5 +1,6 @@
use super::*;
mod permutation;
mod remote_private_route_info;
mod route_set_spec_detail;
mod route_spec_store;
@ -7,6 +8,7 @@ mod route_spec_store_cache;
mod route_spec_store_content;
mod route_stats;
pub use permutation::*;
pub use remote_private_route_info::*;
pub use route_set_spec_detail::*;
pub use route_spec_store::*;
@ -27,9 +29,3 @@ const REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY: TimestampDuration = TimestampDuration::
const ROUTE_MIN_IDLE_TIME_MS: u32 = 30_000;
/// The size of the compiled route cache
const COMPILED_ROUTE_CACHE_SIZE: usize = 256;
/// The type of an allocated route set id
pub type RouteSetSpecId = String;
/// Type type of an imported remote route set id
pub type RemotePrivateRouteId = String;

70
veilid-core/src/routing_table/route_spec_store/permutation.rs Normal file
View File

@ -0,0 +1,70 @@
use super::*;
/// number of route permutations is the number of unique orderings
/// for a set of nodes, given that the first node is fixed
fn _get_route_permutation_count(hop_count: usize) -> usize {
if hop_count == 0 {
unreachable!();
}
// a single node or two nodes is always fixed
if hop_count == 1 || hop_count == 2 {
return 1;
}
// more than two nodes has factorial permutation
// hop_count = 3 -> 2! -> 2
// hop_count = 4 -> 3! -> 6
(3..hop_count).into_iter().fold(2usize, |acc, x| acc * x)
}
pub type PermReturnType = (Vec<usize>, bool);
pub type PermFunc<'t> = Box<dyn Fn(&[usize]) -> Option<PermReturnType> + Send + 't>;
/// get the route permutation at particular 'perm' index, starting at the 'start' index
/// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum
/// number of permutations is given by get_route_permutation_count()
pub fn with_route_permutations(
hop_count: usize,
start: usize,
f: &PermFunc,
) -> Option<PermReturnType> {
if hop_count == 0 {
unreachable!();
}
// initial permutation
let mut permutation: Vec<usize> = Vec::with_capacity(hop_count);
for n in 0..hop_count {
permutation.push(start + n);
}
// if we have one hop or two, then there's only one permutation
if hop_count == 1 || hop_count == 2 {
return f(&permutation);
}
// heaps algorithm, but skipping the first element
fn heaps_permutation(
permutation: &mut [usize],
size: usize,
f: &PermFunc,
) -> Option<PermReturnType> {
if size == 1 {
return f(&permutation);
}
for i in 0..size {
let out = heaps_permutation(permutation, size - 1, f);
if out.is_some() {
return out;
}
if size % 2 == 1 {
permutation.swap(1, size);
} else {
permutation.swap(1 + i, size);
}
}
None
}
// recurse
heaps_permutation(&mut permutation, hop_count - 1, f)
}

7
veilid-core/src/routing_table/route_spec_store/remote_private_route_info.rs
View File

@ -32,6 +32,13 @@ impl RemotePrivateRouteInfo {
&mut self.stats
}
pub fn has_seen_our_node_info_ts(&mut self, our_node_info_ts: Timestamp) -> bool {
self.last_seen_our_node_info_ts == our_node_info_ts
}
pub fn set_last_seen_our_node_info_ts(&mut self, last_seen_our_node_info_ts: Timestamp) {
self.last_seen_our_node_info_ts = last_seen_our_node_info_ts;
}
// Check to see if this remote private route has expired
pub fn did_expire(&self, cur_ts: Timestamp) -> bool {
cur_ts.saturating_sub(self.last_touched_ts) >= REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY

31
veilid-core/src/routing_table/route_spec_store/route_set_spec_detail.rs
View File

@ -49,11 +49,22 @@ impl RouteSetSpecDetail {
}
tks
}
pub fn get_best_route_set_key(&self) -> Option<PublicKey> {
self.get_route_set_keys().best().map(|k| k.key)
}
pub fn set_hop_node_refs(&mut self, node_refs: Vec<NodeRef>) {
self.hop_node_refs = node_refs;
}
pub fn iter_route_set(
&self,
) -> alloc::collections::btree_map::Iter<PublicKey, RouteSpecDetail> {
self.route_set.iter()
}
pub fn iter_route_set_mut(
&self,
) -> alloc::collections::btree_map::IterMut<PublicKey, RouteSpecDetail> {
self.route_set.iter_mut()
}
pub fn get_stats(&self) -> &RouteStats {
&self.stats
}
@ -89,24 +100,4 @@ impl RouteSetSpecDetail {
}
cache
}
/// Generate a user-facing identifier for this allocated route
pub fn make_id(&self) -> RouteSetSpecId {
let mut idbytes = [0u8; 16];
for (pk, _) in self.route_set.iter() {
for (i, x) in pk.bytes.iter().enumerate() {
idbytes[i % 16] ^= *x;
}
}
let id = format!(
"{:08x}-{:04x}-{:04x}-{:04x}-{:08x}{:04x}",
u32::from_be_bytes(idbytes[0..4].try_into().expect("32 bits")),
u16::from_be_bytes(idbytes[4..6].try_into().expect("16 bits")),
u16::from_be_bytes(idbytes[6..8].try_into().expect("16 bits")),
u16::from_be_bytes(idbytes[8..10].try_into().expect("16 bits")),
u32::from_be_bytes(idbytes[10..14].try_into().expect("32 bits")),
u16::from_be_bytes(idbytes[14..16].try_into().expect("16 bits"))
);
id
}
}

487
veilid-core/src/routing_table/route_spec_store/route_spec_store.rs
View File

@ -1,4 +1,5 @@
use super::*;
use permutation::*;
#[derive(Debug)]
pub struct RouteSpecStoreInner {
@ -33,75 +34,6 @@ pub struct RouteSpecStore {
unlocked_inner: Arc<RouteSpecStoreUnlockedInner>,
}
/// number of route permutations is the number of unique orderings
/// for a set of nodes, given that the first node is fixed
fn _get_route_permutation_count(hop_count: usize) -> usize {
if hop_count == 0 {
unreachable!();
}
// a single node or two nodes is always fixed
if hop_count == 1 || hop_count == 2 {
return 1;
}
// more than two nodes has factorial permutation
// hop_count = 3 -> 2! -> 2
// hop_count = 4 -> 3! -> 6
(3..hop_count).into_iter().fold(2usize, |acc, x| acc * x)
}
type PermReturnType = (Vec<usize>, Vec<u8>, bool);
type PermFunc<'t> = Box<dyn Fn(&[usize]) -> Option<PermReturnType> + Send + 't>;
/// get the route permutation at particular 'perm' index, starting at the 'start' index
/// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum
/// number of permutations is given by get_route_permutation_count()
fn with_route_permutations(
hop_count: usize,
start: usize,
f: &PermFunc,
) -> Option<PermReturnType> {
if hop_count == 0 {
unreachable!();
}
// initial permutation
let mut permutation: Vec<usize> = Vec::with_capacity(hop_count);
for n in 0..hop_count {
permutation.push(start + n);
}
// if we have one hop or two, then there's only one permutation
if hop_count == 1 || hop_count == 2 {
return f(&permutation);
}
// heaps algorithm, but skipping the first element
fn heaps_permutation(
permutation: &mut [usize],
size: usize,
f: &PermFunc,
) -> Option<PermReturnType> {
if size == 1 {
return f(&permutation);
}
for i in 0..size {
let out = heaps_permutation(permutation, size - 1, f);
if out.is_some() {
return out;
}
if size % 2 == 1 {
permutation.swap(1, size);
} else {
permutation.swap(1 + i, size);
}
}
None
}
// recurse
heaps_permutation(&mut permutation, hop_count - 1, f)
}
impl RouteSpecStore {
pub fn new(routing_table: RoutingTable) -> Self {
let config = routing_table.network_manager().config();
@ -135,69 +67,7 @@ impl RouteSpecStore {
};
// Get frozen blob from table store
let table_store = routing_table.network_manager().table_store();
let rsstdb = table_store.open("RouteSpecStore", 1).await?;
let mut content: RouteSpecStoreContent =
rsstdb.load_rkyv(0, b"content")?.unwrap_or_default();
// Look up all route hop noderefs since we can't serialize those
let mut dead_ids = Vec::new();
for (rsid, rssd) in content.iter_details_mut() {
// Get first route since they all should resolve
let Some((pk, rsd)) = rssd.route_set.first_key_value() else {
dead_ids.push(rsid.clone());
continue;
};
// Go through first route
for h in &rsd.hops {
let Some(nr) = routing_table.lookup_node_ref(TypedKey::new(rsd.crypto_kind, *h)) else {
dead_ids.push(rsid.clone());
break;
};
rssd.hop_node_refs.push(nr);
}
}
for id in dead_ids {
log_rtab!(debug "no entry, killing off private route: {}", id);
content.remove_detail(&id);
}
// Load secrets from pstore
let pstore = routing_table.network_manager().protected_store();
let secret_key_map: HashMap<PublicKey, SecretKey> = pstore
.load_user_secret_rkyv("RouteSpecStore")
.await?
.unwrap_or_default();
// Ensure we got secret keys for all the public keys
let mut got_secret_key_ids = HashSet::new();
for (rsid, rssd) in content.iter_details_mut() {
let mut found_all = true;
for (pk, rsd) in &mut rssd.route_set {
if let Some(sk) = secret_key_map.get(pk) {
rsd.secret_key = *sk;
} else {
found_all = false;
break;
}
}
if found_all {
got_secret_key_ids.insert(rsid.clone());
}
}
// If we missed any, nuke those route ids
let dead_ids:Vec<String> = content.keys().filter_map(|id| {
if !got_secret_key_ids.contains(id) {
Some(id.clone())
} else {
None
}
}).collect();
for id in dead_ids {
log_rtab!(debug "missing secret key, killing off private route: {}", id);
content.remove_detail(&id);
}
let content = RouteSpecStoreContent::load(routing_table.clone()).await?;
let mut inner = RouteSpecStoreInner {
content,
@ -230,31 +100,8 @@ impl RouteSpecStore {
inner.content.clone()
};
// Save all the fields we care about to the frozen blob in table storage
// This skips #[with(Skip)] saving the secret keys, we save them in the protected store instead
let table_store = self
.unlocked_inner
.routing_table
.network_manager()
.table_store();
let rsstdb = table_store.open("RouteSpecStore", 1).await?;
rsstdb.store_rkyv(0, b"content", &content).await?;
// // Keep secrets in protected store as well
let pstore = self
.unlocked_inner
.routing_table
.network_manager()
.protected_store();
let mut out: HashMap<PublicKey, SecretKey> = HashMap::new();
for (rsid, rssd) in content.iter_details() {
for (pk, rsd) in &rssd.route_set {
out.insert(*pk, rsd.secret_key);
}
}
let _ = pstore.save_user_secret_rkyv("RouteSpecStore", &out).await?; // ignore if this previously existed or not
// Save our content
content.save(self.unlocked_inner.routing_table.clone()).await?;
Ok(())
}
@ -263,7 +110,11 @@ impl RouteSpecStore {
pub fn send_route_update(&self) {
let (dead_routes, dead_remote_routes) = {
let mut inner = self.inner.lock();
inner.cache.take_dead_routes()
let Some(dr) = inner.cache.take_dead_routes() else {
// Nothing to do
return;
};
dr
};
let update = VeilidUpdate::Route(VeilidStateRoute {
@ -275,8 +126,6 @@ impl RouteSpecStore {
update_callback(update);
}
/// Purge the route spec store
pub async fn purge(&self) -> EyreResult<()> {
{
@ -301,7 +150,7 @@ impl RouteSpecStore {
hop_count: usize,
directions: DirectionSet,
avoid_nodes: &[TypedKey],
) -> EyreResult<Option<String>> {
) -> EyreResult<Option<RouteId>> {
let inner = &mut *self.inner.lock();
let routing_table = self.unlocked_inner.routing_table.clone();
let rti = &mut *routing_table.inner.write();
@ -329,7 +178,7 @@ impl RouteSpecStore {
hop_count: usize,
directions: DirectionSet,
avoid_nodes: &[TypedKey],
) -> EyreResult<Option<String>> {
) -> EyreResult<Option<RouteId>> {
use core::cmp::Ordering;
if hop_count < 1 {
@ -625,7 +474,7 @@ impl RouteSpecStore {
for start in 0..(nodes.len() - hop_count) {
// Try the permutations available starting with 'start'
if let Some((rn, ck, cds)) = with_route_permutations(hop_count, start, &perm_func) {
if let Some((rn, cds)) = with_route_permutations(hop_count, start, &perm_func) {
route_nodes = rn;
can_do_sequenced = cds;
break;
@ -666,11 +515,14 @@ impl RouteSpecStore {
drop(perm_func);
// make id
let id = self.generate_allocated_route_id(&rssd)?;
// Add to cache
inner.cache.add_to_cache(&rssd);
// Keep route in spec store
let id = inner.content.add_detail(rssd);
inner.content.add_detail(id.clone(), rssd);
Ok(Some(id))
}
@ -737,23 +589,18 @@ impl RouteSpecStore {
}
#[instrument(level = "trace", skip(self), ret, err)]
async fn test_allocated_route(&self, id: &String) -> EyreResult<bool> {
async fn test_allocated_route(&self, private_route_id: RouteId) -> EyreResult<bool> {
// Make loopback route to test with
let dest = {
// Get best route from set
// Match the private route's hop length for safety route length
let (key, hop_count) = {
let hop_count = {
let inner = &mut *self.inner.lock();
let Some(rssd) = inner.content.get_detail(id) else {
let Some(rssd) = inner.content.get_detail(&private_route_id) else {
bail!("route id not allocated");
};
let Some(tkey) = rssd.get_route_set_keys().best() else {
bail!("route does not have best key");
};
(tkey.key, rssd.hop_count())
rssd.hop_count()
};
let private_route = self.assemble_private_route(&key, None)?;
// Always test routes with safety routes that are more likely to succeed
let stability = Stability::Reliable;
@ -761,7 +608,7 @@ impl RouteSpecStore {
let sequencing = Sequencing::NoPreference;
let safety_spec = SafetySpec {
preferred_route: Some(key.clone()),
preferred_route: Some(private_route_id),
hop_count,
stability,
sequencing,
@ -769,7 +616,7 @@ impl RouteSpecStore {
let safety_selection = SafetySelection::Safe(safety_spec);
Destination::PrivateRoute {
private_route,
private_route_id,
safety_selection,
}
};
@ -788,18 +635,10 @@ impl RouteSpecStore {
}
#[instrument(level = "trace", skip(self), ret, err)]
async fn test_remote_route(&self, id: &String) -> EyreResult<bool> {
async fn test_remote_route(&self, private_route_id: RouteId) -> EyreResult<bool> {
// Make private route test
let dest = {
// Get best remote route from imported set
// Get the route to test
let private_route = match self.peek_remote_private_route(id) {
Some(pr) => pr,
None => return Ok(false),
};
// Get a safety route that is good enough
let safety_spec = SafetySpec {
preferred_route: None,
@ -811,7 +650,7 @@ impl RouteSpecStore {
let safety_selection = SafetySelection::Safe(safety_spec);
Destination::PrivateRoute {
private_route,
private_route_id,
safety_selection,
}
};
@ -829,52 +668,44 @@ impl RouteSpecStore {
Ok(true)
}
/// Test an allocated route for continuity
#[instrument(level = "trace", skip(self), ret, err)]
pub async fn test_route(&self, key: &[TypedKey]) -> EyreResult<bool> {
let is_remote = {
let inner = &mut *self.inner.lock();
let cur_ts = get_aligned_timestamp();
Self::with_peek_remote_private_route(inner, cur_ts, key, |_| {}).is_some()
};
if is_remote {
self.test_remote_route(key).await
} else {
self.test_allocated_route(key).await
}
}
/// Release an allocated route that is no longer in use
#[instrument(level = "trace", skip(self), ret)]
fn release_allocated_route(&self, id: &String) -> bool {
fn release_allocated_route(&self, id: RouteId) -> bool {
let mut inner = self.inner.lock();
let Some(rssd) = inner.content.remove_detail(id) else {
let Some(rssd) = inner.content.remove_detail(&id) else {
return false;
};
// Remove from hop cache
if !inner.cache.remove_from_cache(&rssd) {
if !inner.cache.remove_from_cache(id, &rssd) {
panic!("hop cache should have contained cache key");
}
true
}
/// Check if a route id is remote or not
fn is_route_id_remote(&self, id: &RouteId) -> bool {
let inner = &mut *self.inner.lock();
let cur_ts = get_aligned_timestamp();
inner.cache.peek_remote_private_route_mut(cur_ts, &id).is_some()
}
/// Test an allocated route for continuity
#[instrument(level = "trace", skip(self), ret, err)]
pub async fn test_route(&self, id: RouteId) -> EyreResult<bool> {
let is_remote = self.is_route_id_remote(&id);
if is_remote {
self.test_remote_route(id).await
} else {
self.test_allocated_route(id).await
}
}
/// Release an allocated or remote route that is no longer in use
#[instrument(level = "trace", skip(self), ret)]
pub fn release_route(&self, id: &String) -> bool {
let is_remote = {
let inner = &mut *self.inner.lock();
// Release from compiled route cache if it's used there
self.invalidate_compiled_route_cache(inner, id);
// Check to see if this is a remote route
let cur_ts = get_aligned_timestamp();
Self::with_peek_remote_private_route(inner, cur_ts, id, |_| {}).is_some()
};
pub fn release_route(&self, id: RouteId) -> bool {
let is_remote = self.is_route_id_remote(&id);
if is_remote {
self.release_remote_private_route(id)
} else {
@ -943,11 +774,11 @@ impl RouteSpecStore {
/// List all allocated routes
pub fn list_allocated_routes<F, R>(&self, mut filter: F) -> Vec<R>
where
F: FnMut(&PublicKey, &RouteSetSpecDetail) -> Option<R>,
F: FnMut(&RouteId, &RouteSetSpecDetail) -> Option<R>,
{
let inner = self.inner.lock();
let mut out = Vec::with_capacity(inner.content.details.len());
for detail in &inner.content.details {
let mut out = Vec::with_capacity(inner.content.get_detail_count());
for detail in inner.content.iter_details() {
if let Some(x) = filter(detail.0, detail.1) {
out.push(x);
}
@ -958,11 +789,11 @@ impl RouteSpecStore {
/// List all allocated routes
pub fn list_remote_routes<F, R>(&self, mut filter: F) -> Vec<R>
where
F: FnMut(&PublicKey, &RemotePrivateRouteInfo) -> Option<R>,
F: FnMut(&RouteId, &RemotePrivateRouteInfo) -> Option<R>,
{
let inner = self.inner.lock();
let mut out = Vec::with_capacity(inner.cache.remote_private_route_cache.len());
for info in &inner.cache.remote_private_route_cache {
let mut out = Vec::with_capacity(inner.cache.get_remote_private_route_count());
for info in inner.cache.iter_remote_private_routes() {
if let Some(x) = filter(info.0, info.1) {
out.push(x);
}
@ -971,56 +802,20 @@ impl RouteSpecStore {
}
/// Get the debug description of a route
pub fn debug_route(&self, key: &PublicKey) -> Option<String> {
pub fn debug_route(&self, id: &RouteId) -> Option<String> {
let inner = &mut *self.inner.lock();
let cur_ts = get_aligned_timestamp();
// If this is a remote route, print it
if let Some(s) =
Self::with_peek_remote_private_route(inner, cur_ts, key, |rpi| format!("{:#?}", rpi))
{
return Some(s);
if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &id) {
return Some(format!("{:#?}", rpri));
}
// Otherwise check allocated routes
Self::detail(inner, key).map(|rsd| format!("{:#?}", rsd))
if let Some(rssd) = inner.content.get_detail(id) {
return Some(format!("{:#?}", rssd));
}
None
}
//////////////////////////////////////////////////////////////////////
// Route cache
fn add_to_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, pr_pubkey: PublicKey, safety_route: SafetyRoute)
{
let key = CompiledRouteCacheKey {
sr_pubkey: safety_route.public_key,
pr_pubkey,
};
if let Some(v) = inner.cache.compiled_route_cache.insert(key, safety_route) {
log_rtab!(error "route cache already contained key: sr_pubkey={:?}, pr_pubkey={:?}", v.public_key, pr_pubkey);
}
}
fn lookup_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, sr_pubkey: PublicKey, pr_pubkey: PublicKey) -> Option<SafetyRoute> {
let key = CompiledRouteCacheKey {
sr_pubkey,
pr_pubkey,
};
inner.cache.compiled_route_cache.get(&key).cloned()
}
fn invalidate_compiled_route_cache(&self, inner: &mut RouteSpecStoreInner, dead_key: &PublicKey) {
let mut dead_entries = Vec::new();
for (k, _v) in inner.cache.compiled_route_cache.iter() {
if k.sr_pubkey == *dead_key || k.pr_pubkey == *dead_key {
dead_entries.push(k.clone());
}
}
for d in dead_entries {
inner.cache.compiled_route_cache.remove(&d);
}
}
/// Compiles a safety route to the private route, with caching
/// Returns an Err() if the parameters are wrong
/// Returns Ok(None) if no allocation could happen at this time (not an error)
@ -1441,16 +1236,20 @@ impl RouteSpecStore {
}
/// Import a remote private route for compilation
/// returns a route set id
/// It is safe to import the same route more than once and it will return the same route id
/// Returns a route set id
#[instrument(level = "trace", skip(self, blob), ret, err)]
pub fn import_remote_private_route(&self, blob: Vec<u8>) -> EyreResult<String> {
pub fn import_remote_private_route(&self, blob: Vec<u8>) -> EyreResult<RouteId> {
let cur_ts = get_aligned_timestamp();
// decode the pr blob
let private_routes = RouteSpecStore::blob_to_private_routes(self.unlocked_inner.routing_table.crypto(), blob)?;
let inner = &mut *self.inner.lock();
// make the route id
let id = self.generate_remote_route_id(&private_routes)?;
// validate the private routes
let inner = &mut *self.inner.lock();
for private_route in private_routes {
// ensure private route has first hop
@ -1464,62 +1263,63 @@ impl RouteSpecStore {
}
}
let cur_ts = get_aligned_timestamp();
let id = inner.cache.import_remote_private_route(cur_ts, private_routes);
inner.cache.cache_remote_private_route(cur_ts, id, private_routes);
Ok(id)
}
/// Release a remote private route that is no longer in use
#[instrument(level = "trace", skip(self), ret)]
fn release_remote_private_route(&self, id: &String) -> bool {
pub fn release_remote_private_route(&self, id: RouteId) -> bool {
let inner = &mut *self.inner.lock();
inner.cache.remove_remote_private_route(id)
}
/// Retrieve an imported remote private route by its public key
pub fn get_remote_private_route(&self, id: &String) -> Option<PrivateRoute> {
/// Check if a remote private route id is valid
#[instrument(level = "trace", skip(self), ret)]
pub fn is_valid_remote_private_route(&self, id: &RouteId) -> bool {
let inner = &mut *self.inner.lock();
let cur_ts = get_aligned_timestamp();
Self::with_get_remote_private_route(inner, cur_ts, key, |r| {
r.private_route.as_ref().unwrap().clone()
})
inner.cache.peek_remote_private_route_mut(cur_ts, id).is_some()
}
/// Retrieve an imported remote private route by its public key but don't 'touch' it
pub fn peek_remote_private_route(&self, id: &String) -> Option<PrivateRoute> {
xx fix these
let inner = &mut *self.inner.lock();
let cur_ts = get_aligned_timestamp();
Self::with_peek_remote_private_route(inner, cur_ts, key, |r| {
r.private_route.as_ref().unwrap().clone()
})
}
// /// Retrieve an imported remote private route by its public key
// pub fn get_remote_private_route(&self, id: &String) -> Option<PrivateRoute> {
// let inner = &mut *self.inner.lock();
// let cur_ts = get_aligned_timestamp();
// Self::with_get_remote_private_route(inner, cur_ts, key, |r| {
// r.private_route.as_ref().unwrap().clone()
// })
// }
// /// Retrieve an imported remote private route by its public key but don't 'touch' it
// fn peek_remote_private_route(&self, id: &String) -> Option<PrivateRoute> {
// let inner = &mut *self.inner.lock();
// let cur_ts = get_aligned_timestamp();
// inner.cache.with_peek_remote_private_route(cur_ts, id, f)
// Self::with_peek_remote_private_route(inner, cur_ts, key, |r| {
// r.private_route.as_ref().unwrap().clone()
// })
// }
/// Check to see if this remote (not ours) private route has seen our current node info yet
/// This happens when you communicate with a private route without a safety route
pub fn has_remote_private_route_seen_our_node_info(&self, key: &PublicKey) -> bool {
pub fn has_remote_private_route_seen_our_node_info(&self, id: &RouteId) -> bool {
let our_node_info_ts = {
let rti = &*self.unlocked_inner.routing_table.inner.read();
let Some(ts) = rti.get_own_node_info_ts(RoutingDomain::PublicInternet) else {
// Node info is invalid, skip this
return false;
};
ts
};
let opt_rpr_node_info_ts = {
let inner = &mut *self.inner.lock();
let cur_ts = get_aligned_timestamp();
Self::with_peek_remote_private_route(inner, cur_ts, key, |rpr| {
rpr.last_seen_our_node_info_ts
})
};
let Some(rpr_node_info_ts) = opt_rpr_node_info_ts else {
let inner = &mut *self.inner.lock();
let cur_ts = get_aligned_timestamp();
let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &id) else {
return false;
};
our_node_info_ts == rpr_node_info_ts
rpri.has_seen_our_node_info_ts(our_node_info_ts)
}
/// Mark a remote private route as having seen our current node info
@ -1543,20 +1343,23 @@ impl RouteSpecStore {
};
let inner = &mut *self.inner.lock();
// Check for local route. If this is not a remote private route
// then we just skip the recording. We may be running a test and using
// our own local route as the destination private route.
if let Some(_) = Self::detail_mut(inner, key) {
if let Some(_) = inner.content.get_id_by_key(key) {
return Ok(());
}
if Self::with_get_remote_private_route(inner, cur_ts, key, |rpr| {
rpr.last_seen_our_node_info_ts = our_node_info_ts;
})
.is_none()
{
bail!("private route is missing from store: {}", key);
if let Some(rrid) = inner.cache.get_remote_private_route_id_by_key(key) {
if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &rrid)
{
rpri.set_last_seen_our_node_info_ts(our_node_info_ts);
return Ok(());
}
}
Ok(())
bail!("private route is missing from store: {}", key);
}
/// Get the route statistics for any route we know about, local or remote
@ -1570,15 +1373,20 @@ impl RouteSpecStore {
if self.unlocked_inner.routing_table.matches_own_node_id_key(key) {
return None;
}
// Check for local route
if let Some(rsd) = Self::detail_mut(inner, key) {
return Some(f(&mut rsd.stats));
if let Some(rsid) = inner.content.get_id_by_key(key) {
if let Some(rsd) = inner.content.get_detail_mut(&rsid) {
return Some(f(rsd.get_stats_mut()));
}
}
// Check for remote route
if let Some(res) =
Self::with_peek_remote_private_route(inner, cur_ts, key, |rpr| f(&mut rpr.stats))
{
return Some(res);
if let Some(rrid) = inner.cache.get_remote_private_route_id_by_key(key) {
if let Some(rpri) = inner.cache.peek_remote_private_route_mut(cur_ts, &rrid)
{
return Some(f(rpri.get_stats_mut()));
}
}
None
@ -1613,13 +1421,10 @@ impl RouteSpecStore {
let inner = &mut *self.inner.lock();
// Roll transfers for locally allocated routes
for rssd in inner.content.details.values_mut() {
rssd.stats.roll_transfers(last_ts, cur_ts);
}
inner.content.roll_transfers(last_ts, cur_ts);
// Roll transfers for remote private routes
for (_k, v) in inner.cache.remote_private_route_cache.iter_mut() {
v.stats.roll_transfers(last_ts, cur_ts);
}
inner.cache.roll_transfers(last_ts, cur_ts);
}
/// Convert private route list to binary blob
@ -1681,6 +1486,54 @@ impl RouteSpecStore {
out.push(private_route);
}
// Don't trust the order of the blob
out.sort_by(|a,b| {
a.public_key.cmp(&b.public_key)
});
Ok(out)
}
/// Generate RouteId from typed key set of route public keys
fn generate_allocated_route_id(&self, rssd: &RouteSetSpecDetail) -> EyreResult<RouteId> {
let route_set_keys = rssd.get_route_set_keys();
let crypto = self.unlocked_inner.routing_table.crypto();
let mut idbytes = Vec::with_capacity(PUBLIC_KEY_LENGTH * route_set_keys.len());
let mut best_kind : Option<CryptoKind> = None;
for tk in route_set_keys.iter() {
if best_kind.is_none() || compare_crypto_kind(&tk.kind, best_kind.as_ref().unwrap()) == cmp::Ordering::Less {
best_kind = Some(tk.kind);
}
idbytes.extend_from_slice(&tk.key.bytes);
}
let Some(best_kind) = best_kind else {
bail!("no compatible crypto kinds in route");
};
let vcrypto = crypto.get(best_kind).unwrap();
Ok(RouteId::new(vcrypto.generate_hash(&idbytes).bytes))
}
/// Generate RouteId from set of private routes
fn generate_remote_route_id(&self, private_routes: &[PrivateRoute]) -> EyreResult<RouteId> {
let crypto = self.unlocked_inner.routing_table.crypto();
let mut idbytes = Vec::with_capacity(PUBLIC_KEY_LENGTH * private_routes.len());
let mut best_kind : Option<CryptoKind> = None;
for private_route in private_routes {
if best_kind.is_none() || compare_crypto_kind(&private_route.public_key.kind, best_kind.as_ref().unwrap()) == cmp::Ordering::Less {
best_kind = Some(private_route.public_key.kind);
}
idbytes.extend_from_slice(&private_route.public_key.key.bytes);
}
let Some(best_kind) = best_kind else {
bail!("no compatible crypto kinds in route");
};
let vcrypto = crypto.get(best_kind).unwrap();
Ok(RouteId::new(vcrypto.generate_hash(&idbytes).bytes))
}
}

239
veilid-core/src/routing_table/route_spec_store/route_spec_store_cache.rs
View File

@ -28,15 +28,15 @@ pub struct RouteSpecStoreCache {
/// Route spec hop cache, used to quickly disqualify routes
hop_cache: HashSet<Vec<u8>>,
/// Remote private routes we've imported and statistics
remote_private_route_set_cache: LruCache<RemotePrivateRouteId, RemotePrivateRouteInfo>,
remote_private_route_set_cache: LruCache<RouteId, RemotePrivateRouteInfo>,
/// Remote private routes indexed by public key
remote_private_routes_by_key: HashMap<PublicKey, RemotePrivateRouteId>,
remote_private_routes_by_key: HashMap<PublicKey, RouteId>,
/// Compiled route cache
compiled_route_cache: LruCache<CompiledRouteCacheKey, SafetyRoute>,
/// List of dead allocated routes
dead_routes: Vec<RouteSetSpecId>,
dead_routes: Vec<RouteId>,
/// List of dead remote routes
dead_remote_routes: Vec<RemotePrivateRouteId>,
dead_remote_routes: Vec<RouteId>,
}
impl RouteSpecStoreCache {
@ -66,7 +66,7 @@ impl RouteSpecStoreCache {
}
/// removes an allocated route set from our cache
pub fn remove_from_cache(&mut self, rssd: &RouteSetSpecDetail) -> bool {
pub fn remove_from_cache(&mut self, id: RouteId, rssd: &RouteSetSpecDetail) -> bool {
let cache_key = rssd.make_cache_key();
// Remove from hop cache
@ -100,10 +100,13 @@ impl RouteSpecStoreCache {
panic!("used_end_nodes cache should have contained hop");
}
}
// Invalidate compiled route cache
self.invalidate_compiled_route_cache(pk);
}
// Mark it as dead for the update
self.dead_routes.push(rssd.make_id());
self.dead_routes.push(id);
true
}
@ -122,43 +125,27 @@ impl RouteSpecStoreCache {
})
}
/// generate unique remote private route set id for a remote private route set
fn make_remote_private_route_id(private_routes: &[PrivateRoute]) -> String {
let mut idbytes = [0u8; 16];
for (pk, _) in &rprinfo.private_routes {
for (i, x) in pk.bytes.iter().enumerate() {
idbytes[i % 16] ^= *x;
}
}
let id = format!(
"{:08x}-{:04x}-{:04x}-{:04x}-{:08x}{:04x}",
u32::from_be_bytes(idbytes[0..4].try_into().expect("32 bits")),
u16::from_be_bytes(idbytes[4..6].try_into().expect("16 bits")),
u16::from_be_bytes(idbytes[6..8].try_into().expect("16 bits")),
u16::from_be_bytes(idbytes[8..10].try_into().expect("16 bits")),
u32::from_be_bytes(idbytes[10..14].try_into().expect("32 bits")),
u16::from_be_bytes(idbytes[14..16].try_into().expect("16 bits"))
);
id
}
/// add remote private route to caches
/// returns a remote private route set id
fn add_remote_private_route(
&mut self,
id: RouteId,
rprinfo: RemotePrivateRouteInfo,
) -> RemotePrivateRouteId {
let id = Self::make_remote_private_route_id(rprinfo.get_private_routes());
) -> RouteId {
// also store in id by key table
for (pk, _) in rprinfo.get_private_routes() {
self.remote_private_routes_by_key.insert(*pk, id.clone());
for private_route in rprinfo.get_private_routes() {
self.remote_private_routes_by_key
.insert(private_route.public_key.key, id.clone());
}
self.remote_private_route_set_cache
.insert(id.clone(), rprinfo, |dead_id, dead_rpri| {
.insert(id, rprinfo, |dead_id, dead_rpri| {
// If anything LRUs out, remove from the by-key table
for (dead_pk, _) in dead_rpri.get_private_routes() {
self.remote_private_routes_by_key.remove(&dead_pk).unwrap();
// Follow the same logic as 'remove_remote_private_route' here
for dead_private_route in dead_rpri.get_private_routes() {
self.remote_private_routes_by_key
.remove(&dead_private_route.public_key.key)
.unwrap();
self.invalidate_compiled_route_cache(&dead_private_route.public_key.key);
}
self.dead_remote_routes.push(dead_id);
});
@ -166,33 +153,68 @@ impl RouteSpecStoreCache {
id
}
/// get count of remote private routes in cache
pub fn get_remote_private_route_count(&self) -> usize {
self.remote_private_route_set_cache.len()
}
/// iterate all of the remote private routes we have in the cache
pub fn iter_remote_private_routes(
&self,
) -> hashlink::linked_hash_map::Iter<RouteId, RemotePrivateRouteInfo> {
self.remote_private_route_set_cache.iter()
}
/// remote private route cache accessor
fn get_remote_private_route(
/// will LRU entries and may expire entries and not return them if they are stale
pub fn get_remote_private_route(
&mut self,
id: &RemotePrivateRouteId,
cur_ts: Timestamp,
id: &RouteId,
) -> Option<&RemotePrivateRouteInfo> {
self.remote_private_route_set_cache.get(id)
if let Some(rpri) = self.remote_private_route_set_cache.get(id) {
if !rpri.did_expire(cur_ts) {
rpri.touch(cur_ts);
return Some(rpri);
}
}
None
}
/// mutable remote private route cache accessor
fn get_remote_private_route_mut(
/// will LRU entries and may expire entries and not return them if they are stale
pub fn get_remote_private_route_mut(
&mut self,
id: &RemotePrivateRouteId,
cur_ts: Timestamp,
id: &RouteId,
) -> Option<&mut RemotePrivateRouteInfo> {
self.remote_private_route_set_cache.get_mut(id)
if let Some(rpri) = self.remote_private_route_set_cache.get_mut(id) {
if !rpri.did_expire(cur_ts) {
rpri.touch(cur_ts);
return Some(rpri);
}
}
None
}
/// mutable remote private route cache accessor without lru action
fn peek_remote_private_route_mut(
/// will not LRU entries but may expire entries and not return them if they are stale
pub fn peek_remote_private_route_mut(
&mut self,
id: &RemotePrivateRouteId,
cur_ts: Timestamp,
id: &RouteId,
) -> Option<&mut RemotePrivateRouteInfo> {
self.remote_private_route_set_cache.peek_mut(id)
if let Some(rpri) = self.remote_private_route_set_cache.peek_mut(id) {
if !rpri.did_expire(cur_ts) {
rpri.touch(cur_ts);
return Some(rpri);
}
}
None
}
/// look up a remote private route id by one of the route public keys
pub fn get_remote_private_route_id_by_key(
&self,
key: &PublicKey,
) -> Option<RemotePrivateRouteId> {
pub fn get_remote_private_route_id_by_key(&self, key: &PublicKey) -> Option<RouteId> {
self.remote_private_routes_by_key.get(key).cloned()
}
@ -200,14 +222,14 @@ impl RouteSpecStoreCache {
/// may LRU and/or expire other cache entries to make room for the new one
/// or update an existing entry with the same private route set
/// returns the route set id
pub fn import_remote_private_route(
pub fn cache_remote_private_route(
&mut self,
cur_ts: Timestamp,
id: RouteId,
private_routes: Vec<PrivateRoute>,
) -> RemotePrivateRouteId {
) {
// get id for this route set
let id = RouteSpecStoreCache::make_remote_private_route_id(&private_routes);
let rpri = if let Some(rpri) = self.get_remote_private_route_mut(&id) {
if let Some(rpri) = self.get_remote_private_route_mut(cur_ts, &id) {
if rpri.did_expire(cur_ts) {
// Start fresh if this had expired
rpri.unexpire(cur_ts);
@ -223,88 +245,95 @@ impl RouteSpecStoreCache {
last_touched_ts: cur_ts,
stats: RouteStats::new(cur_ts),
};
let new_id = self.add_remote_private_route(rpri);
assert_eq!(id, new_id);
if self.get_remote_private_route_mut(&id).is_none() {
bail!("remote private route should exist");
self.add_remote_private_route(id, rpri);
if self.peek_remote_private_route_mut(cur_ts, &id).is_none() {
panic!("remote private route should exist");
};
};
id
}
/// remove a remote private route from the cache
pub fn remove_remote_private_route(&mut self, id: &RemotePrivateRouteId) -> bool {
let Some(rprinfo) = self.remote_private_route_set_cache.remove(id) else {
pub fn remove_remote_private_route(&mut self, id: RouteId) -> bool {
let Some(rprinfo) = self.remote_private_route_set_cache.remove(&id) else {
return false;
};
for (pk, _) in rprinfo.get_private_routes() {
self.remote_private_routes_by_key.remove(&pk).unwrap();
for private_route in rprinfo.get_private_routes() {
self.remote_private_routes_by_key
.remove(&private_route.public_key.key)
.unwrap();
self.invalidate_compiled_route_cache(&private_route.public_key.key);
}
self.dead_remote_routes.push(id.clone());
self.dead_remote_routes.push(id);
true
}
/// get an existing remote private route cache entry
/// will LRU entries and may expire entries and not return them if they are stale
/// calls a callback with the remote private route info if returned
pub fn with_get_remote_private_route<F, R>(
&mut self,
cur_ts: Timestamp,
id: &RemotePrivateRouteId,
f: F,
) -> Option<R>
where
F: FnOnce(&mut RemotePrivateRouteInfo) -> R,
{
if let Some(rpri) = self.get_remote_private_route_mut(&id) {
if !rpri.did_expire(cur_ts) {
rpri.touch(cur_ts);
return Some(f(rpri));
}
/// Stores a compiled 'safety + private' route so we don't have to compile it again later
pub fn add_to_compiled_route_cache(&self, pr_pubkey: PublicKey, safety_route: SafetyRoute) {
let key = CompiledRouteCacheKey {
sr_pubkey: safety_route.public_key.key,
pr_pubkey,
};
if let Some(v) = self
.compiled_route_cache
.insert(key, safety_route, |_k, _v| {
// Do nothing on LRU evict
})
{
log_rtab!(error "route cache already contained key: sr_pubkey={:?}, pr_pubkey={:?}", v.public_key, pr_pubkey);
}
self.remove_remote_private_route(&id);
None
}
// peek a remote private route cache entry
// will not LRU entries but may expire entries and not return them if they are stale
/// calls a callback with the remote private route info if returned
pub fn with_peek_remote_private_route<F, R>(
&mut self,
cur_ts: Timestamp,
id: &RemotePrivateRouteId,
f: F,
) -> Option<R>
where
F: FnOnce(&mut RemotePrivateRouteInfo) -> R,
{
if let Some(rpri) = self.peek_remote_private_route_mut(&id) {
if !rpri.did_expire(cur_ts) {
rpri.touch(cur_ts);
return Some(f(rpri));
/// Looks up an existing compiled route from the safety and private route components
pub fn lookup_compiled_route_cache(
&self,
sr_pubkey: PublicKey,
pr_pubkey: PublicKey,
) -> Option<SafetyRoute> {
let key = CompiledRouteCacheKey {
sr_pubkey,
pr_pubkey,
};
self.compiled_route_cache.get(&key).cloned()
}
/// When routes are dropped, they should be removed from the compiled route cache
fn invalidate_compiled_route_cache(&self, dead_key: &PublicKey) {
let mut dead_entries = Vec::new();
for (k, _v) in self.compiled_route_cache.iter() {
if k.sr_pubkey == *dead_key || k.pr_pubkey == *dead_key {
dead_entries.push(k.clone());
}
}
self.remove_remote_private_route(&id);
None
for d in dead_entries {
self.compiled_route_cache.remove(&d);
}
}
/// Take the dead local and remote routes so we can update clients
pub fn take_dead_routes(&mut self) -> (Vec<RouteSetSpecId>, Vec<RemotePrivateRouteId>) {
pub fn take_dead_routes(&mut self) -> Option<(Vec<RouteId>, Vec<RouteId>)> {
if self.dead_routes.is_empty() && self.dead_remote_routes.is_empty() {
// Nothing to do
return;
return None;
}
let dead_routes = core::mem::take(&mut self.dead_routes);
let dead_remote_routes = core::mem::take(&mut self.dead_remote_routes);
(dead_routes, dead_remote_routes)
Some((dead_routes, dead_remote_routes))
}
/// Clean up imported remote routes
/// Resets statistics for when our node info changes
pub fn reset_details(&mut self) {
for (_k, v) in self.remote_private_route_cache {
// Restart stats for routes so we test the route again
v.stats.reset();
pub fn reset_remote_private_routes(&mut self) {
// Restart stats for routes so we test the route again
for (_k, v) in self.remote_private_route_set_cache {
v.get_stats_mut().reset();
}
}
/// Roll transfer statistics
pub fn roll_transfers(&mut self, last_ts: Timestamp, cur_ts: Timestamp) {
for (_k, v) in self.remote_private_route_set_cache {
v.get_stats_mut().roll_transfers(last_ts, cur_ts);
}
}
}

142
veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs
View File

@ -5,15 +5,116 @@ use super::*;
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
pub struct RouteSpecStoreContent {
/// All of the route sets we have allocated so far indexed by key
id_by_key: HashMap<PublicKey, RouteSetSpecId>,
id_by_key: HashMap<PublicKey, RouteId>,
/// All of the route sets we have allocated so far
details: HashMap<RouteSetSpecId, RouteSetSpecDetail>,
details: HashMap<RouteId, RouteSetSpecDetail>,
}
impl RouteSpecStoreContent {
pub fn add_detail(&mut self, detail: RouteSetSpecDetail) -> RouteSetSpecId {
// generate unique key string
let id = detail.make_id();
pub async fn load(routing_table: RoutingTable) -> EyreResult<RouteSpecStoreContent> {
// Deserialize what we can
let table_store = routing_table.network_manager().table_store();
let rsstdb = table_store.open("RouteSpecStore", 1).await?;
let mut content: RouteSpecStoreContent =
rsstdb.load_rkyv(0, b"content")?.unwrap_or_default();
// Look up all route hop noderefs since we can't serialize those
let mut dead_ids = Vec::new();
for (rsid, rssd) in content.details.iter_mut() {
// Get best route since they all should resolve
let Some(pk) = rssd.get_best_route_set_key() else {
dead_ids.push(rsid.clone());
continue;
};
let Some(rsd) = rssd.get_route_by_key(&pk) else {
dead_ids.push(rsid.clone());
continue;
};
// Go through best route and resolve noderefs
let mut hop_node_refs = Vec::with_capacity(rsd.hops.len());
for h in &rsd.hops {
let Some(nr) = routing_table.lookup_node_ref(TypedKey::new(rsd.crypto_kind, *h)) else {
dead_ids.push(rsid.clone());
break;
};
hop_node_refs.push(nr);
}
// Apply noderefs
rssd.set_hop_node_refs(hop_node_refs);
}
for id in dead_ids {
log_rtab!(debug "no entry, killing off private route: {}", id);
content.remove_detail(&id);
}
// Load secrets from pstore
let pstore = routing_table.network_manager().protected_store();
let secret_key_map: HashMap<PublicKey, SecretKey> = pstore
.load_user_secret_rkyv("RouteSpecStore")
.await?
.unwrap_or_default();
// Ensure we got secret keys for all the public keys
let mut got_secret_key_ids = HashSet::new();
for (rsid, rssd) in content.details.iter_mut() {
let mut found_all = true;
for (pk, rsd) in rssd.iter_route_set_mut() {
if let Some(sk) = secret_key_map.get(pk) {
rsd.secret_key = *sk;
} else {
found_all = false;
break;
}
}
if found_all {
got_secret_key_ids.insert(rsid.clone());
}
}
// If we missed any, nuke those route ids
let dead_ids: Vec<RouteId> = content
.details
.keys()
.filter_map(|id| {
if !got_secret_key_ids.contains(id) {
Some(*id)
} else {
None
}
})
.collect();
for id in dead_ids {
log_rtab!(debug "missing secret key, killing off private route: {}", id);
content.remove_detail(&id);
}
Ok(content)
}
pub async fn save(&self, routing_table: RoutingTable) -> EyreResult<()> {
// Save all the fields we care about to the frozen blob in table storage
// This skips #[with(Skip)] saving the secret keys, we save them in the protected store instead
let table_store = routing_table.network_manager().table_store();
let rsstdb = table_store.open("RouteSpecStore", 1).await?;
rsstdb.store_rkyv(0, b"content", self).await?;
// // Keep secrets in protected store as well
let pstore = routing_table.network_manager().protected_store();
let mut out: HashMap<PublicKey, SecretKey> = HashMap::new();
for (rsid, rssd) in self.details.iter() {
for (pk, rsd) in rssd.iter_route_set() {
out.insert(*pk, rsd.secret_key);
}
}
let _ = pstore.save_user_secret_rkyv("RouteSpecStore", &out).await?; // ignore if this previously existed or not
Ok(())
}
pub fn add_detail(&mut self, id: RouteId, detail: RouteSetSpecDetail) {
assert!(!self.details.contains_key(&id));
// also store in id by key table
@ -21,38 +122,32 @@ impl RouteSpecStoreContent {
self.id_by_key.insert(*pk, id.clone());
}
self.details.insert(id.clone(), detail);
id
}
pub fn remove_detail(&mut self, id: &RouteSetSpecId) -> Option<RouteSetSpecDetail> {
pub fn remove_detail(&mut self, id: &RouteId) -> Option<RouteSetSpecDetail> {
let detail = self.details.remove(id)?;
for (pk, _) in detail.iter_route_set() {
self.id_by_key.remove(&pk).unwrap();
}
Some(detail)
}
pub fn get_detail(&self, id: &RouteSetSpecId) -> Option<&RouteSetSpecDetail> {
pub fn get_detail_count(&self) -> usize {
self.details.len()
}
pub fn get_detail(&self, id: &RouteId) -> Option<&RouteSetSpecDetail> {
self.details.get(id)
}
pub fn get_detail_mut(&mut self, id: &RouteSetSpecId) -> Option<&mut RouteSetSpecDetail> {
pub fn get_detail_mut(&mut self, id: &RouteId) -> Option<&mut RouteSetSpecDetail> {
self.details.get_mut(id)
}
pub fn get_id_by_key(&self, key: &PublicKey) -> Option<RouteSetSpecId> {
pub fn get_id_by_key(&self, key: &PublicKey) -> Option<RouteId> {
self.id_by_key.get(key).cloned()
}
pub fn iter_ids(&self) -> std::collections::hash_map::Keys<RouteSetSpecId, RouteSetSpecDetail> {
pub fn iter_ids(&self) -> std::collections::hash_map::Keys<RouteId, RouteSetSpecDetail> {
self.details.keys()
}
pub fn iter_details(
&self,
) -> std::collections::hash_map::Iter<RouteSetSpecId, RouteSetSpecDetail> {
pub fn iter_details(&self) -> std::collections::hash_map::Iter<RouteId, RouteSetSpecDetail> {
self.details.iter()
}
pub fn iter_details_mut(
&mut self,
) -> std::collections::hash_map::IterMut<RouteSetSpecId, RouteSetSpecDetail> {
self.details.iter_mut()
}
/// Clean up local allocated routes
/// Resets publication status and statistics for when our node info changes
@ -65,4 +160,11 @@ impl RouteSpecStoreContent {
v.get_stats_mut().reset();
}
}
/// Roll transfer statistics
pub fn roll_transfers(&mut self, last_ts: Timestamp, cur_ts: Timestamp) {
for rssd in self.details.values_mut() {
rssd.get_stats_mut().roll_transfers(last_ts, cur_ts);
}
}
}

80
veilid-core/src/routing_table/tasks/private_route_management.rs
View File

@ -8,7 +8,7 @@ const BACKGROUND_SAFETY_ROUTE_COUNT: usize = 2;
impl RoutingTable {
/// Fastest routes sort
fn route_sort_latency_fn(a: &(TypedKey, u64), b: &(TypedKey, u64)) -> cmp::Ordering {
fn route_sort_latency_fn(a: &(RouteId, u64), b: &(RouteId, u64)) -> cmp::Ordering {
let mut al = a.1;
let mut bl = b.1;
// Treat zero latency as uncalculated
@ -35,14 +35,14 @@ impl RoutingTable {
///
/// If a route doesn't 'need_testing', then we neither test nor drop it
#[instrument(level = "trace", skip(self))]
fn get_allocated_routes_to_test(&self, cur_ts: Timestamp) -> Vec<TypedKey> {
fn get_allocated_routes_to_test(&self, cur_ts: Timestamp) -> Vec<RouteId> {
let default_route_hop_count =
self.with_config(|c| c.network.rpc.default_route_hop_count as usize);
let rss = self.route_spec_store();
let mut must_test_routes = Vec::<TypedKey>::new();
let mut unpublished_routes = Vec::<(TypedKey, u64)>::new();
let mut expired_routes = Vec::<TypedKey>::new();
let mut must_test_routes = Vec::<RouteId>::new();
let mut unpublished_routes = Vec::<(RouteId, u64)>::new();
let mut expired_routes = Vec::<RouteId>::new();
rss.list_allocated_routes(|k, v| {
let stats = v.get_stats();
// Ignore nodes that don't need testing
@ -81,7 +81,7 @@ impl RoutingTable {
}
// Process dead routes
for r in &expired_routes {
for r in expired_routes {
log_rtab!(debug "Expired route: {}", r);
rss.release_route(r);
}
@ -95,7 +95,7 @@ impl RoutingTable {
async fn test_route_set(
&self,
stop_token: StopToken,
routes_needing_testing: Vec<TypedKey>,
routes_needing_testing: Vec<RouteId>,
) -> EyreResult<()> {
if routes_needing_testing.is_empty() {
return Ok(());
@ -107,43 +107,45 @@ impl RoutingTable {
#[derive(Default, Debug)]
struct TestRouteContext {
failed: bool,
dead_routes: Vec<TypedKey>,
dead_routes: Vec<RouteId>,
}
let mut unord = FuturesUnordered::new();
let ctx = Arc::new(Mutex::new(TestRouteContext::default()));
for r in routes_needing_testing {
let rss = rss.clone();
let ctx = ctx.clone();
unord.push(
async move {
let success = match rss.test_route(&r).await {
Ok(v) => v,
Err(e) => {
log_rtab!(error "Test route failed: {}", e);
ctx.lock().failed = true;
{
let mut unord = FuturesUnordered::new();
for r in routes_needing_testing {
let rss = rss.clone();
let ctx = ctx.clone();
unord.push(
async move {
let success = match rss.test_route(r).await {
Ok(v) => v,
Err(e) => {
log_rtab!(error "Test route failed: {}", e);
ctx.lock().failed = true;
return;
}
};
if success {
// Route is okay, leave it alone
return;
}
};
if success {
// Route is okay, leave it alone
return;
// Route test failed
ctx.lock().dead_routes.push(r);
}
// Route test failed
ctx.lock().dead_routes.push(r);
}
.instrument(Span::current())
.boxed(),
);
.instrument(Span::current())
.boxed(),
);
}
// Wait for test_route futures to complete in parallel
while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {}
}
// Wait for test_route futures to complete in parallel
while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {}
// Process failed routes
let ctx = &mut *ctx.lock();
for r in &ctx.dead_routes {
log_rtab!(debug "Dead route failed to test: {}", &r);
let ctx = Arc::try_unwrap(ctx).unwrap().into_inner();
for r in ctx.dead_routes {
log_rtab!(debug "Dead route failed to test: {}", r);
rss.release_route(r);
}
@ -176,13 +178,16 @@ impl RoutingTable {
.await?;
}
// Ensure we have a minimum of N allocated local, unpublished routes with the default number of hops
// Ensure we have a minimum of N allocated local, unpublished routes with the default number of hops and all our supported crypto kinds
let default_route_hop_count =
self.with_config(|c| c.network.rpc.default_route_hop_count as usize);
let mut local_unpublished_route_count = 0usize;
let rss = self.route_spec_store();
rss.list_allocated_routes(|_k, v| {
if !v.is_published() && v.hop_count() == default_route_hop_count {
if !v.is_published()
&& v.hop_count() == default_route_hop_count
&& v.get_route_set_keys().kinds() == VALID_CRYPTO_KINDS
{
local_unpublished_route_count += 1;
}
Option::<()>::None
@ -196,6 +201,7 @@ impl RoutingTable {
// Parameters here must be the default safety route spec
// These will be used by test_remote_route as well
if let Some(k) = rss.allocate_route(
&VALID_CRYPTO_KINDS,
Stability::default(),
Sequencing::default(),
default_route_hop_count,

18
veilid-core/src/rpc_processor/destination.rs
View File

@ -22,7 +22,7 @@ pub enum Destination {
/// Send to private route (privateroute)
PrivateRoute {
/// A private route set id to send to
private_route: String,
private_route_id: RouteId,
/// Require safety route or not
safety_selection: SafetySelection,
},
@ -44,9 +44,9 @@ impl Destination {
safety_selection: SafetySelection::Unsafe(sequencing),
}
}
pub fn private_route(private_route: PrivateRoute, safety_selection: SafetySelection) -> Self {
pub fn private_route(private_route_id: RouteId, safety_selection: SafetySelection) -> Self {
Self::PrivateRoute {
private_route,
private_route_id,
safety_selection,
}
}
@ -70,10 +70,10 @@ impl Destination {
safety_selection,
},
Destination::PrivateRoute {
private_route,
private_route_id,
safety_selection: _,
} => Self::PrivateRoute {
private_route,
private_route_id,
safety_selection,
},
}
@ -91,7 +91,7 @@ impl Destination {
safety_selection,
} => safety_selection,
Destination::PrivateRoute {
private_route: _,
private_route_id: _,
safety_selection,
} => safety_selection,
}
@ -127,7 +127,7 @@ impl fmt::Display for Destination {
write!(f, "{}@{}{}", target, relay, sr)
}
Destination::PrivateRoute {
private_route,
private_route_id,
safety_selection,
} => {
let sr = if matches!(safety_selection, SafetySelection::Safe(_)) {
@ -136,7 +136,7 @@ impl fmt::Display for Destination {
""
};
write!(f, "{}{}", private_route, sr)
write!(f, "{}{}", private_route_id, sr)
}
}
}
@ -207,7 +207,7 @@ impl RPCProcessor {
}
},
Destination::PrivateRoute {
private_route,
private_route_id,
safety_selection,
} => {
let Some(avoid_node_id) = private_route.first_hop_node_id() else {

4
veilid-core/src/rpc_processor/mod.rs
View File

@ -674,7 +674,7 @@ impl RPCProcessor {
};
}
Destination::PrivateRoute {
private_route,
private_route_id,
safety_selection,
} => {
// Send to private route
@ -726,7 +726,7 @@ impl RPCProcessor {
}
}
Destination::PrivateRoute {
private_route: _,
private_route_id: _,
safety_selection: _,
} => {
return Ok(SenderPeerInfo::default());

2
veilid-core/src/rpc_processor/rpc_find_node.rs
View File

@ -17,7 +17,7 @@ impl RPCProcessor {
if matches!(
dest,
Destination::PrivateRoute {
private_route: _,
private_route_id: _,
safety_selection: _
}
) {

2
veilid-core/src/rpc_processor/rpc_signal.rs
View File

@ -13,7 +13,7 @@ impl RPCProcessor {
if matches!(
dest,
Destination::PrivateRoute {
private_route: _,
private_route_id: _,
safety_selection: _
}
) {

4
veilid-core/src/rpc_processor/rpc_status.rs
View File

@ -53,7 +53,7 @@ impl RPCProcessor {
(Some(target.clone()), routing_domain)
}
Destination::PrivateRoute {
private_route: _,
private_route_id: _,
safety_selection: _,
} => (None, RoutingDomain::PublicInternet),
};
@ -169,7 +169,7 @@ impl RPCProcessor {
safety_selection: _,
}
| Destination::PrivateRoute {
private_route: _,
private_route_id: _,
safety_selection: _,
} => {
// sender info is irrelevant over relays and routes

9
veilid-core/src/veilid_api/routing_context.rs
View File

@ -4,8 +4,8 @@ use super::*;
#[derive(Clone, Debug)]
pub enum Target {
NodeId(PublicKey), // Node by any of its public keys
PrivateRoute(String), // Private route by its route set id
NodeId(PublicKey), // Node by any of its public keys
PrivateRoute(RouteId), // Remote private route by its id
}
pub struct RoutingContextInner {}
@ -121,14 +121,13 @@ impl RoutingContext {
Target::PrivateRoute(rsid) => {
// Get remote private route
let rss = self.api.routing_table()?.route_spec_store();
let Some(private_route) = rss
.get_remote_private_route(&rsid)
if !rss.is_valid_remote_private_route(&rsid)
else {
apibail_key_not_found!(pr);
};
Ok(rpc_processor::Destination::PrivateRoute {
private_route: rsid,
private_route_id: rsid,
safety_selection: self.unlocked_inner.safety_selection,
})
}

6
veilid-core/src/veilid_api/types.rs
View File

@ -279,8 +279,8 @@ pub struct VeilidStateNetwork {
)]
#[archive_attr(repr(C), derive(CheckBytes))]
pub struct VeilidStateRoute {
pub dead_routes: Vec<PublicKey>,
pub dead_remote_routes: Vec<PublicKey>,
pub dead_routes: Vec<RouteId>,
pub dead_remote_routes: Vec<RouteId>,
}
#[derive(
@ -513,7 +513,7 @@ impl SafetySelection {
#[archive_attr(repr(C), derive(CheckBytes))]
pub struct SafetySpec {
/// preferred safety route set id if it still exists
pub preferred_route: Option<String>,
pub preferred_route: Option<RouteId>,
/// must be greater than 0
pub hop_count: usize,
/// prefer reliability over speed