From baa171494332d63d32d76e4e7c8134ff77b2a670 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 25 Sep 2022 18:04:53 -0400 Subject: [PATCH 01/67] app call/message and private routing checkpoint --- veilid-cli/src/client_api_connection.rs | 4 +- veilid-cli/src/command_processor.rs | 2 +- veilid-core/proto/veilid.capnp | 30 +- veilid-core/src/api_tracing_layer.rs | 2 +- veilid-core/src/network_manager/mod.rs | 10 +- veilid-core/src/routing_table/find_nodes.rs | 10 +- veilid-core/src/routing_table/node_ref.rs | 23 +- .../rpc_processor/coders/operations/answer.rs | 11 +- .../rpc_processor/coders/operations/mod.rs | 4 + .../coders/operations/operation_app_call.rs | 44 ++ .../operations/operation_app_message.rs | 23 + .../coders/operations/question.rs | 8 + .../coders/operations/statement.rs | 10 + .../coders/private_safety_route.rs | 74 ---- veilid-core/src/rpc_processor/mod.rs | 319 +++++++------- .../src/rpc_processor/operation_waiter.rs | 136 ++++++ .../src/rpc_processor/private_route.rs | 2 +- veilid-core/src/rpc_processor/rpc_app_call.rs | 99 +++++ .../src/rpc_processor/rpc_app_message.rs | 42 ++ veilid-core/src/rpc_processor/rpc_error.rs | 17 +- veilid-core/src/rpc_processor/rpc_signal.rs | 2 +- .../rpc_processor/rpc_validate_dial_info.rs | 8 +- veilid-core/src/veilid_api/mod.rs | 403 +++++++----------- veilid-core/src/veilid_api/privacy.rs | 124 ++++++ veilid-core/src/veilid_api/routing_context.rs | 213 +++++++++ veilid-core/src/xx/eventual_value.rs | 8 + veilid-core/src/xx/timeout_or.rs | 9 + veilid-flutter/example/lib/main.dart | 26 +- veilid-flutter/lib/veilid.dart | 14 +- 29 files changed, 1139 insertions(+), 538 deletions(-) create mode 100644 veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs create mode 100644 veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs create mode 100644 veilid-core/src/rpc_processor/operation_waiter.rs create mode 100644 veilid-core/src/rpc_processor/rpc_app_call.rs create mode 100644 veilid-core/src/rpc_processor/rpc_app_message.rs create mode 100644 veilid-core/src/veilid_api/privacy.rs create mode 100644 veilid-core/src/veilid_api/routing_context.rs diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index 2ae3de6c..22828531 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -196,9 +196,9 @@ impl ClientApiConnection { // Wait until rpc system completion or disconnect was requested let res = rpc_jh.await; - #[cfg(feature="rt-tokio")] + #[cfg(feature = "rt-tokio")] let res = res.map_err(|e| format!("join error: {}", e))?; - res.map_err(|e| format!("client RPC system error: {}", e)) + res.map_err(|e| format!("client RPC system error: {}", e)) } async fn handle_connection(&mut self) -> Result<(), String> { diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index 67dcaad8..1e6d4ad6 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -331,7 +331,7 @@ change_log_level - change the log level for a tracing layer ); } - pub fn update_log(&mut self, log: veilid_core::VeilidStateLog) { + pub fn update_log(&mut self, log: veilid_core::VeilidLog) { self.inner().ui.add_node_event(format!( "{}: {}{}", log.log_level, diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index a0f791a4..558686b2 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -294,6 +294,19 @@ struct OperationNodeInfoUpdate { signedNodeInfo @0 :SignedNodeInfo; # Our signed node info } + +struct OperationAppCallQ { + message @0 :Data; # Opaque request to application +} + +struct OperationAppCallA { + message @0 :Data; # Opaque response from application +} + +struct OperationAppMessage { + message @0 :Data; # Opaque message to application +} + struct OperationGetValueQ { key @0 :ValueKey; # key for value to get } @@ -445,11 +458,12 @@ struct Question { watchValueQ @6 :OperationWatchValueQ; supplyBlockQ @7 :OperationSupplyBlockQ; findBlockQ @8 :OperationFindBlockQ; + appCallQ @9 :OperationAppCallQ; # Tunnel operations - startTunnelQ @9 :OperationStartTunnelQ; - completeTunnelQ @10 :OperationCompleteTunnelQ; - cancelTunnelQ @11 :OperationCancelTunnelQ; + startTunnelQ @10 :OperationStartTunnelQ; + completeTunnelQ @11 :OperationCompleteTunnelQ; + cancelTunnelQ @12 :OperationCancelTunnelQ; } } @@ -465,6 +479,7 @@ struct Statement { valueChanged @3 :OperationValueChanged; signal @4 :OperationSignal; returnReceipt @5 :OperationReturnReceipt; + appMessage @6 :OperationAppMessage; } } @@ -480,12 +495,13 @@ struct Answer { setValueA @3 :OperationSetValueA; watchValueA @4 :OperationWatchValueA; supplyBlockA @5 :OperationSupplyBlockA; - findBlockA @6 :OperationFindBlockA; + findBlockA @6 :OperationFindBlockA; + appCallA @7 :OperationAppCallA; # Tunnel operations - startTunnelA @7 :OperationStartTunnelA; - completeTunnelA @8 :OperationCompleteTunnelA; - cancelTunnelA @9 :OperationCancelTunnelA; + startTunnelA @8 :OperationStartTunnelA; + completeTunnelA @9 :OperationCompleteTunnelA; + cancelTunnelA @10 :OperationCancelTunnelA; } } diff --git a/veilid-core/src/api_tracing_layer.rs b/veilid-core/src/api_tracing_layer.rs index 3106e2f9..dac00241 100644 --- a/veilid-core/src/api_tracing_layer.rs +++ b/veilid-core/src/api_tracing_layer.rs @@ -103,7 +103,7 @@ impl registry::LookupSpan<'a>> Layer for ApiTracingLa None }; - (inner.update_callback)(VeilidUpdate::Log(VeilidStateLog { + (inner.update_callback)(VeilidUpdate::Log(VeilidLog { log_level, message, backtrace, diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index f9090069..3fe83c63 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -358,7 +358,15 @@ impl NetworkManager { self.routing_table(), connection_manager.clone(), ); - let rpc_processor = RPCProcessor::new(self.clone()); + let rpc_processor = RPCProcessor::new( + self.clone(), + self.unlocked_inner + .update_callback + .read() + .as_ref() + .unwrap() + .clone(), + ); let receipt_manager = ReceiptManager::new(self.clone()); *self.unlocked_inner.components.write() = Some(NetworkComponents { net: net.clone(), diff --git a/veilid-core/src/routing_table/find_nodes.rs b/veilid-core/src/routing_table/find_nodes.rs index 581c98a4..ca9841a2 100644 --- a/veilid-core/src/routing_table/find_nodes.rs +++ b/veilid-core/src/routing_table/find_nodes.rs @@ -22,7 +22,9 @@ impl RoutingTable { move |e| { if let Some(ni) = e.node_info(routing_domain) { if ni - .first_filtered_dial_info_detail(|did| did.matches_filter(&dial_info_filter)) + .first_filtered_dial_info_detail(DialInfoDetail::NO_SORT, |did| { + did.matches_filter(&dial_info_filter) + }) .is_some() { return true; @@ -436,8 +438,10 @@ impl RoutingTable { let can_serve_as_relay = e .node_info(RoutingDomain::PublicInternet) .map(|n| { - let dids = - n.all_filtered_dial_info_details(|did| did.matches_filter(&outbound_dif)); + let dids = n.all_filtered_dial_info_details( + Some(DialInfoDetail::reliable_sort), // By default, choose reliable protocol for relay + |did| did.matches_filter(&outbound_dif), + ); for did in &dids { let pt = did.dial_info.protocol_type(); let at = did.dial_info.address_type(); diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 26db4a8b..0b64de14 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -71,6 +71,7 @@ pub struct NodeRef { node_id: DHTKey, entry: Arc, filter: Option, + reliable: bool, #[cfg(feature = "tracking")] track_id: usize, } @@ -89,6 +90,7 @@ impl NodeRef { node_id, entry, filter, + reliable: true, #[cfg(feature = "tracking")] track_id: entry.track(), } @@ -126,6 +128,10 @@ impl NodeRef { self.filter = filter } + pub fn set_reliable(&mut self) { + self.reliable = true; + } + pub fn merge_filter(&mut self, filter: NodeRefFilter) { if let Some(self_filter) = self.filter.take() { self.filter = Some(self_filter.filtered(&filter)); @@ -267,11 +273,17 @@ impl NodeRef { let routing_domain_set = self.routing_domain_set(); let dial_info_filter = self.dial_info_filter(); + let sort = if self.reliable { + Some(DialInfoDetail::reliable_sort) + } else { + None + }; + self.operate(|_rt, e| { for routing_domain in routing_domain_set { if let Some(ni) = e.node_info(routing_domain) { let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); - if let Some(did) = ni.first_filtered_dial_info_detail(filter) { + if let Some(did) = ni.first_filtered_dial_info_detail(sort, filter) { return Some(did); } } @@ -284,12 +296,18 @@ impl NodeRef { let routing_domain_set = self.routing_domain_set(); let dial_info_filter = self.dial_info_filter(); + let sort = if self.reliable { + Some(DialInfoDetail::reliable_sort) + } else { + None + }; + let mut out = Vec::new(); self.operate(|_rt, e| { for routing_domain in routing_domain_set { if let Some(ni) = e.node_info(routing_domain) { let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); - if let Some(did) = ni.first_filtered_dial_info_detail(filter) { + if let Some(did) = ni.first_filtered_dial_info_detail(sort, filter) { out.push(did); } } @@ -390,6 +408,7 @@ impl Clone for NodeRef { node_id: self.node_id, entry: self.entry.clone(), filter: self.filter.clone(), + reliable: self.reliable, #[cfg(feature = "tracking")] track_id: e.track(), } diff --git a/veilid-core/src/rpc_processor/coders/operations/answer.rs b/veilid-core/src/rpc_processor/coders/operations/answer.rs index 906132f2..e6d801a3 100644 --- a/veilid-core/src/rpc_processor/coders/operations/answer.rs +++ b/veilid-core/src/rpc_processor/coders/operations/answer.rs @@ -11,9 +11,6 @@ impl RPCAnswer { pub fn new(detail: RPCAnswerDetail) -> Self { Self { detail } } - // pub fn detail(&self) -> &RPCAnswerDetail { - // &self.detail - // } pub fn into_detail(self) -> RPCAnswerDetail { self.detail } @@ -35,6 +32,7 @@ impl RPCAnswer { pub enum RPCAnswerDetail { StatusA(RPCOperationStatusA), FindNodeA(RPCOperationFindNodeA), + AppCallA(RPCOperationAppCallA), GetValueA(RPCOperationGetValueA), SetValueA(RPCOperationSetValueA), WatchValueA(RPCOperationWatchValueA), @@ -50,6 +48,7 @@ impl RPCAnswerDetail { match self { RPCAnswerDetail::StatusA(_) => "StatusA", RPCAnswerDetail::FindNodeA(_) => "FindNodeA", + RPCAnswerDetail::AppCallA(_) => "AppCallA", RPCAnswerDetail::GetValueA(_) => "GetValueA", RPCAnswerDetail::SetValueA(_) => "SetValueA", RPCAnswerDetail::WatchValueA(_) => "WatchValueA", @@ -76,6 +75,11 @@ impl RPCAnswerDetail { let out = RPCOperationFindNodeA::decode(&op_reader)?; RPCAnswerDetail::FindNodeA(out) } + veilid_capnp::answer::detail::AppCallA(r) => { + let op_reader = r.map_err(RPCError::protocol)?; + let out = RPCOperationAppCallA::decode(&op_reader)?; + RPCAnswerDetail::AppCallA(out) + } veilid_capnp::answer::detail::GetValueA(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationGetValueA::decode(&op_reader)?; @@ -126,6 +130,7 @@ impl RPCAnswerDetail { match self { RPCAnswerDetail::StatusA(d) => d.encode(&mut builder.reborrow().init_status_a()), RPCAnswerDetail::FindNodeA(d) => d.encode(&mut builder.reborrow().init_find_node_a()), + RPCAnswerDetail::AppCallA(d) => d.encode(&mut builder.reborrow().init_app_call_a()), RPCAnswerDetail::GetValueA(d) => d.encode(&mut builder.reborrow().init_get_value_a()), RPCAnswerDetail::SetValueA(d) => d.encode(&mut builder.reborrow().init_set_value_a()), RPCAnswerDetail::WatchValueA(d) => { diff --git a/veilid-core/src/rpc_processor/coders/operations/mod.rs b/veilid-core/src/rpc_processor/coders/operations/mod.rs index 8426ff1b..687ca68e 100644 --- a/veilid-core/src/rpc_processor/coders/operations/mod.rs +++ b/veilid-core/src/rpc_processor/coders/operations/mod.rs @@ -1,5 +1,7 @@ mod answer; mod operation; +mod operation_app_call; +mod operation_app_message; mod operation_cancel_tunnel; mod operation_complete_tunnel; mod operation_find_block; @@ -22,6 +24,8 @@ mod statement; pub use answer::*; pub use operation::*; +pub use operation_app_call::*; +pub use operation_app_message::*; pub use operation_cancel_tunnel::*; pub use operation_complete_tunnel::*; pub use operation_find_block::*; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs b/veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs new file mode 100644 index 00000000..609999bb --- /dev/null +++ b/veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs @@ -0,0 +1,44 @@ +use crate::*; +use rpc_processor::*; + +#[derive(Debug, Clone)] +pub struct RPCOperationAppCallQ { + pub message: Vec, +} + +impl RPCOperationAppCallQ { + pub fn decode( + reader: &veilid_capnp::operation_app_call_q::Reader, + ) -> Result { + let message = reader.get_message().map_err(RPCError::protocol)?.to_vec(); + Ok(RPCOperationAppCallQ { message }) + } + pub fn encode( + &self, + builder: &mut veilid_capnp::operation_app_call_q::Builder, + ) -> Result<(), RPCError> { + builder.set_message(&self.message); + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct RPCOperationAppCallA { + pub message: Vec, +} + +impl RPCOperationAppCallA { + pub fn decode( + reader: &veilid_capnp::operation_app_call_a::Reader, + ) -> Result { + let message = reader.get_message().map_err(RPCError::protocol)?.to_vec(); + Ok(RPCOperationAppCallA { message }) + } + pub fn encode( + &self, + builder: &mut veilid_capnp::operation_app_call_a::Builder, + ) -> Result<(), RPCError> { + builder.set_message(&self.message); + Ok(()) + } +} diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs b/veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs new file mode 100644 index 00000000..5a844f02 --- /dev/null +++ b/veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs @@ -0,0 +1,23 @@ +use crate::*; +use rpc_processor::*; + +#[derive(Debug, Clone)] +pub struct RPCOperationAppMessage { + pub message: Vec, +} + +impl RPCOperationAppMessage { + pub fn decode( + reader: &veilid_capnp::operation_app_message::Reader, + ) -> Result { + let message = reader.get_message().map_err(RPCError::protocol)?.to_vec(); + Ok(RPCOperationAppMessage { message }) + } + pub fn encode( + &self, + builder: &mut veilid_capnp::operation_app_message::Builder, + ) -> Result<(), RPCError> { + builder.set_message(&self.message); + Ok(()) + } +} diff --git a/veilid-core/src/rpc_processor/coders/operations/question.rs b/veilid-core/src/rpc_processor/coders/operations/question.rs index ae62ca72..02995b26 100644 --- a/veilid-core/src/rpc_processor/coders/operations/question.rs +++ b/veilid-core/src/rpc_processor/coders/operations/question.rs @@ -40,6 +40,7 @@ impl RPCQuestion { pub enum RPCQuestionDetail { StatusQ(RPCOperationStatusQ), FindNodeQ(RPCOperationFindNodeQ), + AppCallQ(RPCOperationAppCallQ), GetValueQ(RPCOperationGetValueQ), SetValueQ(RPCOperationSetValueQ), WatchValueQ(RPCOperationWatchValueQ), @@ -55,6 +56,7 @@ impl RPCQuestionDetail { match self { RPCQuestionDetail::StatusQ(_) => "StatusQ", RPCQuestionDetail::FindNodeQ(_) => "FindNodeQ", + RPCQuestionDetail::AppCallQ(_) => "AppCallQ", RPCQuestionDetail::GetValueQ(_) => "GetValueQ", RPCQuestionDetail::SetValueQ(_) => "SetValueQ", RPCQuestionDetail::WatchValueQ(_) => "WatchValueQ", @@ -81,6 +83,11 @@ impl RPCQuestionDetail { let out = RPCOperationFindNodeQ::decode(&op_reader)?; RPCQuestionDetail::FindNodeQ(out) } + veilid_capnp::question::detail::Which::AppCallQ(r) => { + let op_reader = r.map_err(RPCError::protocol)?; + let out = RPCOperationAppCallQ::decode(&op_reader)?; + RPCQuestionDetail::AppCallQ(out) + } veilid_capnp::question::detail::GetValueQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationGetValueQ::decode(&op_reader)?; @@ -131,6 +138,7 @@ impl RPCQuestionDetail { match self { RPCQuestionDetail::StatusQ(d) => d.encode(&mut builder.reborrow().init_status_q()), RPCQuestionDetail::FindNodeQ(d) => d.encode(&mut builder.reborrow().init_find_node_q()), + RPCQuestionDetail::AppCallQ(d) => d.encode(&mut builder.reborrow().init_app_call_q()), RPCQuestionDetail::GetValueQ(d) => d.encode(&mut builder.reborrow().init_get_value_q()), RPCQuestionDetail::SetValueQ(d) => d.encode(&mut builder.reborrow().init_set_value_q()), RPCQuestionDetail::WatchValueQ(d) => { diff --git a/veilid-core/src/rpc_processor/coders/operations/statement.rs b/veilid-core/src/rpc_processor/coders/operations/statement.rs index 4eb75f5c..bbac6455 100644 --- a/veilid-core/src/rpc_processor/coders/operations/statement.rs +++ b/veilid-core/src/rpc_processor/coders/operations/statement.rs @@ -42,6 +42,7 @@ pub enum RPCStatementDetail { ValueChanged(RPCOperationValueChanged), Signal(RPCOperationSignal), ReturnReceipt(RPCOperationReturnReceipt), + AppMessage(RPCOperationAppMessage), } impl RPCStatementDetail { @@ -53,6 +54,7 @@ impl RPCStatementDetail { RPCStatementDetail::ValueChanged(_) => "ValueChanged", RPCStatementDetail::Signal(_) => "Signal", RPCStatementDetail::ReturnReceipt(_) => "ReturnReceipt", + RPCStatementDetail::AppMessage(_) => "AppMessage", } } pub fn decode( @@ -91,6 +93,11 @@ impl RPCStatementDetail { let out = RPCOperationReturnReceipt::decode(&op_reader)?; RPCStatementDetail::ReturnReceipt(out) } + veilid_capnp::statement::detail::AppMessage(r) => { + let op_reader = r.map_err(RPCError::protocol)?; + let out = RPCOperationAppMessage::decode(&op_reader)?; + RPCStatementDetail::AppMessage(out) + } }; Ok(out) } @@ -113,6 +120,9 @@ impl RPCStatementDetail { RPCStatementDetail::ReturnReceipt(d) => { d.encode(&mut builder.reborrow().init_return_receipt()) } + RPCStatementDetail::AppMessage(d) => { + d.encode(&mut builder.reborrow().init_app_message()) + } } } } diff --git a/veilid-core/src/rpc_processor/coders/private_safety_route.rs b/veilid-core/src/rpc_processor/coders/private_safety_route.rs index 6c3c70f6..f96261cb 100644 --- a/veilid-core/src/rpc_processor/coders/private_safety_route.rs +++ b/veilid-core/src/rpc_processor/coders/private_safety_route.rs @@ -2,80 +2,6 @@ use super::*; //////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Clone, Debug)] -pub struct RouteHopData { - pub nonce: Nonce, - pub blob: Vec, -} - -#[derive(Clone, Debug)] -pub struct RouteHop { - pub dial_info: NodeDialInfo, - pub next_hop: Option, -} - -#[derive(Clone, Debug)] -pub struct PrivateRoute { - pub public_key: DHTKey, - pub hop_count: u8, - pub hops: Option, -} - -impl PrivateRoute { - pub fn new_stub(public_key: DHTKey) -> Self { - Self { - public_key, - hop_count: 0, - hops: None, - } - } -} - -impl fmt::Display for PrivateRoute { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "PR({:?}+{}{})", - self.public_key, - self.hop_count, - if let Some(hops) = &self.hops { - format!("->{}", hops.dial_info) - } else { - "".to_owned() - } - ) - } -} - -#[derive(Clone, Debug)] -pub enum SafetyRouteHops { - Data(RouteHopData), - Private(PrivateRoute), -} - -#[derive(Clone, Debug)] -pub struct SafetyRoute { - pub public_key: DHTKey, - pub hop_count: u8, - pub hops: SafetyRouteHops, -} - -impl fmt::Display for SafetyRoute { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "SR({:?}+{}{})", - self.public_key, - self.hop_count, - match &self.hops { - SafetyRouteHops::Data(_) => "".to_owned(), - SafetyRouteHops::Private(p) => format!("->{}", p), - } - ) - } -} -//////////////////////////////////////////////////////////////////////////////////////////////////// - pub fn encode_route_hop_data( route_hop_data: &RouteHopData, builder: &mut veilid_capnp::route_hop_data::Builder, diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index b4c38203..6a6ac7ad 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -1,6 +1,9 @@ mod coders; mod destination; +mod operation_waiter; mod private_route; +mod rpc_app_call; +mod rpc_app_message; mod rpc_cancel_tunnel; mod rpc_complete_tunnel; mod rpc_error; @@ -20,6 +23,7 @@ mod rpc_value_changed; mod rpc_watch_value; pub use destination::*; +pub use operation_waiter::*; pub use private_route::*; pub use rpc_error::*; @@ -108,8 +112,7 @@ where #[derive(Debug)] struct WaitableReply { - op_id: OperationId, - eventual: EventualValue<(Option, RPCMessage)>, + handle: OperationWaitHandle, timeout: u64, node_ref: NodeRef, send_ts: u64, @@ -138,62 +141,162 @@ struct RenderedOperation { ///////////////////////////////////////////////////////////////////// pub struct RPCProcessorInner { - network_manager: NetworkManager, - routing_table: RoutingTable, - node_id: DHTKey, - node_id_secret: DHTKeySecret, send_channel: Option, RPCMessageEncoded)>>, - timeout: u64, - max_route_hop_count: usize, - waiting_rpc_table: BTreeMap, RPCMessage)>>, stop_source: Option, worker_join_handles: Vec>, } +pub struct RPCProcessorUnlockedInner { + node_id: DHTKey, + node_id_secret: DHTKeySecret, + timeout: u64, + queue_size: u32, + concurrency: u32, + max_route_hop_count: usize, + validate_dial_info_receipt_time_ms: u32, + update_callback: UpdateCallback, + waiting_rpc_table: OperationWaiter, + waiting_app_call_table: OperationWaiter>, +} + #[derive(Clone)] pub struct RPCProcessor { crypto: Crypto, config: VeilidConfig, + network_manager: NetworkManager, + routing_table: RoutingTable, inner: Arc>, + unlocked_inner: Arc, } impl RPCProcessor { - fn new_inner(network_manager: NetworkManager) -> RPCProcessorInner { + fn new_inner() -> RPCProcessorInner { RPCProcessorInner { - network_manager: network_manager.clone(), - routing_table: network_manager.routing_table(), - node_id: DHTKey::default(), - node_id_secret: DHTKeySecret::default(), send_channel: None, - timeout: 10000000, - max_route_hop_count: 7, - waiting_rpc_table: BTreeMap::new(), stop_source: None, worker_join_handles: Vec::new(), } } - pub fn new(network_manager: NetworkManager) -> Self { + fn new_unlocked_inner( + config: VeilidConfig, + update_callback: UpdateCallback, + ) -> RPCProcessorUnlockedInner { + // make local copy of node id for easy access + let c = config.get(); + let node_id = c.network.node_id; + let node_id_secret = c.network.node_id_secret; + + // set up channel + let mut concurrency = c.network.rpc.concurrency; + let mut queue_size = c.network.rpc.queue_size; + let mut timeout = ms_to_us(c.network.rpc.timeout_ms); + let mut max_route_hop_count = c.network.rpc.max_route_hop_count as usize; + if concurrency == 0 { + concurrency = intf::get_concurrency() / 2; + if concurrency == 0 { + concurrency = 1; + } + } + if queue_size == 0 { + queue_size = 1024; + } + if timeout == 0 { + timeout = 10000000; + } + if max_route_hop_count == 0 { + max_route_hop_count = 7usize; + } + let validate_dial_info_receipt_time_ms = c.network.dht.validate_dial_info_receipt_time_ms; + + RPCProcessorUnlockedInner { + node_id, + node_id_secret, + timeout, + queue_size, + concurrency, + max_route_hop_count, + validate_dial_info_receipt_time_ms, + update_callback, + waiting_rpc_table: OperationWaiter::new(), + waiting_app_call_table: OperationWaiter::new(), + } + } + pub fn new(network_manager: NetworkManager, update_callback: UpdateCallback) -> Self { + let config = network_manager.config(); Self { crypto: network_manager.crypto(), - config: network_manager.config(), - inner: Arc::new(Mutex::new(Self::new_inner(network_manager))), + config: config.clone(), + network_manager: network_manager.clone(), + routing_table: network_manager.routing_table(), + inner: Arc::new(Mutex::new(Self::new_inner())), + unlocked_inner: Arc::new(Self::new_unlocked_inner(config, update_callback)), } } pub fn network_manager(&self) -> NetworkManager { - self.inner.lock().network_manager.clone() + self.network_manager.clone() } pub fn routing_table(&self) -> RoutingTable { - self.inner.lock().routing_table.clone() + self.routing_table.clone() } - pub fn node_id(&self) -> DHTKey { - self.inner.lock().node_id + ////////////////////////////////////////////////////////////////////// + + #[instrument(level = "debug", skip_all, err)] + pub async fn startup(&self) -> EyreResult<()> { + trace!("startup rpc processor"); + let mut inner = self.inner.lock(); + + let channel = flume::bounded(self.unlocked_inner.queue_size as usize); + inner.send_channel = Some(channel.0.clone()); + inner.stop_source = Some(StopSource::new()); + + // spin up N workers + trace!( + "Spinning up {} RPC workers", + self.unlocked_inner.concurrency + ); + for _ in 0..self.unlocked_inner.concurrency { + let this = self.clone(); + let receiver = channel.1.clone(); + let jh = intf::spawn(Self::rpc_worker( + this, + inner.stop_source.as_ref().unwrap().token(), + receiver, + )); + inner.worker_join_handles.push(jh); + } + + Ok(()) } - pub fn node_id_secret(&self) -> DHTKeySecret { - self.inner.lock().node_id_secret + #[instrument(level = "debug", skip_all)] + pub async fn shutdown(&self) { + debug!("starting rpc processor shutdown"); + + // Stop the rpc workers + let mut unord = FuturesUnordered::new(); + { + let mut inner = self.inner.lock(); + // take the join handles out + for h in inner.worker_join_handles.drain(..) { + unord.push(h); + } + // drop the stop + drop(inner.stop_source.take()); + } + debug!("stopping {} rpc worker tasks", unord.len()); + + // Wait for them to complete + while unord.next().await.is_some() {} + + debug!("resetting rpc processor state"); + + // Release the rpc processor + *self.inner.lock() = Self::new_inner(); + + debug!("finished rpc processor shutdown"); } ////////////////////////////////////////////////////////////////////// @@ -278,71 +381,18 @@ impl RPCProcessor { }) } - // set up wait for reply - fn add_op_id_waiter(&self, op_id: OperationId) -> EventualValue<(Option, RPCMessage)> { - let mut inner = self.inner.lock(); - let e = EventualValue::new(); - inner.waiting_rpc_table.insert(op_id, e.clone()); - e - } - - // remove wait for reply - fn cancel_op_id_waiter(&self, op_id: OperationId) { - let mut inner = self.inner.lock(); - inner.waiting_rpc_table.remove(&op_id); - } - - // complete the reply - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - async fn complete_op_id_waiter(&self, msg: RPCMessage) -> Result<(), RPCError> { - let op_id = msg.operation.op_id(); - let eventual = { - let mut inner = self.inner.lock(); - inner - .waiting_rpc_table - .remove(&op_id) - .ok_or_else(RPCError::else_internal(format!( - "Unmatched operation id: {:#?}", - msg - )))? - }; - eventual.resolve((Span::current().id(), msg)).await; - Ok(()) - } - - // wait for reply - async fn do_wait_for_reply( - &self, - waitable_reply: &WaitableReply, - ) -> Result, RPCError> { - let timeout_ms = u32::try_from(waitable_reply.timeout / 1000u64) - .map_err(RPCError::map_internal("invalid timeout"))?; - // wait for eventualvalue - let start_ts = intf::get_timestamp(); - let res = intf::timeout(timeout_ms, waitable_reply.eventual.instance()) - .await - .into_timeout_or(); - Ok(res.map(|res| { - let (_span_id, rpcreader) = res.take_value().unwrap(); - let end_ts = intf::get_timestamp(); - - // fixme: causes crashes? "Missing otel data span extensions"?? - //Span::current().follows_from(span_id); - - (rpcreader, end_ts - start_ts) - })) - } - #[instrument(level = "trace", skip(self, waitable_reply), err)] async fn wait_for_reply( &self, waitable_reply: WaitableReply, ) -> Result, RPCError> { - let out = self.do_wait_for_reply(&waitable_reply).await; + let out = self + .unlocked_inner + .waiting_rpc_table + .wait_for_op(waitable_reply.handle, waitable_reply.timeout) + .await; match &out { Err(_) | Ok(TimeoutOr::Timeout) => { - self.cancel_op_id_waiter(waitable_reply.op_id); - waitable_reply.node_ref.stats_question_lost(); } Ok(TimeoutOr::Value((rpcreader, _))) => { @@ -476,7 +526,7 @@ impl RPCProcessor { } // Verify hop count isn't larger than out maximum routed hop count - if out_hop_count > self.inner.lock().max_route_hop_count { + if out_hop_count > self.unlocked_inner.max_route_hop_count { return Err(RPCError::internal("hop count too long for route")) .map_err(logthru_rpc!(warn)); } @@ -574,10 +624,10 @@ impl RPCProcessor { // Calculate answer timeout // Timeout is number of hops times the timeout per hop - let timeout = self.inner.lock().timeout * (hop_count as u64); + let timeout = self.unlocked_inner.timeout * (hop_count as u64); // Set up op id eventual - let eventual = self.add_op_id_waiter(op_id); + let handle = self.unlocked_inner.waiting_rpc_table.add_op_waiter(op_id); // Send question let bytes = message.len() as u64; @@ -588,13 +638,11 @@ impl RPCProcessor { .await .map_err(|e| { // If we're returning an error, clean up - self.cancel_op_id_waiter(op_id); node_ref .stats_failed_to_send(send_ts, true); RPCError::network(e) })? => { // If we couldn't send we're still cleaning up - self.cancel_op_id_waiter(op_id); node_ref .stats_failed_to_send(send_ts, true); } @@ -605,8 +653,7 @@ impl RPCProcessor { // Pass back waitable reply completion Ok(NetworkResult::value(WaitableReply { - op_id, - eventual, + handle, timeout, node_ref, send_ts, @@ -794,7 +841,7 @@ impl RPCProcessor { let mut opt_sender_nr: Option = None; if let Some(sender_node_info) = operation.sender_node_info() { // Sender NodeInfo was specified, update our routing table with it - if !self.filter_node_info(RoutingDomain::PublicInternet, &sender_node_info.node_info) { + if !self.filter_node_info(routing_domain, &sender_node_info.node_info) { return Err(RPCError::invalid_format( "sender signednodeinfo has invalid peer scope", )); @@ -853,6 +900,7 @@ impl RPCProcessor { RPCOperationKind::Question(q) => match q.detail() { RPCQuestionDetail::StatusQ(_) => self.process_status_q(msg).await, RPCQuestionDetail::FindNodeQ(_) => self.process_find_node_q(msg).await, + RPCQuestionDetail::AppCallQ(_) => self.process_app_call_q(msg).await, RPCQuestionDetail::GetValueQ(_) => self.process_get_value_q(msg).await, RPCQuestionDetail::SetValueQ(_) => self.process_set_value_q(msg).await, RPCQuestionDetail::WatchValueQ(_) => self.process_watch_value_q(msg).await, @@ -871,8 +919,14 @@ impl RPCProcessor { RPCStatementDetail::ValueChanged(_) => self.process_value_changed(msg).await, RPCStatementDetail::Signal(_) => self.process_signal(msg).await, RPCStatementDetail::ReturnReceipt(_) => self.process_return_receipt(msg).await, + RPCStatementDetail::AppMessage(_) => self.process_app_message(msg).await, }, - RPCOperationKind::Answer(_) => self.complete_op_id_waiter(msg).await, + RPCOperationKind::Answer(_) => { + self.unlocked_inner + .waiting_rpc_table + .complete_op_waiter(msg.operation.op_id(), msg) + .await + } } } @@ -908,85 +962,6 @@ impl RPCProcessor { } } - #[instrument(level = "debug", skip_all, err)] - pub async fn startup(&self) -> EyreResult<()> { - trace!("startup rpc processor"); - let mut inner = self.inner.lock(); - // make local copy of node id for easy access - let c = self.config.get(); - inner.node_id = c.network.node_id; - inner.node_id_secret = c.network.node_id_secret; - - // set up channel - let mut concurrency = c.network.rpc.concurrency; - let mut queue_size = c.network.rpc.queue_size; - let mut timeout = ms_to_us(c.network.rpc.timeout_ms); - let mut max_route_hop_count = c.network.rpc.max_route_hop_count as usize; - if concurrency == 0 { - concurrency = intf::get_concurrency() / 2; - if concurrency == 0 { - concurrency = 1; - } - } - if queue_size == 0 { - queue_size = 1024; - } - if timeout == 0 { - timeout = 10000000; - } - if max_route_hop_count == 0 { - max_route_hop_count = 7usize; - } - inner.timeout = timeout; - inner.max_route_hop_count = max_route_hop_count; - let channel = flume::bounded(queue_size as usize); - inner.send_channel = Some(channel.0.clone()); - inner.stop_source = Some(StopSource::new()); - - // spin up N workers - trace!("Spinning up {} RPC workers", concurrency); - for _ in 0..concurrency { - let this = self.clone(); - let receiver = channel.1.clone(); - let jh = intf::spawn(Self::rpc_worker( - this, - inner.stop_source.as_ref().unwrap().token(), - receiver, - )); - inner.worker_join_handles.push(jh); - } - - Ok(()) - } - - #[instrument(level = "debug", skip_all)] - pub async fn shutdown(&self) { - debug!("starting rpc processor shutdown"); - - // Stop the rpc workers - let mut unord = FuturesUnordered::new(); - { - let mut inner = self.inner.lock(); - // take the join handles out - for h in inner.worker_join_handles.drain(..) { - unord.push(h); - } - // drop the stop - drop(inner.stop_source.take()); - } - debug!("stopping {} rpc worker tasks", unord.len()); - - // Wait for them to complete - while unord.next().await.is_some() {} - - debug!("resetting rpc processor state"); - - // Release the rpc processor - *self.inner.lock() = Self::new_inner(self.network_manager()); - - debug!("finished rpc processor shutdown"); - } - #[instrument(level = "trace", skip(self, body), err)] pub fn enqueue_message( &self, diff --git a/veilid-core/src/rpc_processor/operation_waiter.rs b/veilid-core/src/rpc_processor/operation_waiter.rs new file mode 100644 index 00000000..cb5d4afa --- /dev/null +++ b/veilid-core/src/rpc_processor/operation_waiter.rs @@ -0,0 +1,136 @@ +use super::*; + +#[derive(Debug)] +pub struct OperationWaitHandle +where + T: Unpin, +{ + waiter: OperationWaiter, + op_id: OperationId, + eventual_instance: Option, T)>>, +} + +impl Drop for OperationWaitHandle +where + T: Unpin, +{ + fn drop(&mut self) { + if self.eventual_instance.is_some() { + self.waiter.cancel_op_waiter(self.op_id); + } + } +} + +#[derive(Debug)] +pub struct OperationWaiterInner +where + T: Unpin, +{ + waiting_op_table: HashMap, T)>>, +} + +#[derive(Debug)] +pub struct OperationWaiter +where + T: Unpin, +{ + inner: Arc>>, +} + +impl Clone for OperationWaiter +where + T: Unpin, +{ + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl OperationWaiter +where + T: Unpin, +{ + pub fn new() -> Self { + Self { + inner: Arc::new(Mutex::new(OperationWaiterInner { + waiting_op_table: HashMap::new(), + })), + } + } + + // set up wait for op + pub fn add_op_waiter(&self, op_id: OperationId) -> OperationWaitHandle { + let mut inner = self.inner.lock(); + let e = EventualValue::new(); + if inner.waiting_op_table.insert(op_id, e.clone()).is_some() { + error!( + "add_op_waiter collision should not happen for op_id {}", + op_id + ); + } + + OperationWaitHandle { + waiter: self.clone(), + op_id, + eventual_instance: Some(e.instance()), + } + } + + // remove wait for op + fn cancel_op_waiter(&self, op_id: OperationId) { + let mut inner = self.inner.lock(); + inner.waiting_op_table.remove(&op_id); + } + + // complete the app call + #[instrument(level = "trace", skip(self, message), err)] + pub async fn complete_op_waiter(&self, op_id: OperationId, message: T) -> Result<(), RPCError> { + let eventual = { + let mut inner = self.inner.lock(); + inner + .waiting_op_table + .remove(&op_id) + .ok_or_else(RPCError::else_internal(format!( + "Unmatched app call id, possibly too late for timeout: {}", + op_id + )))? + }; + eventual.resolve((Span::current().id(), message)).await; + Ok(()) + } + + pub async fn wait_for_op( + &self, + mut handle: OperationWaitHandle, + timeout: u64, + ) -> Result, RPCError> { + let timeout_ms = u32::try_from(timeout / 1000u64) + .map_err(|e| RPCError::map_internal("invalid timeout")(e))?; + + // Take the instance + // After this, we must manually cancel since the cancel on handle drop is disabled + let eventual_instance = handle.eventual_instance.take().unwrap(); + + // wait for eventualvalue + let start_ts = intf::get_timestamp(); + let res = intf::timeout(timeout_ms, eventual_instance) + .await + .into_timeout_or(); + Ok(res + .on_timeout(|| { + log_rpc!(debug "op wait timed out: {}", handle.op_id); + self.cancel_op_waiter(handle.op_id); + }) + .map(|res| { + let (_span_id, ret) = res.take_value().unwrap(); + let end_ts = intf::get_timestamp(); + + // fixme: causes crashes? "Missing otel data span extensions"?? + //Span::current().follows_from(span_id); + + (ret, end_ts - start_ts) + })) + } +} diff --git a/veilid-core/src/rpc_processor/private_route.rs b/veilid-core/src/rpc_processor/private_route.rs index dba50d60..b552e768 100644 --- a/veilid-core/src/rpc_processor/private_route.rs +++ b/veilid-core/src/rpc_processor/private_route.rs @@ -11,7 +11,7 @@ impl RPCProcessor { let pr_hopcount = private_route.hop_count as usize; let sr_hopcount = safety_route_spec.hops.len(); let hopcount = 1 + sr_hopcount + pr_hopcount; - if hopcount > self.inner.lock().max_route_hop_count { + if hopcount > self.unlocked_inner.max_route_hop_count { return Err(RPCError::internal("hop count too long for route")); } diff --git a/veilid-core/src/rpc_processor/rpc_app_call.rs b/veilid-core/src/rpc_processor/rpc_app_call.rs new file mode 100644 index 00000000..13466f4a --- /dev/null +++ b/veilid-core/src/rpc_processor/rpc_app_call.rs @@ -0,0 +1,99 @@ +use super::*; + +impl RPCProcessor { + // Sends a high level app request and wait for response + // Can be sent via all methods including relays and routes + #[instrument(level = "trace", skip(self), ret, err)] + pub async fn rpc_call_app_call( + self, + dest: Destination, + message: Vec, + ) -> Result>>, RPCError> { + let app_call_q = RPCOperationAppCallQ { message }; + let question = RPCQuestion::new(RespondTo::Sender, RPCQuestionDetail::AppCallQ(app_call_q)); + + // Send the app call question + let waitable_reply = network_result_try!(self.question(dest, question).await?); + + // Wait for reply + let (msg, latency) = match self.wait_for_reply(waitable_reply).await? { + TimeoutOr::Timeout => return Ok(NetworkResult::Timeout), + TimeoutOr::Value(v) => v, + }; + + // Get the right answer type + let app_call_a = match msg.operation.into_kind() { + RPCOperationKind::Answer(a) => match a.into_detail() { + RPCAnswerDetail::AppCallA(a) => a, + _ => return Err(RPCError::invalid_format("not an appcall answer")), + }, + _ => return Err(RPCError::invalid_format("not an answer")), + }; + + Ok(NetworkResult::value(Answer::new( + latency, + app_call_a.message, + ))) + } + + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] + pub(crate) async fn process_app_call_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + // Get the question + let app_call_q = match msg.operation.kind() { + RPCOperationKind::Question(q) => match q.detail() { + RPCQuestionDetail::AppCallQ(q) => q, + _ => panic!("not an appcall question"), + }, + _ => panic!("not a question"), + }; + + // Register a waiter for this app call + let id = msg.operation.op_id(); + let handle = self.unlocked_inner.waiting_app_call_table.add_op_waiter(id); + + // Pass the call up through the update callback + let sender = msg + .opt_sender_nr + .as_ref() + .map(|nr| NodeId::new(nr.node_id())); + let message = app_call_q.message.clone(); + (self.unlocked_inner.update_callback)(VeilidUpdate::AppCall(VeilidAppCall { + sender, + message, + id, + })); + + // Wait for an app call answer to come back from the app + let res = self + .unlocked_inner + .waiting_app_call_table + .wait_for_op(handle, self.unlocked_inner.timeout) + .await?; + let (message, _latency) = match res { + TimeoutOr::Timeout => { + // No message sent on timeout, but this isn't an error + log_rpc!(debug "App call timed out for id {}", id); + return Ok(()); + } + TimeoutOr::Value(v) => v, + }; + + // Return the appcall answer + let app_call_a = RPCOperationAppCallA { message }; + + // Send status answer + let res = self + .answer(msg, RPCAnswer::new(RPCAnswerDetail::AppCallA(app_call_a))) + .await?; + tracing::Span::current().record("res", &tracing::field::display(res)); + Ok(()) + } + + /// Exposed to API for apps to return app call answers + pub async fn app_call_reply(&self, id: u64, message: Vec) -> Result<(), RPCError> { + self.unlocked_inner + .waiting_app_call_table + .complete_op_waiter(id, message) + .await + } +} diff --git a/veilid-core/src/rpc_processor/rpc_app_message.rs b/veilid-core/src/rpc_processor/rpc_app_message.rs new file mode 100644 index 00000000..0b069280 --- /dev/null +++ b/veilid-core/src/rpc_processor/rpc_app_message.rs @@ -0,0 +1,42 @@ +use super::*; + +impl RPCProcessor { + // Sends a high level app message + // Can be sent via all methods including relays and routes + #[instrument(level = "trace", skip(self), ret, err)] + pub async fn rpc_call_app_message( + self, + dest: Destination, + message: Vec, + ) -> Result, RPCError> { + let app_message = RPCOperationAppMessage { message }; + let statement = RPCStatement::new(RPCStatementDetail::AppMessage(app_message)); + + // Send the app message request + network_result_try!(self.statement(dest, statement).await?); + + Ok(NetworkResult::value(())) + } + + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] + pub(crate) async fn process_app_message(&self, msg: RPCMessage) -> Result<(), RPCError> { + // Get the statement + let app_message = match msg.operation.into_kind() { + RPCOperationKind::Statement(s) => match s.into_detail() { + RPCStatementDetail::AppMessage(s) => s, + _ => panic!("not an app message"), + }, + _ => panic!("not a statement"), + }; + + // Pass the message up through the update callback + let sender = msg.opt_sender_nr.map(|nr| NodeId::new(nr.node_id())); + let message = app_message.message; + (self.unlocked_inner.update_callback)(VeilidUpdate::AppMessage(VeilidAppMessage { + sender, + message, + })); + + Ok(()) + } +} diff --git a/veilid-core/src/rpc_processor/rpc_error.rs b/veilid-core/src/rpc_processor/rpc_error.rs index bf650778..89c9eb11 100644 --- a/veilid-core/src/rpc_processor/rpc_error.rs +++ b/veilid-core/src/rpc_processor/rpc_error.rs @@ -3,8 +3,6 @@ use super::*; #[derive(ThisError, Debug, Clone, PartialOrd, PartialEq, Eq, Ord)] #[must_use] pub enum RPCError { - #[error("[RPCError: Unreachable({0})]")] - Unreachable(DHTKey), #[error("[RPCError: Unimplemented({0})]")] Unimplemented(String), #[error("[RPCError: InvalidFormat({0})]")] @@ -18,9 +16,6 @@ pub enum RPCError { } impl RPCError { - pub fn unreachable(key: DHTKey) -> Self { - Self::Unreachable(key) - } pub fn unimplemented(x: X) -> Self { Self::Unimplemented(x.to_string()) } @@ -52,3 +47,15 @@ impl RPCError { move |x| Self::Network(format!("{}: {}", message.to_string(), x.to_string())) } } + +impl From for VeilidAPIError { + fn from(e: RPCError) -> Self { + match e { + RPCError::Unimplemented(message) => VeilidAPIError::Unimplemented { message }, + RPCError::InvalidFormat(message) => VeilidAPIError::Generic { message }, + RPCError::Protocol(message) => VeilidAPIError::Generic { message }, + RPCError::Internal(message) => VeilidAPIError::Internal { message }, + RPCError::Network(message) => VeilidAPIError::Generic { message }, + } + } +} diff --git a/veilid-core/src/rpc_processor/rpc_signal.rs b/veilid-core/src/rpc_processor/rpc_signal.rs index 3773e1bc..08cb3713 100644 --- a/veilid-core/src/rpc_processor/rpc_signal.rs +++ b/veilid-core/src/rpc_processor/rpc_signal.rs @@ -24,7 +24,7 @@ impl RPCProcessor { let signal = match msg.operation.into_kind() { RPCOperationKind::Statement(s) => match s.into_detail() { RPCStatementDetail::Signal(s) => s, - _ => panic!("not a node info update"), + _ => panic!("not a signal"), }, _ => panic!("not a statement"), }; diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 3ff0f693..e1cc7788 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -10,13 +10,7 @@ impl RPCProcessor { redirect: bool, ) -> Result { let network_manager = self.network_manager(); - let receipt_time = ms_to_us( - self.config - .get() - .network - .dht - .validate_dial_info_receipt_time_ms, - ); + let receipt_time = ms_to_us(self.unlocked_inner.validate_dial_info_receipt_time_ms); // Generate receipt and waitable eventual so we can see if we get the receipt back let (receipt, eventual_value) = network_manager diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 79d68d3f..50835a13 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -1,8 +1,13 @@ #![allow(dead_code)] mod debug; +mod privacy; +mod routing_context; mod serialize_helpers; + pub use debug::*; +pub use privacy::*; +pub use routing_context::*; pub use serialize_helpers::*; use crate::*; @@ -15,17 +20,17 @@ pub use alloc::string::ToString; pub use attachment_manager::AttachmentManager; pub use core::str::FromStr; pub use dht::Crypto; -pub use dht::{generate_secret, sign, verify, DHTKey, DHTKeySecret, DHTSignature}; +pub use dht::{generate_secret, sign, verify, DHTKey, DHTKeySecret, DHTSignature, Nonce}; pub use intf::BlockStore; pub use intf::ProtectedStore; pub use intf::TableStore; pub use network_manager::NetworkManager; pub use routing_table::RoutingTable; -//pub use rpc_processor::RPCProcessor; use core::fmt; use core_context::{api_shutdown, VeilidCoreContext}; use enumset::*; +use rpc_processor::RPCProcessor; use serde::*; use xx::*; @@ -68,8 +73,8 @@ pub enum VeilidAPIError { Shutdown, #[error("Node not found: {node_id}")] NodeNotFound { node_id: NodeId }, - #[error("No dial info: {node_id}")] - NoDialInfo { node_id: NodeId }, + #[error("No connection: {message}")] + NoConnection { message: String }, #[error("No peer info: {node_id}")] NoPeerInfo { node_id: NodeId }, #[error("Internal: {message}")] @@ -106,8 +111,8 @@ impl VeilidAPIError { pub fn node_not_found(node_id: NodeId) -> Self { Self::NodeNotFound { node_id } } - pub fn no_dial_info(node_id: NodeId) -> Self { - Self::NoDialInfo { node_id } + pub fn no_connection(message: String) -> Self { + Self::NoConnection { message } } pub fn no_peer_info(node_id: NodeId) -> Self { Self::NoPeerInfo { node_id } @@ -216,12 +221,30 @@ impl fmt::Display for VeilidLogLevel { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct VeilidStateLog { +pub struct VeilidLog { pub log_level: VeilidLogLevel, pub message: String, pub backtrace: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct VeilidAppMessage { + /// Some(sender) if the message was sent directly, None if received via a private/safety route + pub sender: Option, + /// The content of the message to deliver to the application + pub message: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct VeilidAppCall { + /// Some(sender) if the request was sent directly, None if received via a private/safety route + pub sender: Option, + /// The content of the request to deliver to the application + pub message: Vec, + /// The id to reply to + pub id: u64, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct VeilidStateAttachment { pub state: AttachmentState, @@ -247,7 +270,9 @@ pub struct VeilidStateNetwork { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind")] pub enum VeilidUpdate { - Log(VeilidStateLog), + Log(VeilidLog), + AppMessage(VeilidAppMessage), + AppCall(VeilidAppCall), Attachment(VeilidStateAttachment), Network(VeilidStateNetwork), Shutdown, @@ -380,6 +405,24 @@ impl MatchesDialInfoFilter for DialInfoDetail { } } +impl DialInfoDetail { + pub fn reliable_sort(a: &DialInfoDetail, b: &DialInfoDetail) -> core::cmp::Ordering { + if a.class < b.class { + return core::cmp::Ordering::Less; + } + if a.class > b.class { + return core::cmp::Ordering::Greater; + } + DialInfo::reliable_sort(&a.dial_info, &b.dial_info) + } + pub const NO_SORT: std::option::Option< + for<'r, 's> fn( + &'r veilid_api::DialInfoDetail, + &'s veilid_api::DialInfoDetail, + ) -> std::cmp::Ordering, + > = None:: core::cmp::Ordering>; +} + #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)] pub enum NetworkClass { InboundCapable = 0, // I = Inbound capable without relay, may require signal @@ -471,29 +514,59 @@ pub struct NodeInfo { } impl NodeInfo { - pub fn first_filtered_dial_info_detail(&self, filter: F) -> Option + pub fn first_filtered_dial_info_detail( + &self, + sort: Option, + filter: F, + ) -> Option where + S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, F: Fn(&DialInfoDetail) -> bool, { - for did in &self.dial_info_detail_list { - if filter(did) { - return Some(did.clone()); + if let Some(sort) = sort { + let mut dids = self.dial_info_detail_list.clone(); + dids.sort_by(sort); + for did in dids { + if filter(&did) { + return Some(did); + } } - } + } else { + for did in &self.dial_info_detail_list { + if filter(did) { + return Some(did.clone()); + } + } + }; None } - pub fn all_filtered_dial_info_details(&self, filter: F) -> Vec + pub fn all_filtered_dial_info_details( + &self, + sort: Option, + filter: F, + ) -> Vec where + S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, F: Fn(&DialInfoDetail) -> bool, { let mut dial_info_detail_list = Vec::new(); - for did in &self.dial_info_detail_list { - if filter(did) { - dial_info_detail_list.push(did.clone()); + if let Some(sort) = sort { + let mut dids = self.dial_info_detail_list.clone(); + dids.sort_by(sort); + for did in dids { + if filter(&did) { + dial_info_detail_list.push(did); + } } - } + } else { + for did in &self.dial_info_detail_list { + if filter(did) { + dial_info_detail_list.push(did.clone()); + } + } + }; dial_info_detail_list } @@ -599,6 +672,38 @@ impl ProtocolType { ProtocolType::TCP | ProtocolType::WS | ProtocolType::WSS => LowLevelProtocolType::TCP, } } + pub fn sort_order(&self, reliable: bool) -> usize { + match self { + ProtocolType::UDP => { + if reliable { + 3 + } else { + 0 + } + } + ProtocolType::TCP => { + if reliable { + 0 + } else { + 1 + } + } + ProtocolType::WS => { + if reliable { + 1 + } else { + 2 + } + } + ProtocolType::WSS => { + if reliable { + 2 + } else { + 3 + } + } + } + } } pub type ProtocolTypeSet = EnumSet; @@ -1372,6 +1477,24 @@ impl DialInfo { } } } + + pub fn reliable_sort(a: &DialInfo, b: &DialInfo) -> core::cmp::Ordering { + let ca = a.protocol_type().sort_order(true); + let cb = b.protocol_type().sort_order(true); + if ca < cb { + return core::cmp::Ordering::Less; + } + if ca > cb { + return core::cmp::Ordering::Greater; + } + match (a, b) { + (DialInfo::UDP(a), DialInfo::UDP(b)) => a.cmp(b), + (DialInfo::TCP(a), DialInfo::TCP(b)) => a.cmp(b), + (DialInfo::WS(a), DialInfo::WS(b)) => a.cmp(b), + (DialInfo::WSS(a), DialInfo::WSS(b)) => a.cmp(b), + _ => unreachable!(), + } + } } impl MatchesDialInfoFilter for DialInfo { @@ -1712,128 +1835,6 @@ pub struct PartialTunnel { ///////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct RouteHopSpec { - pub dial_info: NodeDialInfo, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct PrivateRouteSpec { - // - pub public_key: DHTKey, - pub secret_key: DHTKeySecret, - pub hops: Vec, -} - -impl PrivateRouteSpec { - pub fn new() -> Self { - let (pk, sk) = generate_secret(); - PrivateRouteSpec { - public_key: pk, - secret_key: sk, - hops: Vec::new(), - } - } -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct SafetyRouteSpec { - pub public_key: DHTKey, - pub secret_key: DHTKeySecret, - pub hops: Vec, -} - -impl SafetyRouteSpec { - pub fn new() -> Self { - let (pk, sk) = generate_secret(); - SafetyRouteSpec { - public_key: pk, - secret_key: sk, - hops: Vec::new(), - } - } -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct RoutingContextOptions { - pub safety_route_spec: Option, - pub private_route_spec: Option, -} - -///////////////////////////////////////////////////////////////////////////////////////////////////// - -pub struct RoutingContextInner { - api: VeilidAPI, - options: RoutingContextOptions, -} - -impl Drop for RoutingContextInner { - fn drop(&mut self) { - // self.api - // .borrow_mut() - // .routing_contexts - // //.remove(&self.id); - } -} - -#[derive(Clone)] -pub struct RoutingContext { - inner: Arc>, -} - -impl RoutingContext { - fn new(api: VeilidAPI, options: RoutingContextOptions) -> Self { - Self { - inner: Arc::new(Mutex::new(RoutingContextInner { api, options })), - } - } - - pub fn api(&self) -> VeilidAPI { - self.inner.lock().api.clone() - } - - /////////////////////////////////// - /// - - pub async fn get_value(&self, _value_key: ValueKey) -> Result, VeilidAPIError> { - panic!("unimplemented"); - } - - pub async fn set_value( - &self, - _value_key: ValueKey, - _value: Vec, - ) -> Result { - panic!("unimplemented"); - } - - pub async fn watch_value( - &self, - _value_key: ValueKey, - _callback: ValueChangeCallback, - ) -> Result { - panic!("unimplemented"); - } - - pub async fn cancel_watch_value(&self, _value_key: ValueKey) -> Result { - panic!("unimplemented"); - } - - pub async fn find_block(&self, _block_id: BlockId) -> Result, VeilidAPIError> { - panic!("unimplemented"); - } - - pub async fn supply_block(&self, _block_id: BlockId) -> Result { - panic!("unimplemented"); - } - - pub async fn signal(&self, _data: Vec) -> Result { - panic!("unimplemented"); - } -} - -///////////////////////////////////////////////////////////////////////////////////////////////////// - struct VeilidAPIInner { context: Option, } @@ -1930,14 +1931,13 @@ impl VeilidAPI { } Err(VeilidAPIError::not_initialized()) } - - // pub fn rpc_processor(&self) -> Result { - // let inner = self.inner.lock(); - // if let Some(context) = &inner.context { - // return Ok(context.attachment_manager.network_manager().rpc_processor()); - // } - // Err(VeilidAPIError::NotInitialized) - // } + pub fn rpc_processor(&self) -> Result { + let inner = self.inner.lock(); + if let Some(context) = &inner.context { + return Ok(context.attachment_manager.network_manager().rpc_processor()); + } + Err(VeilidAPIError::NotInitialized) + } //////////////////////////////////////////////////////////////// // Attach/Detach @@ -1978,60 +1978,6 @@ impl VeilidAPI { .map_err(|e| VeilidAPIError::internal(e)) } - //////////////////////////////////////////////////////////////// - // Direct Node Access (pretty much for testing only) - - // #[instrument(level = "debug", err, skip(self))] - // pub async fn search_dht(&self, node_id: NodeId) -> Result { - // let rpc_processor = self.rpc_processor()?; - // let config = self.config()?; - // let (count, fanout, timeout) = { - // let c = config.get(); - // ( - // c.network.dht.resolve_node_count, - // c.network.dht.resolve_node_fanout, - // c.network.dht.resolve_node_timeout_ms.map(ms_to_us), - // ) - // }; - - // let node_ref = rpc_processor - // .search_dht_single_key(node_id.key, count, fanout, timeout) - // .await - // .map_err(map_rpc_error!())?; - - // let answer = node_ref.peer_info(); - // if let Some(answer) = answer { - // Ok(answer) - // } else { - // Err(VeilidAPIError::NoPeerInfo { - // node_id: NodeId::new(node_ref.node_id()), - // }) - // } - // } - - // #[instrument(level = "debug", err, skip(self))] - // pub async fn search_dht_multi(&self, node_id: NodeId) -> Result, VeilidAPIError> { - // let rpc_processor = self.rpc_processor()?; - // let config = self.config()?; - // let (count, fanout, timeout) = { - // let c = config.get(); - // ( - // c.network.dht.resolve_node_count, - // c.network.dht.resolve_node_fanout, - // c.network.dht.resolve_node_timeout_ms.map(ms_to_us), - // ) - // }; - - // let node_refs = rpc_processor - // .search_dht_multi_key(node_id.key, count, fanout, timeout) - // .await - // .map_err(map_rpc_error!())?; - - // let answer = node_refs.iter().filter_map(|x| x.peer_info()).collect(); - - // Ok(answer) - // } - //////////////////////////////////////////////////////////////// // Safety / Private Route Handling @@ -2052,54 +1998,23 @@ impl VeilidAPI { } //////////////////////////////////////////////////////////////// - // Routing Contexts - // - // Safety route specified here is for _this_ node's anonymity as a sender, used via the 'route' operation - // Private route specified here is for _this_ node's anonymity as a receiver, passed out via the 'respond_to' field for replies - - #[instrument(skip(self))] - pub async fn safe_private( - &self, - safety_route_spec: SafetyRouteSpec, - private_route_spec: PrivateRouteSpec, - ) -> RoutingContext { - self.routing_context(RoutingContextOptions { - safety_route_spec: Some(safety_route_spec), - private_route_spec: Some(private_route_spec), - }) - .await - } + // Routing Context #[instrument(level = "debug", skip(self))] - pub async fn safe_public(&self, safety_route_spec: SafetyRouteSpec) -> RoutingContext { - self.routing_context(RoutingContextOptions { - safety_route_spec: Some(safety_route_spec), - private_route_spec: None, - }) - .await + pub fn routing_context(&self) -> RoutingContext { + RoutingContext::new(self.clone()) } - #[instrument(level = "debug", skip(self))] - pub async fn unsafe_private(&self, private_route_spec: PrivateRouteSpec) -> RoutingContext { - self.routing_context(RoutingContextOptions { - safety_route_spec: None, - private_route_spec: Some(private_route_spec), - }) - .await - } + //////////////////////////////////////////////////////////////// + // App Calls #[instrument(level = "debug", skip(self))] - pub async fn unsafe_public(&self) -> RoutingContext { - self.routing_context(RoutingContextOptions { - safety_route_spec: None, - private_route_spec: None, - }) - .await - } - - #[instrument(level = "debug", skip(self))] - pub async fn routing_context(&self, options: RoutingContextOptions) -> RoutingContext { - RoutingContext::new(self.clone(), options) + pub async fn app_call_reply(&self, id: u64, message: Vec) -> Result<(), VeilidAPIError> { + let rpc_processor = self.rpc_processor()?; + rpc_processor + .app_call_reply(id, message) + .await + .map_err(|e| e.into()) } //////////////////////////////////////////////////////////////// diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs new file mode 100644 index 00000000..046785a9 --- /dev/null +++ b/veilid-core/src/veilid_api/privacy.rs @@ -0,0 +1,124 @@ +use super::*; + +///////////////////////////////////////////////////////////////////////////////////////////////////// +// Privacy Specs + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct RouteHopSpec { + pub dial_info: NodeDialInfo, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct PrivateRouteSpec { + // + pub public_key: DHTKey, + pub secret_key: DHTKeySecret, + pub hops: Vec, +} + +impl PrivateRouteSpec { + pub fn new() -> Self { + let (pk, sk) = generate_secret(); + PrivateRouteSpec { + public_key: pk, + secret_key: sk, + hops: Vec::new(), + } + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct SafetyRouteSpec { + pub public_key: DHTKey, + pub secret_key: DHTKeySecret, + pub hops: Vec, +} + +impl SafetyRouteSpec { + pub fn new() -> Self { + let (pk, sk) = generate_secret(); + SafetyRouteSpec { + public_key: pk, + secret_key: sk, + hops: Vec::new(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Compiled Privacy Objects + +#[derive(Clone, Debug)] +pub struct RouteHopData { + pub nonce: Nonce, + pub blob: Vec, +} + +#[derive(Clone, Debug)] +pub struct RouteHop { + pub dial_info: NodeDialInfo, + pub next_hop: Option, +} + +#[derive(Clone, Debug)] +pub struct PrivateRoute { + pub public_key: DHTKey, + pub hop_count: u8, + pub hops: Option, +} + +impl PrivateRoute { + pub fn new_stub(public_key: DHTKey) -> Self { + Self { + public_key, + hop_count: 0, + hops: None, + } + } +} + +impl fmt::Display for PrivateRoute { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "PR({:?}+{}{})", + self.public_key, + self.hop_count, + if let Some(hops) = &self.hops { + format!("->{}", hops.dial_info) + } else { + "".to_owned() + } + ) + } +} + +#[derive(Clone, Debug)] +pub enum SafetyRouteHops { + Data(RouteHopData), + Private(PrivateRoute), +} + +#[derive(Clone, Debug)] +pub struct SafetyRoute { + pub public_key: DHTKey, + pub hop_count: u8, + pub hops: SafetyRouteHops, +} + +impl fmt::Display for SafetyRoute { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "SR({:?}+{}{})", + self.public_key, + self.hop_count, + match &self.hops { + SafetyRouteHops::Data(_) => "".to_owned(), + SafetyRouteHops::Private(p) => format!("->{}", p), + } + ) + } +} + +// xxx impl to_blob and from_blob using capnp here diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs new file mode 100644 index 00000000..5cfac250 --- /dev/null +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -0,0 +1,213 @@ +use super::*; +/////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, Debug)] +pub enum Target { + NodeId(NodeId), + PrivateRoute(PrivateRoute), +} + +pub struct RoutingContextInner {} + +pub struct RoutingContextUnlockedInner { + /// Safety route specified here is for _this_ node's anonymity as a sender, used via the 'route' operation + safety_route_spec: Option>, + /// Private route specified here is for _this_ node's anonymity as a receiver, passed out via the 'respond_to' field for replies + private_route_spec: Option>, + /// Choose reliable protocols over unreliable/faster protocols when available + reliable: bool, +} + +impl Drop for RoutingContextInner { + fn drop(&mut self) { + // self.api + // .borrow_mut() + // .routing_contexts + // //.remove(&self.id); + } +} + +#[derive(Clone)] +pub struct RoutingContext { + /// Veilid API handle + api: VeilidAPI, + inner: Arc>, + unlocked_inner: Arc, +} + +impl RoutingContext { + //////////////////////////////////////////////////////////////// + + pub(super) fn new(api: VeilidAPI) -> Self { + Self { + api, + inner: Arc::new(Mutex::new(RoutingContextInner {})), + unlocked_inner: Arc::new(RoutingContextUnlockedInner { + safety_route_spec: None, + private_route_spec: None, + reliable: false, + }), + } + } + + pub fn with_privacy( + self, + safety_route_spec: SafetyRouteSpec, + private_route_spec: PrivateRouteSpec, + ) -> Self { + Self { + api: self.api.clone(), + inner: Arc::new(Mutex::new(RoutingContextInner {})), + unlocked_inner: Arc::new(RoutingContextUnlockedInner { + safety_route_spec: Some(Arc::new(safety_route_spec)), + private_route_spec: Some(Arc::new(private_route_spec)), + reliable: self.unlocked_inner.reliable, + }), + } + } + + pub fn with_reliability(self) -> Self { + Self { + api: self.api.clone(), + inner: Arc::new(Mutex::new(RoutingContextInner {})), + unlocked_inner: Arc::new(RoutingContextUnlockedInner { + safety_route_spec: self.unlocked_inner.safety_route_spec.clone(), + private_route_spec: self.unlocked_inner.private_route_spec.clone(), + reliable: true, + }), + } + } + + pub fn api(&self) -> VeilidAPI { + self.api.clone() + } + + async fn get_destination( + &self, + target: Target, + ) -> Result { + let rpc_processor = self.api.rpc_processor()?; + + match target { + Target::NodeId(node_id) => { + // Resolve node + let mut nr = match rpc_processor.resolve_node(node_id.key).await { + Ok(Some(nr)) => nr, + Ok(None) => return Err(VeilidAPIError::NodeNotFound { node_id }), + Err(e) => return Err(e.into()), + }; + // Apply reliability sort + if self.unlocked_inner.reliable { + nr.set_reliable(); + } + Ok(rpc_processor::Destination::Direct { + target: nr, + safety_route_spec: self.unlocked_inner.safety_route_spec.clone(), + }) + } + Target::PrivateRoute(pr) => Ok(rpc_processor::Destination::PrivateRoute { + private_route: pr, + safety_route_spec: self.unlocked_inner.safety_route_spec.clone(), + }), + } + } + + //////////////////////////////////////////////////////////////// + // App-level Messaging + + #[instrument(level = "debug", err, skip(self))] + pub async fn app_call( + &self, + target: Target, + request: Vec, + ) -> Result, VeilidAPIError> { + let rpc_processor = self.api.rpc_processor()?; + + // Get destination + let dest = self.get_destination(target).await?; + + // Send app message + let answer = match rpc_processor.rpc_call_app_call(dest, request).await { + Ok(NetworkResult::Value(v)) => v, + Ok(NetworkResult::Timeout) => return Err(VeilidAPIError::Timeout), + Ok(NetworkResult::NoConnection(e)) => { + return Err(VeilidAPIError::NoConnection { + message: e.to_string(), + }) + } + Ok(NetworkResult::InvalidMessage(message)) => { + return Err(VeilidAPIError::Generic { message }) + } + Err(e) => return Err(e.into()), + }; + + Ok(answer.answer) + } + + #[instrument(level = "debug", err, skip(self))] + pub async fn app_message( + &self, + target: Target, + message: Vec, + ) -> Result<(), VeilidAPIError> { + let rpc_processor = self.api.rpc_processor()?; + + // Get destination + let dest = self.get_destination(target).await?; + + // Send app message + match rpc_processor.rpc_call_app_message(dest, message).await { + Ok(NetworkResult::Value(())) => {} + Ok(NetworkResult::Timeout) => return Err(VeilidAPIError::Timeout), + Ok(NetworkResult::NoConnection(e)) => { + return Err(VeilidAPIError::NoConnection { + message: e.to_string(), + }) + } + Ok(NetworkResult::InvalidMessage(message)) => { + return Err(VeilidAPIError::Generic { message }) + } + Err(e) => return Err(e.into()), + }; + + Ok(()) + } + + /////////////////////////////////// + /// DHT Values + + pub async fn get_value(&self, _value_key: ValueKey) -> Result, VeilidAPIError> { + panic!("unimplemented"); + } + + pub async fn set_value( + &self, + _value_key: ValueKey, + _value: Vec, + ) -> Result { + panic!("unimplemented"); + } + + pub async fn watch_value( + &self, + _value_key: ValueKey, + _callback: ValueChangeCallback, + ) -> Result { + panic!("unimplemented"); + } + + pub async fn cancel_watch_value(&self, _value_key: ValueKey) -> Result { + panic!("unimplemented"); + } + + /////////////////////////////////// + /// Block Store + + pub async fn find_block(&self, _block_id: BlockId) -> Result, VeilidAPIError> { + panic!("unimplemented"); + } + + pub async fn supply_block(&self, _block_id: BlockId) -> Result { + panic!("unimplemented"); + } +} diff --git a/veilid-core/src/xx/eventual_value.rs b/veilid-core/src/xx/eventual_value.rs index ab0825aa..2bdf2a43 100644 --- a/veilid-core/src/xx/eventual_value.rs +++ b/veilid-core/src/xx/eventual_value.rs @@ -61,6 +61,14 @@ pub struct EventualValueFuture { eventual: EventualValue, } +impl core::fmt::Debug for EventualValueFuture { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("EventualValueFuture") + .field("id", &self.id) + .finish() + } +} + impl Future for EventualValueFuture { type Output = EventualValue; fn poll(mut self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll { diff --git a/veilid-core/src/xx/timeout_or.rs b/veilid-core/src/xx/timeout_or.rs index 04f57d53..62786c36 100644 --- a/veilid-core/src/xx/timeout_or.rs +++ b/veilid-core/src/xx/timeout_or.rs @@ -104,6 +104,15 @@ impl TimeoutOr { Self::Value(v) => TimeoutOr::::Value(f(v)), } } + pub fn on_timeout(self, f: F) -> Self { + match self { + Self::Timeout => { + f(); + Self::Timeout + } + Self::Value(v) => Self::Value(v), + } + } pub fn into_timeout_error(self) -> Result { match self { Self::Timeout => Err(TimeoutError {}), diff --git a/veilid-flutter/example/lib/main.dart b/veilid-flutter/example/lib/main.dart index e635f0dd..568b67e5 100644 --- a/veilid-flutter/example/lib/main.dart +++ b/veilid-flutter/example/lib/main.dart @@ -160,31 +160,31 @@ class _MyAppState extends State with UiLoggy { }); } - Future processUpdateLog(VeilidUpdateLog update) async { + Future processLog(VeilidLog log) async { StackTrace? stackTrace; Object? error; - final backtrace = update.backtrace; + final backtrace = log.backtrace; if (backtrace != null) { stackTrace = StackTrace.fromString("$backtrace\n${StackTrace.current.toString()}"); - error = 'embedded stack trace for ${update.logLevel} ${update.message}'; + error = 'embedded stack trace for ${log.logLevel} ${log.message}'; } - switch (update.logLevel) { + switch (log.logLevel) { case VeilidLogLevel.error: - loggy.error(update.message, error, stackTrace); + loggy.error(log.message, error, stackTrace); break; case VeilidLogLevel.warn: - loggy.warning(update.message, error, stackTrace); + loggy.warning(log.message, error, stackTrace); break; case VeilidLogLevel.info: - loggy.info(update.message, error, stackTrace); + loggy.info(log.message, error, stackTrace); break; case VeilidLogLevel.debug: - loggy.debug(update.message, error, stackTrace); + loggy.debug(log.message, error, stackTrace); break; case VeilidLogLevel.trace: - loggy.trace(update.message, error, stackTrace); + loggy.trace(log.message, error, stackTrace); break; } } @@ -193,8 +193,12 @@ class _MyAppState extends State with UiLoggy { var stream = _updateStream; if (stream != null) { await for (final update in stream) { - if (update is VeilidUpdateLog) { - await processUpdateLog(update); + if (update is VeilidLog) { + await processLog(update); + } else if (update is VeilidAppMessage) { + loggy.info("AppMessage: ${update.json}"); + } else if (update is VeilidAppCall) { + loggy.info("AppCall: ${update.json}"); } else { loggy.trace("Update: ${update.json}"); } diff --git a/veilid-flutter/lib/veilid.dart b/veilid-flutter/lib/veilid.dart index b859eca5..85eb8f8f 100644 --- a/veilid-flutter/lib/veilid.dart +++ b/veilid-flutter/lib/veilid.dart @@ -1232,11 +1232,19 @@ abstract class VeilidUpdate { switch (json["kind"]) { case "Log": { - return VeilidUpdateLog( + return VeilidLog( logLevel: veilidLogLevelFromJson(json["log_level"]), message: json["message"], backtrace: json["backtrace"]); } + case "AppMessage": + { + return VeilidAppMessage(); + } + case "AppCall": + { + return VeilidAppCall(); + } case "Attachment": { return VeilidUpdateAttachment( @@ -1256,12 +1264,12 @@ abstract class VeilidUpdate { Map get json; } -class VeilidUpdateLog implements VeilidUpdate { +class VeilidLog implements VeilidUpdate { final VeilidLogLevel logLevel; final String message; final String? backtrace; // - VeilidUpdateLog({ + VeilidLog({ required this.logLevel, required this.message, required this.backtrace, From 046b61d5d8a684c133fc4f6df2bba7dd546a69ea Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 30 Sep 2022 22:37:55 -0400 Subject: [PATCH 02/67] more app message call --- Cargo.lock | 3 + veilid-cli/Cargo.toml | 1 + veilid-cli/src/client_api_connection.rs | 29 +++++ veilid-cli/src/command_processor.rs | 109 +++++++++++++++++- veilid-core/src/veilid_api/mod.rs | 13 +++ .../src/veilid_api/serialize_helpers.rs | 17 +++ veilid-flutter/lib/veilid.dart | 52 ++++++++- veilid-flutter/lib/veilid_ffi.dart | 18 +++ veilid-flutter/lib/veilid_js.dart | 8 ++ veilid-flutter/rust/Cargo.toml | 1 + veilid-flutter/rust/src/dart_ffi.rs | 20 ++++ veilid-server/proto/veilid-client.capnp | 9 +- veilid-server/src/client_api.rs | 19 +++ veilid-wasm/Cargo.toml | 1 + veilid-wasm/src/lib.rs | 18 +++ 15 files changed, 310 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c1f5cbc..66988967 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5100,6 +5100,7 @@ dependencies = [ "directories", "flexi_logger", "futures", + "hex", "log", "parking_lot 0.12.1", "serde", @@ -5218,6 +5219,7 @@ dependencies = [ "async-std", "backtrace", "cfg-if 1.0.0", + "data-encoding", "ffi-support", "futures-util", "hostname", @@ -5292,6 +5294,7 @@ version = "0.1.0" dependencies = [ "cfg-if 1.0.0", "console_error_panic_hook", + "data-encoding", "futures-util", "js-sys", "lazy_static", diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index bc2e5e7a..eb4cc69e 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -42,6 +42,7 @@ bugsalot = "^0" flexi_logger = { version = "^0", features = ["use_chrono_for_offset"] } thiserror = "^1" crossbeam-channel = "^0" +hex = "^0" veilid-core = { path = "../veilid-core", default_features = false} [dev-dependencies] diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index 22828531..682269c3 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -76,6 +76,12 @@ impl veilid_client::Server for VeilidClientImpl { VeilidUpdate::Log(log) => { self.comproc.update_log(log); } + VeilidUpdate::AppMessage(msg) => { + self.comproc.update_app_message(msg); + } + VeilidUpdate::AppCall(call) => { + self.comproc.update_app_call(call); + } VeilidUpdate::Attachment(attachment) => { self.comproc.update_attachment(attachment); } @@ -365,6 +371,29 @@ impl ClientApiConnection { res.map_err(map_to_string) } + pub async fn server_appcall_reply(&mut self, id: u64, msg: Vec) -> Result<(), String> { + trace!("ClientApiConnection::appcall_reply"); + let server = { + let inner = self.inner.borrow(); + inner + .server + .as_ref() + .ok_or_else(|| "Not connected, ignoring change_log_level request".to_owned())? + .clone() + }; + let mut request = server.borrow().app_call_reply_request(); + request.get().set_id(id); + request.get().set_message(&msg); + let response = request.send().promise.await.map_err(map_to_string)?; + let reader = response + .get() + .map_err(map_to_string)? + .get_result() + .map_err(map_to_string)?; + let res: Result<(), VeilidAPIError> = decode_api_result(&reader); + res.map_err(map_to_string) + } + // Start Client API connection pub async fn connect(&mut self, connect_addr: SocketAddr) -> Result<(), String> { trace!("ClientApiConnection::connect"); diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index 1e6d4ad6..a9c3877e 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -49,6 +49,7 @@ struct CommandProcessorInner { autoreconnect: bool, server_addr: Option, connection_waker: Eventual, + last_call_id: Option, } type Handle = Rc>; @@ -70,6 +71,7 @@ impl CommandProcessor { autoreconnect: settings.autoreconnect, server_addr: None, connection_waker: Eventual::new(), + last_call_id: None, })), } } @@ -111,6 +113,7 @@ attach - attach the server to the Veilid network detach - detach the server from the Veilid network debug - send a debugging command to the Veilid server change_log_level - change the log level for a tracing layer +reply - reply to an AppCall not handled directly by the server "# .to_owned(), ); @@ -225,6 +228,66 @@ change_log_level - change the log level for a tracing layer Ok(()) } + pub fn cmd_reply(&self, rest: Option, callback: UICallback) -> Result<(), String> { + trace!("CommandProcessor::cmd_reply"); + + let mut capi = self.capi(); + let ui = self.ui(); + let some_last_id = self.inner_mut().last_call_id.take(); + spawn_detached_local(async move { + let (first, second) = Self::word_split(&rest.clone().unwrap_or_default()); + let (id, msg) = if let Some(second) = second { + let id = match u64::from_str(&first) { + Err(e) => { + ui.add_node_event(format!("invalid appcall id: {}", e)); + ui.send_callback(callback); + return; + } + Ok(v) => v, + }; + (id, second) + } else { + let id = match some_last_id { + None => { + ui.add_node_event("must specify last call id".to_owned()); + ui.send_callback(callback); + return; + } + Some(v) => v, + }; + (id, rest.unwrap_or_default()) + }; + let msg = if msg[0..1] == "#".to_owned() { + match hex::decode(msg[1..].as_bytes().to_vec()) { + Err(e) => { + ui.add_node_event(format!("invalid hex message: {}", e)); + ui.send_callback(callback); + return; + } + Ok(v) => v, + } + } else { + msg[1..].as_bytes().to_vec() + }; + let msglen = msg.len(); + match capi.server_appcall_reply(id, msg).await { + Ok(()) => { + ui.add_node_event(format!("reply sent to {} : {} bytes", id, msglen)); + ui.send_callback(callback); + return; + } + Err(e) => { + ui.display_string_dialog( + "Server command 'appcall_reply' failed", + e.to_string(), + callback, + ); + } + } + }); + Ok(()) + } + pub fn run_command(&self, command_line: &str, callback: UICallback) -> Result<(), String> { // let (cmd, rest) = Self::word_split(command_line); @@ -238,6 +301,7 @@ change_log_level - change the log level for a tracing layer "detach" => self.cmd_detach(callback), "debug" => self.cmd_debug(rest, callback), "change_log_level" => self.cmd_change_log_level(rest, callback), + "reply" => self.cmd_reply(rest, callback), _ => { let ui = self.ui(); ui.send_callback(callback); @@ -344,6 +408,49 @@ change_log_level - change the log level for a tracing layer )); } + pub fn update_app_message(&mut self, msg: veilid_core::VeilidAppMessage) { + // check is message body is ascii printable + let mut printable = true; + for c in &msg.message { + if *c < 32 || *c > 126 { + printable = false; + } + } + + let strmsg = if printable { + String::from_utf8_lossy(&msg.message).to_string() + } else { + hex::encode(&msg.message) + }; + + self.inner() + .ui + .add_node_event(format!("AppMessage ({:?}): {}", msg.sender, strmsg)); + } + + pub fn update_app_call(&mut self, call: veilid_core::VeilidAppCall) { + // check is message body is ascii printable + let mut printable = true; + for c in &call.message { + if *c < 32 || *c > 126 { + printable = false; + } + } + + let strmsg = if printable { + String::from_utf8_lossy(&call.message).to_string() + } else { + format!("#{}", hex::encode(&call.message)) + }; + + self.inner().ui.add_node_event(format!( + "AppCall ({:?}) id = {:016x} : {}", + call.sender, call.id, strmsg + )); + + self.inner_mut().last_call_id = Some(call.id); + } + pub fn update_shutdown(&mut self) { // Do nothing with this, we'll process shutdown when rpc connection closes } @@ -381,7 +488,6 @@ change_log_level - change the log level for a tracing layer // calls into client_api_connection //////////////////////////////////////////// pub fn attach(&mut self) { - trace!("CommandProcessor::attach"); let mut capi = self.capi(); spawn_detached_local(async move { @@ -392,7 +498,6 @@ change_log_level - change the log level for a tracing layer } pub fn detach(&mut self) { - trace!("CommandProcessor::detach"); let mut capi = self.capi(); spawn_detached_local(async move { diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 50835a13..0934ff5c 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -230,18 +230,23 @@ pub struct VeilidLog { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct VeilidAppMessage { /// Some(sender) if the message was sent directly, None if received via a private/safety route + #[serde(with = "opt_json_as_string")] pub sender: Option, /// The content of the message to deliver to the application + #[serde(with = "json_as_base64")] pub message: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct VeilidAppCall { /// Some(sender) if the request was sent directly, None if received via a private/safety route + #[serde(with = "opt_json_as_string")] pub sender: Option, /// The content of the request to deliver to the application + #[serde(with = "json_as_base64")] pub message: Vec, /// The id to reply to + #[serde(with = "json_as_string")] pub id: u64, } @@ -301,6 +306,14 @@ impl fmt::Display for NodeId { write!(f, "{}", self.key.encode()) } } +impl FromStr for NodeId { + type Err = VeilidAPIError; + fn from_str(s: &str) -> Result { + Ok(Self { + key: DHTKey::try_decode(s)?, + }) + } +} #[derive(Clone, Debug, Default, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] pub struct ValueKey { diff --git a/veilid-core/src/veilid_api/serialize_helpers.rs b/veilid-core/src/veilid_api/serialize_helpers.rs index f2d6d28d..58b51909 100644 --- a/veilid-core/src/veilid_api/serialize_helpers.rs +++ b/veilid-core/src/veilid_api/serialize_helpers.rs @@ -41,6 +41,23 @@ pub fn serialize_json(val: T) -> String { } } +pub mod json_as_base64 { + use data_encoding::BASE64URL_NOPAD; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(v: &Vec, s: S) -> Result { + let base64 = BASE64URL_NOPAD.encode(v); + String::serialize(&base64, s) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let base64 = String::deserialize(d)?; + BASE64URL_NOPAD + .decode(base64.as_bytes()) + .map_err(|e| serde::de::Error::custom(e)) + } +} + pub mod json_as_string { use std::fmt::Display; use std::str::FromStr; diff --git a/veilid-flutter/lib/veilid.dart b/veilid-flutter/lib/veilid.dart index 85eb8f8f..843e2938 100644 --- a/veilid-flutter/lib/veilid.dart +++ b/veilid-flutter/lib/veilid.dart @@ -1,4 +1,6 @@ import 'dart:async'; +import 'dart:typed_data'; +import 'dart:convert'; import 'package:change_case/change_case.dart'; @@ -1239,11 +1241,13 @@ abstract class VeilidUpdate { } case "AppMessage": { - return VeilidAppMessage(); + return VeilidAppMessage( + sender: json["sender"], message: json["message"]); } case "AppCall": { - return VeilidAppCall(); + return VeilidAppCall( + sender: json["sender"], message: json["message"], id: json["id"]); } case "Attachment": { @@ -1286,6 +1290,49 @@ class VeilidLog implements VeilidUpdate { } } +class VeilidAppMessage implements VeilidUpdate { + final String? sender; + final Uint8List message; + + // + VeilidAppMessage({ + required this.sender, + required this.message, + }); + + @override + Map get json { + return { + 'kind': "AppMessage", + 'sender': sender, + 'message': base64UrlEncode(message) + }; + } +} + +class VeilidAppCall implements VeilidUpdate { + final String? sender; + final Uint8List message; + final String id; + + // + VeilidAppCall({ + required this.sender, + required this.message, + required this.id, + }); + + @override + Map get json { + return { + 'kind': "AppMessage", + 'sender': sender, + 'message': base64UrlEncode(message), + 'id': id, + }; + } +} + class VeilidUpdateAttachment implements VeilidUpdate { final VeilidStateAttachment state; // @@ -1580,6 +1627,7 @@ abstract class Veilid { Future detach(); Future shutdownVeilidCore(); Future debug(String command); + Future appCallReply(String id, Uint8List message); String veilidVersionString(); VeilidVersion veilidVersion(); } diff --git a/veilid-flutter/lib/veilid_ffi.dart b/veilid-flutter/lib/veilid_ffi.dart index 4b41bd83..23ce9ef6 100644 --- a/veilid-flutter/lib/veilid_ffi.dart +++ b/veilid-flutter/lib/veilid_ffi.dart @@ -3,6 +3,7 @@ import 'dart:ffi'; import 'dart:io'; import 'dart:isolate'; import 'dart:convert'; +import 'dart:typed_data'; import 'package:ffi/ffi.dart'; @@ -50,6 +51,10 @@ typedef _DetachDart = void Function(int); // fn debug(port: i64, log_level: FfiStr) typedef _DebugC = Void Function(Int64, Pointer); typedef _DebugDart = void Function(int, Pointer); +// fn app_call_reply(port: i64, id: FfiStr, message: FfiStr) +typedef _AppCallReplyC = Void Function(Int64, Pointer, Pointer); +typedef _AppCallReplyDart = void Function(int, Pointer, Pointer); + // fn shutdown_veilid_core(port: i64) typedef _ShutdownVeilidCoreC = Void Function(Int64); typedef _ShutdownVeilidCoreDart = void Function(int); @@ -304,6 +309,7 @@ class VeilidFFI implements Veilid { final _DetachDart _detach; final _ShutdownVeilidCoreDart _shutdownVeilidCore; final _DebugDart _debug; + final _AppCallReplyDart _appCallReply; final _VeilidVersionStringDart _veilidVersionString; final _VeilidVersionDart _veilidVersion; @@ -328,6 +334,8 @@ class VeilidFFI implements Veilid { dylib.lookupFunction<_ShutdownVeilidCoreC, _ShutdownVeilidCoreDart>( 'shutdown_veilid_core'), _debug = dylib.lookupFunction<_DebugC, _DebugDart>('debug'), + _appCallReply = dylib.lookupFunction<_AppCallReplyC, _AppCallReplyDart>( + 'app_call_reply'), _veilidVersionString = dylib.lookupFunction<_VeilidVersionStringC, _VeilidVersionStringDart>('veilid_version_string'), _veilidVersion = @@ -420,6 +428,16 @@ class VeilidFFI implements Veilid { return processFuturePlain(recvPort.first); } + @override + Future appCallReply(String id, Uint8List message) async { + var nativeId = id.toNativeUtf8(); + var nativeEncodedMessage = base64UrlEncode(message).toNativeUtf8(); + final recvPort = ReceivePort("app_call_reply"); + final sendPort = recvPort.sendPort; + _appCallReply(sendPort.nativePort, nativeId, nativeEncodedMessage); + return processFutureVoid(recvPort.first); + } + @override String veilidVersionString() { final versionString = _veilidVersionString(); diff --git a/veilid-flutter/lib/veilid_js.dart b/veilid-flutter/lib/veilid_js.dart index ba9a4dc1..1d5e907c 100644 --- a/veilid-flutter/lib/veilid_js.dart +++ b/veilid-flutter/lib/veilid_js.dart @@ -5,6 +5,7 @@ import 'dart:js' as js; import 'dart:js_util' as js_util; import 'dart:async'; import 'dart:convert'; +import 'dart:typed_data'; ////////////////////////////////////////////////////////// @@ -82,6 +83,13 @@ class VeilidJS implements Veilid { return _wrapApiPromise(js_util.callMethod(wasm, "debug", [command])); } + @override + Future appCallReply(String id, Uint8List message) { + var encodedMessage = base64UrlEncode(message); + return _wrapApiPromise( + js_util.callMethod(wasm, "app_call_reply", [id, encodedMessage])); + } + @override String veilidVersionString() { return js_util.callMethod(wasm, "veilid_version_string", []); diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index deeea4d6..1967f5e1 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -20,6 +20,7 @@ serde_json = "^1" serde = "^1" futures-util = { version = "^0", default_features = false, features = ["alloc"] } cfg-if = "^1" +data-encoding = { version = "^2" } # Dependencies for native builds only # Linux, Windows, Mac, iOS, Android diff --git a/veilid-flutter/rust/src/dart_ffi.rs b/veilid-flutter/rust/src/dart_ffi.rs index 3048277f..00080298 100644 --- a/veilid-flutter/rust/src/dart_ffi.rs +++ b/veilid-flutter/rust/src/dart_ffi.rs @@ -327,6 +327,26 @@ pub extern "C" fn debug(port: i64, command: FfiStr) { }); } +#[no_mangle] +pub extern "C" fn app_call_reply(port: i64, id: FfiStr, message: FfiStr) { + let id = id.into_opt_string().unwrap_or_default(); + let message = message.into_opt_string().unwrap_or_default(); + DartIsolateWrapper::new(port).spawn_result(async move { + let id = match id.parse() { + Ok(v) => v, + Err(e) => { + return APIResult::Err(veilid_core::VeilidAPIError::invalid_argument(e, "id", id)) + } + }; + let message = data_encoding::BASE64URL_NOPAD + .decode(message.as_bytes()) + .map_err(|e| veilid_core::VeilidAPIError::invalid_argument(e, "message", message))?; + let veilid_api = get_veilid_api().await?; + veilid_api.app_call_reply(id, message).await?; + APIRESULT_VOID + }); +} + #[no_mangle] pub extern "C" fn veilid_version_string() -> *mut c_char { veilid_core::veilid_version_string().into_ffi_value() diff --git a/veilid-server/proto/veilid-client.capnp b/veilid-server/proto/veilid-client.capnp index 1d4066b1..320ffea4 100644 --- a/veilid-server/proto/veilid-client.capnp +++ b/veilid-server/proto/veilid-client.capnp @@ -10,15 +10,16 @@ struct ApiResult { interface Registration {} interface VeilidServer { - register @0 (veilidClient: VeilidClient) -> (registration: Registration, state: Text); - debug @1 (command: Text) -> (result :ApiResult); + register @0 (veilidClient :VeilidClient) -> (registration :Registration, state :Text); + debug @1 (command :Text) -> (result :ApiResult); attach @2 () -> (result :ApiResult); detach @3 () -> (result :ApiResult); shutdown @4 (); getState @5 () -> (result :ApiResult); - changeLogLevel @6 (layer: Text, logLevel: Text) -> (result :ApiResult); + changeLogLevel @6 (layer :Text, logLevel :Text) -> (result :ApiResult); + appCallReply @7 (id :UInt64, message :Data) -> (result :ApiResult); } interface VeilidClient { - update @0 (veilidUpdate: Text); + update @0 (veilidUpdate :Text); } \ No newline at end of file diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index cafe89cb..6207cb5f 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -236,6 +236,25 @@ impl veilid_server::Server for VeilidServerImpl { encode_api_result(&result, &mut results.get().init_result()); Promise::ok(()) } + + #[instrument(level = "trace", skip_all)] + fn app_call_reply( + &mut self, + params: veilid_server::AppCallReplyParams, + mut results: veilid_server::AppCallReplyResults, + ) -> Promise<(), ::capnp::Error> { + trace!("VeilidServerImpl::app_call_reply"); + + let id = pry!(params.get()).get_id(); + let message = pry!(pry!(params.get()).get_message()).to_owned(); + + let veilid_api = self.veilid_api.clone(); + Promise::from_future(async move { + let result = veilid_api.app_call_reply(id, message).await; + encode_api_result(&result, &mut results.get().init_result()); + Ok(()) + }) + } } // --- Client API Server-Side --------------------------------- diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index c5cd2426..7ca4e42f 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -26,6 +26,7 @@ serde = "^1" lazy_static = "^1" send_wrapper = "^0" futures-util = { version = "^0", default_features = false, features = ["alloc"] } +data-encoding = { version = "^2" } [dev-dependencies] wasm-bindgen-test = "^0" diff --git a/veilid-wasm/src/lib.rs b/veilid-wasm/src/lib.rs index 9f0e1b4b..6baf1e80 100644 --- a/veilid-wasm/src/lib.rs +++ b/veilid-wasm/src/lib.rs @@ -268,6 +268,24 @@ pub fn debug(command: String) -> Promise { }) } +#[wasm_bindgen()] +pub fn app_call_reply(id: String, message: String) -> Promise { + wrap_api_future(async move { + let id = match id.parse() { + Ok(v) => v, + Err(e) => { + return APIResult::Err(veilid_core::VeilidAPIError::invalid_argument(e, "id", id)) + } + }; + let message = data_encoding::BASE64URL_NOPAD + .decode(message.as_bytes()) + .map_err(|e| veilid_core::VeilidAPIError::invalid_argument(e, "message", message))?; + let veilid_api = get_veilid_api()?; + let out = veilid_api.app_call_reply(id, message).await?; + Ok(out) + }) +} + #[wasm_bindgen()] pub fn veilid_version_string() -> String { veilid_core::veilid_version_string() From 0a01c0d23e5761439d2ffdd82d16b15e4595b223 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 2 Oct 2022 18:47:36 -0400 Subject: [PATCH 03/67] debugging, add async_tag_lock --- .../src/network_manager/connection_handle.rs | 18 +- .../src/network_manager/connection_manager.rs | 36 +-- .../src/network_manager/connection_table.rs | 10 +- veilid-core/src/network_manager/mod.rs | 152 ++++++----- .../src/network_manager/native/network_tcp.rs | 44 ++- .../native/protocol/sockets.rs | 32 ++- .../network_manager/native/protocol/tcp.rs | 21 +- .../src/network_manager/native/protocol/ws.rs | 15 +- .../network_manager/native/start_protocols.rs | 6 +- .../src/network_manager/network_connection.rs | 31 ++- veilid-core/src/network_manager/tasks.rs | 252 ++++++++++-------- .../src/network_manager/wasm/protocol/ws.rs | 2 +- veilid-core/src/rpc_processor/mod.rs | 14 +- veilid-core/src/tests/common/mod.rs | 1 + .../src/tests/common/test_async_tag_lock.rs | 159 +++++++++++ veilid-core/src/tests/native/mod.rs | 14 + veilid-core/src/veilid_api/mod.rs | 11 + veilid-core/src/xx/async_tag_lock.rs | 126 +++++++++ veilid-core/src/xx/mod.rs | 5 + veilid-core/tests/node.rs | 7 + veilid-core/tests/web.rs | 7 + 21 files changed, 690 insertions(+), 273 deletions(-) create mode 100644 veilid-core/src/tests/common/test_async_tag_lock.rs create mode 100644 veilid-core/src/xx/async_tag_lock.rs diff --git a/veilid-core/src/network_manager/connection_handle.rs b/veilid-core/src/network_manager/connection_handle.rs index aa37070a..9eeb1b63 100644 --- a/veilid-core/src/network_manager/connection_handle.rs +++ b/veilid-core/src/network_manager/connection_handle.rs @@ -4,7 +4,7 @@ use super::*; pub struct ConnectionHandle { id: u64, descriptor: ConnectionDescriptor, - channel: flume::Sender>, + channel: flume::Sender<(Option, Vec)>, } #[derive(Debug)] @@ -17,7 +17,7 @@ impl ConnectionHandle { pub(super) fn new( id: u64, descriptor: ConnectionDescriptor, - channel: flume::Sender>, + channel: flume::Sender<(Option, Vec)>, ) -> Self { Self { id, @@ -34,16 +34,22 @@ impl ConnectionHandle { self.descriptor.clone() } + #[instrument(level="trace", skip(self, message), fields(message.len = message.len()))] pub fn send(&self, message: Vec) -> ConnectionHandleSendResult { - match self.channel.send(message) { + match self.channel.send((Span::current().id(), message)) { Ok(()) => ConnectionHandleSendResult::Sent, - Err(e) => ConnectionHandleSendResult::NotSent(e.0), + Err(e) => ConnectionHandleSendResult::NotSent(e.0 .1), } } + #[instrument(level="trace", skip(self, message), fields(message.len = message.len()))] pub async fn send_async(&self, message: Vec) -> ConnectionHandleSendResult { - match self.channel.send_async(message).await { + match self + .channel + .send_async((Span::current().id(), message)) + .await + { Ok(()) => ConnectionHandleSendResult::Sent, - Err(e) => ConnectionHandleSendResult::NotSent(e.0), + Err(e) => ConnectionHandleSendResult::NotSent(e.0 .1), } } } diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index 7686425f..2bceb171 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -140,6 +140,7 @@ impl ConnectionManager { // Internal routine to register new connection atomically. // Registers connection in the connection table for later access // and spawns a message processing loop for the connection + #[instrument(level = "trace", skip(self, inner), ret, err)] fn on_new_protocol_network_connection( &self, inner: &mut ConnectionManagerInner, @@ -195,6 +196,7 @@ impl ConnectionManager { } // Returns a network connection if one already is established + #[instrument(level = "trace", skip(self), ret)] pub fn get_connection(&self, descriptor: ConnectionDescriptor) -> Option { self.arc .connection_table @@ -234,40 +236,43 @@ impl ConnectionManager { did_kill } - // Called when we want to create a new connection or get the current one that already exists - // This will kill off any connections that are in conflict with the new connection to be made - // in order to make room for the new connection in the system's connection table - // This routine needs to be atomic, or connections may exist in the table that are not established + /// Locak remote address + // async fn lock_remote_address(&self, remote_addr: SocketAddr) -> { + + // } + + /// Called when we want to create a new connection or get the current one that already exists + /// This will kill off any connections that are in conflict with the new connection to be made + /// in order to make room for the new connection in the system's connection table + /// This routine needs to be atomic, or connections may exist in the table that are not established + #[instrument(level = "trace", skip(self), ret, err)] pub async fn get_or_create_connection( &self, local_addr: Option, dial_info: DialInfo, ) -> EyreResult> { - log_net!( + warn!( "== get_or_create_connection local_addr={:?} dial_info={:?}", local_addr.green(), dial_info.green() ); + // Make a connection descriptor for this dialinfo + let peer_address = dial_info.to_peer_address(); + + // Async lock on the remote address for atomicity + //let _lock_guard = self.lock_remote_address(peer_address.to_socket_addr()); + // Kill off any possibly conflicting connections let did_kill = self.kill_off_colliding_connections(&dial_info).await; let mut retry_count = if did_kill { 2 } else { 0 }; - // Make a connection descriptor for this dialinfo - let peer_address = dial_info.to_peer_address(); - let descriptor = match local_addr { - Some(la) => { - ConnectionDescriptor::new(peer_address, SocketAddress::from_socket_addr(la)) - } - None => ConnectionDescriptor::new_no_local(peer_address), - }; - // If any connection to this remote exists that has the same protocol, return it // Any connection will do, we don't have to match the local address if let Some(conn) = self .arc .connection_table - .get_last_connection_by_remote(descriptor.remote()) + .get_last_connection_by_remote(peer_address) { log_net!( "== Returning existing connection local_addr={:?} peer_address={:?}", @@ -378,6 +383,7 @@ impl ConnectionManager { // Callback from network connection receive loop when it exits // cleans up the entry in the connection table + #[instrument(level = "trace", skip(self))] pub(super) async fn report_connection_finished(&self, connection_id: u64) { // Get channel sender let sender = { diff --git a/veilid-core/src/network_manager/connection_table.rs b/veilid-core/src/network_manager/connection_table.rs index 3ed4d486..c16214f3 100644 --- a/veilid-core/src/network_manager/connection_table.rs +++ b/veilid-core/src/network_manager/connection_table.rs @@ -72,6 +72,7 @@ impl ConnectionTable { } } + #[instrument(level = "trace", skip(self))] pub async fn join(&self) { let mut unord = { let mut inner = self.inner.lock(); @@ -90,6 +91,7 @@ impl ConnectionTable { while unord.next().await.is_some() {} } + #[instrument(level = "trace", skip(self), ret, err)] pub fn add_connection( &self, network_connection: NetworkConnection, @@ -156,6 +158,7 @@ impl ConnectionTable { Ok(out_conn) } + #[instrument(level = "trace", skip(self), ret)] pub fn get_connection_by_id(&self, id: NetworkConnectionId) -> Option { let mut inner = self.inner.lock(); let protocol_index = *inner.protocol_index_by_id.get(&id)?; @@ -163,6 +166,7 @@ impl ConnectionTable { Some(out.get_handle()) } + #[instrument(level = "trace", skip(self), ret)] pub fn get_connection_by_descriptor( &self, descriptor: ConnectionDescriptor, @@ -175,6 +179,7 @@ impl ConnectionTable { Some(out.get_handle()) } + #[instrument(level = "trace", skip(self), ret)] pub fn get_last_connection_by_remote(&self, remote: PeerAddress) -> Option { let mut inner = self.inner.lock(); @@ -184,7 +189,8 @@ impl ConnectionTable { Some(out.get_handle()) } - pub fn _get_connection_ids_by_remote(&self, remote: PeerAddress) -> Vec { + #[instrument(level = "trace", skip(self), ret)] + pub fn get_connection_ids_by_remote(&self, remote: PeerAddress) -> Vec { let inner = self.inner.lock(); inner .ids_by_remote @@ -219,6 +225,7 @@ impl ConnectionTable { inner.conn_by_id.iter().fold(0, |acc, c| acc + c.len()) } + #[instrument(level = "trace", skip(inner), ret)] fn remove_connection_records( inner: &mut ConnectionTableInner, id: NetworkConnectionId, @@ -251,6 +258,7 @@ impl ConnectionTable { conn } + #[instrument(level = "trace", skip(self), ret)] pub fn remove_connection_by_id(&self, id: NetworkConnectionId) -> Option { let mut inner = self.inner.lock(); diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 3fe83c63..9b2d9f18 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1321,88 +1321,93 @@ impl NetworkManager { data: Vec, ) -> SendPinBoxFuture>> { let this = self.clone(); - Box::pin(async move { - // info!("{}", format!("send_data to: {:?}", node_ref).red()); + Box::pin( + async move { + // info!("{}", format!("send_data to: {:?}", node_ref).red()); - // First try to send data to the last socket we've seen this peer on - let data = if let Some(connection_descriptor) = node_ref.last_connection() { - // info!( - // "{}", - // format!("last_connection to: {:?}", connection_descriptor).red() - // ); + // First try to send data to the last socket we've seen this peer on + let data = if let Some(connection_descriptor) = node_ref.last_connection() { + // info!( + // "{}", + // format!("last_connection to: {:?}", connection_descriptor).red() + // ); - match this - .net() - .send_data_to_existing_connection(connection_descriptor, data) - .await? - { - None => { - // info!( - // "{}", - // format!("sent to existing connection: {:?}", connection_descriptor) - // .red() - // ); + match this + .net() + .send_data_to_existing_connection(connection_descriptor, data) + .await? + { + None => { + // info!( + // "{}", + // format!("sent to existing connection: {:?}", connection_descriptor) + // .red() + // ); - // Update timestamp for this last connection since we just sent to it + // Update timestamp for this last connection since we just sent to it + node_ref + .set_last_connection(connection_descriptor, intf::get_timestamp()); + + return Ok(NetworkResult::value(SendDataKind::Existing( + connection_descriptor, + ))); + } + Some(d) => d, + } + } else { + data + }; + + // info!("{}", "no existing connection".red()); + + // If we don't have last_connection, try to reach out to the peer via its dial info + let contact_method = this.get_contact_method(node_ref.clone()); + log_net!( + "send_data via {:?} to dialinfo {:?}", + contact_method, + node_ref + ); + match contact_method { + ContactMethod::OutboundRelay(relay_nr) + | ContactMethod::InboundRelay(relay_nr) => { + network_result_try!(this.send_data(relay_nr, data).await?); + Ok(NetworkResult::value(SendDataKind::Indirect)) + } + ContactMethod::Direct(dial_info) => { + let connection_descriptor = network_result_try!( + this.net().send_data_to_dial_info(dial_info, data).await? + ); + // If we connected to this node directly, save off the last connection so we can use it again node_ref.set_last_connection(connection_descriptor, intf::get_timestamp()); - return Ok(NetworkResult::value(SendDataKind::Existing( + Ok(NetworkResult::value(SendDataKind::Direct( connection_descriptor, - ))); + ))) } - Some(d) => d, + ContactMethod::SignalReverse(relay_nr, target_node_ref) => { + let connection_descriptor = network_result_try!( + this.do_reverse_connect(relay_nr, target_node_ref, data) + .await? + ); + Ok(NetworkResult::value(SendDataKind::Direct( + connection_descriptor, + ))) + } + ContactMethod::SignalHolePunch(relay_nr, target_node_ref) => { + let connection_descriptor = network_result_try!( + this.do_hole_punch(relay_nr, target_node_ref, data).await? + ); + Ok(NetworkResult::value(SendDataKind::Direct( + connection_descriptor, + ))) + } + ContactMethod::Unreachable => Ok(NetworkResult::no_connection_other( + "Can't send to this node", + )), } - } else { - data - }; - - // info!("{}", "no existing connection".red()); - - // If we don't have last_connection, try to reach out to the peer via its dial info - let contact_method = this.get_contact_method(node_ref.clone()); - log_net!( - "send_data via {:?} to dialinfo {:?}", - contact_method, - node_ref - ); - match contact_method { - ContactMethod::OutboundRelay(relay_nr) | ContactMethod::InboundRelay(relay_nr) => { - network_result_try!(this.send_data(relay_nr, data).await?); - Ok(NetworkResult::value(SendDataKind::Indirect)) - } - ContactMethod::Direct(dial_info) => { - let connection_descriptor = network_result_try!( - this.net().send_data_to_dial_info(dial_info, data).await? - ); - // If we connected to this node directly, save off the last connection so we can use it again - node_ref.set_last_connection(connection_descriptor, intf::get_timestamp()); - - Ok(NetworkResult::value(SendDataKind::Direct( - connection_descriptor, - ))) - } - ContactMethod::SignalReverse(relay_nr, target_node_ref) => { - let connection_descriptor = network_result_try!( - this.do_reverse_connect(relay_nr, target_node_ref, data) - .await? - ); - Ok(NetworkResult::value(SendDataKind::Direct( - connection_descriptor, - ))) - } - ContactMethod::SignalHolePunch(relay_nr, target_node_ref) => { - let connection_descriptor = network_result_try!( - this.do_hole_punch(relay_nr, target_node_ref, data).await? - ); - Ok(NetworkResult::value(SendDataKind::Direct( - connection_descriptor, - ))) - } - ContactMethod::Unreachable => Ok(NetworkResult::no_connection_other( - "Can't send to this node", - )), } - }) + .instrument(trace_span!("send_data")), + ) } // Direct bootstrap request handler (separate fallback mechanism from cheaper TXT bootstrap mechanism) @@ -1466,6 +1471,7 @@ impl NetworkManager { // Called when a packet potentially containing an RPC envelope is received by a low-level // network protocol handler. Processes the envelope, authenticates and decrypts the RPC message // and passes it to the RPC handler + #[instrument(level = "trace", ret, err, skip(self, data), fields(data.len = data.len()))] async fn on_recv_envelope( &self, data: &[u8], diff --git a/veilid-core/src/network_manager/native/network_tcp.rs b/veilid-core/src/network_manager/native/network_tcp.rs index ca2e3792..806abc2c 100644 --- a/veilid-core/src/network_manager/native/network_tcp.rs +++ b/veilid-core/src/network_manager/native/network_tcp.rs @@ -42,7 +42,8 @@ impl Network { &self, tls_acceptor: &TlsAcceptor, stream: AsyncPeekStream, - addr: SocketAddr, + peer_addr: SocketAddr, + local_addr: SocketAddr, protocol_handlers: &[Box], tls_connection_initial_timeout_ms: u32, ) -> EyreResult> { @@ -65,18 +66,20 @@ impl Network { .wrap_err("tls initial timeout")? .wrap_err("failed to peek tls stream")?; - self.try_handlers(ps, addr, protocol_handlers).await + self.try_handlers(ps, peer_addr, local_addr, protocol_handlers) + .await } async fn try_handlers( &self, stream: AsyncPeekStream, - addr: SocketAddr, + peer_addr: SocketAddr, + local_addr: SocketAddr, protocol_accept_handlers: &[Box], ) -> EyreResult> { for ah in protocol_accept_handlers.iter() { if let Some(nc) = ah - .on_accept(stream.clone(), addr) + .on_accept(stream.clone(), peer_addr, local_addr) .await .wrap_err("io error")? { @@ -105,21 +108,35 @@ impl Network { } }; + // XXX + warn!( + "DEBUGACCEPT: local={} remote={}", + tcp_stream.local_addr().unwrap(), + tcp_stream.peer_addr().unwrap(), + ); + let listener_state = listener_state.clone(); let connection_manager = connection_manager.clone(); // Limit the number of connections from the same IP address // and the number of total connections - let addr = match tcp_stream.peer_addr() { + let peer_addr = match tcp_stream.peer_addr() { Ok(addr) => addr, Err(e) => { log_net!(debug "failed to get peer address: {}", e); return; } }; + let local_addr = match tcp_stream.local_addr() { + Ok(addr) => addr, + Err(e) => { + log_net!(debug "failed to get local address: {}", e); + return; + } + }; // XXX limiting here instead for connection table? may be faster and avoids tls negotiation - log_net!("TCP connection from: {}", addr); + log_net!("TCP connection from: {}", peer_addr); // Create a stream we can peek on #[cfg(feature = "rt-tokio")] @@ -139,7 +156,7 @@ impl Network { { // If we fail to get a packet within the connection initial timeout // then we punt this connection - log_net!("connection initial timeout from: {:?}", addr); + log_net!("connection initial timeout from: {:?}", peer_addr); return; } @@ -152,29 +169,30 @@ impl Network { self.try_tls_handlers( ls.tls_acceptor.as_ref().unwrap(), ps, - addr, + peer_addr, + local_addr, &ls.tls_protocol_handlers, tls_connection_initial_timeout_ms, ) .await } else { - self.try_handlers(ps, addr, &ls.protocol_accept_handlers) + self.try_handlers(ps, peer_addr, local_addr, &ls.protocol_accept_handlers) .await }; let conn = match conn { Ok(Some(c)) => { - log_net!("protocol handler found for {:?}: {:?}", addr, c); + log_net!("protocol handler found for {:?}: {:?}", peer_addr, c); c } Ok(None) => { // No protocol handlers matched? drop it. - log_net!(debug "no protocol handler for connection from {:?}", addr); + log_net!(debug "no protocol handler for connection from {:?}", peer_addr); return; } Err(e) => { // Failed to negotiate connection? drop it. - log_net!(debug "failed to negotiate connection from {:?}: {}", addr, e); + log_net!(debug "failed to negotiate connection from {:?}: {}", peer_addr, e); return; } }; @@ -311,7 +329,6 @@ impl Network { .push(new_protocol_accept_handler( self.network_manager().config(), true, - addr, )); } else { ls.write() @@ -319,7 +336,6 @@ impl Network { .push(new_protocol_accept_handler( self.network_manager().config(), false, - addr, )); } diff --git a/veilid-core/src/network_manager/native/protocol/sockets.rs b/veilid-core/src/network_manager/native/protocol/sockets.rs index dd23bf45..0650d686 100644 --- a/veilid-core/src/network_manager/native/protocol/sockets.rs +++ b/veilid-core/src/network_manager/native/protocol/sockets.rs @@ -34,6 +34,7 @@ cfg_if! { } } +#[instrument(level = "trace", ret, err)] pub fn new_unbound_shared_udp_socket(domain: Domain) -> io::Result { let socket = Socket::new(domain, Type::DGRAM, Some(Protocol::UDP))?; if domain == Domain::IPV6 { @@ -49,6 +50,7 @@ pub fn new_unbound_shared_udp_socket(domain: Domain) -> io::Result { Ok(socket) } +#[instrument(level = "trace", ret, err)] pub fn new_bound_shared_udp_socket(local_address: SocketAddr) -> io::Result { let domain = Domain::for_address(local_address); let socket = new_unbound_shared_udp_socket(domain)?; @@ -60,6 +62,7 @@ pub fn new_bound_shared_udp_socket(local_address: SocketAddr) -> io::Result io::Result { let domain = Domain::for_address(local_address); let socket = Socket::new(domain, Type::DGRAM, Some(Protocol::UDP))?; @@ -93,6 +96,7 @@ pub fn new_bound_first_udp_socket(local_address: SocketAddr) -> io::Result io::Result { let socket = Socket::new(domain, Type::STREAM, Some(Protocol::TCP))?; if let Err(e) = socket.set_linger(Some(core::time::Duration::from_secs(0))) { @@ -114,6 +118,7 @@ pub fn new_unbound_shared_tcp_socket(domain: Domain) -> io::Result { Ok(socket) } +#[instrument(level = "trace", ret, err)] pub fn new_bound_shared_tcp_socket(local_address: SocketAddr) -> io::Result { let domain = Domain::for_address(local_address); let socket = new_unbound_shared_tcp_socket(domain)?; @@ -125,6 +130,7 @@ pub fn new_bound_shared_tcp_socket(local_address: SocketAddr) -> io::Result io::Result { let domain = Domain::for_address(local_address); @@ -166,6 +172,7 @@ pub fn new_bound_first_tcp_socket(local_address: SocketAddr) -> io::Result Ok(()), @@ -184,7 +194,27 @@ pub async fn nonblocking_connect( Err(err) if err.raw_os_error() == Some(libc::EINPROGRESS) => Ok(()), Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => Ok(()), Err(e) => Err(e), - }?; + } + .map_err(|e| { + // XXX + warn!( + "DEBUGCONNECT XXXFAILXXX: bind={} local={} remote={}\nbacktrace={:?}", + bind_local_addr, + socket.local_addr().unwrap().as_socket().unwrap(), + addr, + backtrace::Backtrace::new(), + ); + e + })?; + + // XXX + warn!( + "DEBUGCONNECT: bind={} local={} remote={}\nbacktrace={:?}", + bind_local_addr, + socket.local_addr().unwrap().as_socket().unwrap(), + addr, + backtrace::Backtrace::new(), + ); let async_stream = Async::new(std::net::TcpStream::from(socket))?; diff --git a/veilid-core/src/network_manager/native/protocol/tcp.rs b/veilid-core/src/network_manager/native/protocol/tcp.rs index d0f843e0..91167b69 100644 --- a/veilid-core/src/network_manager/native/protocol/tcp.rs +++ b/veilid-core/src/network_manager/native/protocol/tcp.rs @@ -99,30 +99,20 @@ impl RawTcpNetworkConnection { /////////////////////////////////////////////////////////// /// -struct RawTcpProtocolHandlerInner { - local_address: SocketAddr, -} - #[derive(Clone)] pub struct RawTcpProtocolHandler where Self: ProtocolAcceptHandler, { connection_initial_timeout_ms: u32, - inner: Arc>, } impl RawTcpProtocolHandler { - fn new_inner(local_address: SocketAddr) -> RawTcpProtocolHandlerInner { - RawTcpProtocolHandlerInner { local_address } - } - - pub fn new(config: VeilidConfig, local_address: SocketAddr) -> Self { + pub fn new(config: VeilidConfig) -> Self { let c = config.get(); let connection_initial_timeout_ms = c.network.connection_initial_timeout_ms; Self { connection_initial_timeout_ms, - inner: Arc::new(Mutex::new(Self::new_inner(local_address))), } } @@ -131,6 +121,7 @@ impl RawTcpProtocolHandler { self, ps: AsyncPeekStream, socket_addr: SocketAddr, + local_addr: SocketAddr, ) -> io::Result> { log_net!("TCP: on_accept_async: enter"); let mut peekbuf: [u8; PEEK_DETECT_LEN] = [0u8; PEEK_DETECT_LEN]; @@ -147,9 +138,8 @@ impl RawTcpProtocolHandler { SocketAddress::from_socket_addr(socket_addr), ProtocolType::TCP, ); - let local_address = self.inner.lock().local_address; let conn = ProtocolNetworkConnection::RawTcp(RawTcpNetworkConnection::new( - ConnectionDescriptor::new(peer_addr, SocketAddress::from_socket_addr(local_address)), + ConnectionDescriptor::new(peer_addr, SocketAddress::from_socket_addr(local_addr)), ps, )); @@ -158,7 +148,7 @@ impl RawTcpProtocolHandler { Ok(Some(conn)) } - #[instrument(level = "trace", err)] + #[instrument(level = "trace", ret, err)] pub async fn connect( local_address: Option, socket_addr: SocketAddr, @@ -202,7 +192,8 @@ impl ProtocolAcceptHandler for RawTcpProtocolHandler { &self, stream: AsyncPeekStream, peer_addr: SocketAddr, + local_addr: SocketAddr, ) -> SendPinBoxFuture>> { - Box::pin(self.clone().on_accept_async(stream, peer_addr)) + Box::pin(self.clone().on_accept_async(stream, peer_addr, local_addr)) } } diff --git a/veilid-core/src/network_manager/native/protocol/ws.rs b/veilid-core/src/network_manager/native/protocol/ws.rs index a0b48b42..520cd344 100644 --- a/veilid-core/src/network_manager/native/protocol/ws.rs +++ b/veilid-core/src/network_manager/native/protocol/ws.rs @@ -129,7 +129,6 @@ where /// struct WebsocketProtocolHandlerArc { tls: bool, - local_address: SocketAddr, request_path: Vec, connection_initial_timeout_ms: u32, } @@ -142,7 +141,7 @@ where arc: Arc, } impl WebsocketProtocolHandler { - pub fn new(config: VeilidConfig, tls: bool, local_address: SocketAddr) -> Self { + pub fn new(config: VeilidConfig, tls: bool) -> Self { let c = config.get(); let path = if tls { format!("GET /{}", c.network.protocol.ws.path.trim_end_matches('/')) @@ -158,7 +157,6 @@ impl WebsocketProtocolHandler { Self { arc: Arc::new(WebsocketProtocolHandlerArc { tls, - local_address, request_path: path.as_bytes().to_vec(), connection_initial_timeout_ms, }), @@ -170,6 +168,7 @@ impl WebsocketProtocolHandler { self, ps: AsyncPeekStream, socket_addr: SocketAddr, + local_addr: SocketAddr, ) -> io::Result> { log_net!("WS: on_accept_async: enter"); let request_path_len = self.arc.request_path.len() + 2; @@ -209,10 +208,7 @@ impl WebsocketProtocolHandler { PeerAddress::new(SocketAddress::from_socket_addr(socket_addr), protocol_type); let conn = ProtocolNetworkConnection::WsAccepted(WebsocketNetworkConnection::new( - ConnectionDescriptor::new( - peer_addr, - SocketAddress::from_socket_addr(self.arc.local_address), - ), + ConnectionDescriptor::new(peer_addr, SocketAddress::from_socket_addr(local_addr)), ws_stream, )); @@ -221,7 +217,7 @@ impl WebsocketProtocolHandler { Ok(Some(conn)) } - #[instrument(level = "trace", err)] + #[instrument(level = "trace", ret, err)] pub async fn connect( local_address: Option, dial_info: &DialInfo, @@ -296,7 +292,8 @@ impl ProtocolAcceptHandler for WebsocketProtocolHandler { &self, stream: AsyncPeekStream, peer_addr: SocketAddr, + local_addr: SocketAddr, ) -> SendPinBoxFuture>> { - Box::pin(self.clone().on_accept_async(stream, peer_addr)) + Box::pin(self.clone().on_accept_async(stream, peer_addr, local_addr)) } } diff --git a/veilid-core/src/network_manager/native/start_protocols.rs b/veilid-core/src/network_manager/native/start_protocols.rs index bf862fa4..36f285e4 100644 --- a/veilid-core/src/network_manager/native/start_protocols.rs +++ b/veilid-core/src/network_manager/native/start_protocols.rs @@ -387,7 +387,7 @@ impl Network { ip_addrs, ws_port, false, - Box::new(|c, t, a| Box::new(WebsocketProtocolHandler::new(c, t, a))), + Box::new(|c, t| Box::new(WebsocketProtocolHandler::new(c, t))), ) .await?; trace!("WS: listener started on {:#?}", socket_addresses); @@ -496,7 +496,7 @@ impl Network { ip_addrs, wss_port, true, - Box::new(|c, t, a| Box::new(WebsocketProtocolHandler::new(c, t, a))), + Box::new(|c, t| Box::new(WebsocketProtocolHandler::new(c, t))), ) .await?; trace!("WSS: listener started on {:#?}", socket_addresses); @@ -590,7 +590,7 @@ impl Network { ip_addrs, tcp_port, false, - Box::new(move |c, _, a| Box::new(RawTcpProtocolHandler::new(c, a))), + Box::new(move |c, _| Box::new(RawTcpProtocolHandler::new(c))), ) .await?; trace!("TCP: listener started on {:#?}", socket_addresses); diff --git a/veilid-core/src/network_manager/network_connection.rs b/veilid-core/src/network_manager/network_connection.rs index 98d007e2..b9658b4a 100644 --- a/veilid-core/src/network_manager/network_connection.rs +++ b/veilid-core/src/network_manager/network_connection.rs @@ -16,6 +16,7 @@ cfg_if::cfg_if! { &self, stream: AsyncPeekStream, peer_addr: SocketAddr, + local_addr: SocketAddr, ) -> SendPinBoxFuture>>; } @@ -38,7 +39,7 @@ cfg_if::cfg_if! { } pub type NewProtocolAcceptHandler = - dyn Fn(VeilidConfig, bool, SocketAddr) -> Box + Send; + dyn Fn(VeilidConfig, bool) -> Box + Send; } } /////////////////////////////////////////////////////////// @@ -91,7 +92,7 @@ pub struct NetworkConnection { processor: Option>, established_time: u64, stats: Arc>, - sender: flume::Sender>, + sender: flume::Sender<(Option, Vec)>, stop_source: Option, } @@ -120,9 +121,6 @@ impl NetworkConnection { protocol_connection: ProtocolNetworkConnection, connection_id: NetworkConnectionId, ) -> Self { - // Get timeout - let network_manager = connection_manager.network_manager(); - // Get descriptor let descriptor = protocol_connection.descriptor(); @@ -181,6 +179,7 @@ impl NetworkConnection { } } + #[instrument(level="trace", skip(message, stats), fields(message.len = message.len()), ret, err)] async fn send_internal( protocol_connection: &ProtocolNetworkConnection, stats: Arc>, @@ -194,6 +193,8 @@ impl NetworkConnection { Ok(NetworkResult::Value(out)) } + + #[instrument(level="trace", skip(stats), fields(ret.len), err)] async fn recv_internal( protocol_connection: &ProtocolNetworkConnection, stats: Arc>, @@ -204,6 +205,8 @@ impl NetworkConnection { let mut stats = stats.lock(); stats.last_message_recv_time.max_assign(Some(ts)); + tracing::Span::current().record("ret.len", out.len()); + Ok(NetworkResult::Value(out)) } @@ -223,7 +226,7 @@ impl NetworkConnection { manager_stop_token: StopToken, connection_id: NetworkConnectionId, descriptor: ConnectionDescriptor, - receiver: flume::Receiver>, + receiver: flume::Receiver<(Option, Vec)>, protocol_connection: ProtocolNetworkConnection, stats: Arc>, ) -> SendPinBoxFuture<()> { @@ -249,7 +252,7 @@ impl NetworkConnection { }; let timer = MutableFuture::new(new_timer()); - unord.push(system_boxed(timer.clone())); + unord.push(system_boxed(timer.clone().instrument(Span::current()))); loop { // Add another message sender future if necessary @@ -257,13 +260,17 @@ impl NetworkConnection { need_sender = false; let sender_fut = receiver.recv_async().then(|res| async { match res { - Ok(message) => { + Ok((span_id, message)) => { + + let recv_span = span!(parent: None, Level::TRACE, "process_connection recv"); + recv_span.follows_from(span_id); + // send the packet if let Err(e) = Self::send_internal( &protocol_connection, stats.clone(), message, - ) + ).instrument(recv_span) .await { // Sending the packet along can fail, if so, this connection is dead @@ -280,7 +287,7 @@ impl NetworkConnection { } } }); - unord.push(system_boxed(sender_fut)); + unord.push(system_boxed(sender_fut.instrument(Span::current()))); } // Add another message receiver future if necessary @@ -314,7 +321,7 @@ impl NetworkConnection { } }); - unord.push(system_boxed(receiver_fut)); + unord.push(system_boxed(receiver_fut.instrument(Span::current()))); } // Process futures @@ -358,7 +365,7 @@ impl NetworkConnection { connection_manager .report_connection_finished(connection_id) .await; - }) + }.instrument(trace_span!("process_connection"))) } } diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index bc571210..30014fca 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -39,119 +39,125 @@ impl NetworkManager { // Get bootstrap nodes from hostnames concurrently let mut unord = FuturesUnordered::new(); for bsname in bsnames { - unord.push(async move { - // look up boostrap node txt records - let bsnirecords = match intf::txt_lookup(&bsname).await { - Err(e) => { - warn!("bootstrap node txt lookup failed for {}: {}", bsname, e); - return None; - } - Ok(v) => v, - }; - // for each record resolve into key/bootstraprecord pairs - let mut bootstrap_records: Vec<(DHTKey, BootstrapRecord)> = Vec::new(); - for bsnirecord in bsnirecords { - // Bootstrap TXT Record Format Version 0: - // txt_version,min_version,max_version,nodeid,hostname,dialinfoshort* - // - // Split bootstrap node record by commas. Example: - // 0,0,0,7lxDEabK_qgjbe38RtBa3IZLrud84P6NhGP-pRTZzdQ,bootstrap-1.dev.veilid.net,T5150,U5150,W5150/ws - let records: Vec = bsnirecord - .trim() - .split(',') - .map(|x| x.trim().to_owned()) - .collect(); - if records.len() < 6 { - warn!("invalid number of fields in bootstrap txt record"); - continue; - } - - // Bootstrap TXT record version - let txt_version: u8 = match records[0].parse::() { - Ok(v) => v, + unord.push( + async move { + // look up boostrap node txt records + let bsnirecords = match intf::txt_lookup(&bsname).await { Err(e) => { - warn!( + warn!("bootstrap node txt lookup failed for {}: {}", bsname, e); + return None; + } + Ok(v) => v, + }; + // for each record resolve into key/bootstraprecord pairs + let mut bootstrap_records: Vec<(DHTKey, BootstrapRecord)> = Vec::new(); + for bsnirecord in bsnirecords { + // Bootstrap TXT Record Format Version 0: + // txt_version,min_version,max_version,nodeid,hostname,dialinfoshort* + // + // Split bootstrap node record by commas. Example: + // 0,0,0,7lxDEabK_qgjbe38RtBa3IZLrud84P6NhGP-pRTZzdQ,bootstrap-1.dev.veilid.net,T5150,U5150,W5150/ws + let records: Vec = bsnirecord + .trim() + .split(',') + .map(|x| x.trim().to_owned()) + .collect(); + if records.len() < 6 { + warn!("invalid number of fields in bootstrap txt record"); + continue; + } + + // Bootstrap TXT record version + let txt_version: u8 = match records[0].parse::() { + Ok(v) => v, + Err(e) => { + warn!( "invalid txt_version specified in bootstrap node txt record: {}", e ); + continue; + } + }; + if txt_version != BOOTSTRAP_TXT_VERSION { + warn!("unsupported bootstrap txt record version"); continue; } - }; - if txt_version != BOOTSTRAP_TXT_VERSION { - warn!("unsupported bootstrap txt record version"); - continue; - } - // Min/Max wire protocol version - let min_version: u8 = match records[1].parse::() { - Ok(v) => v, - Err(e) => { - warn!( + // Min/Max wire protocol version + let min_version: u8 = match records[1].parse::() { + Ok(v) => v, + Err(e) => { + warn!( "invalid min_version specified in bootstrap node txt record: {}", e ); - continue; - } - }; - let max_version: u8 = match records[2].parse::() { - Ok(v) => v, - Err(e) => { - warn!( + continue; + } + }; + let max_version: u8 = match records[2].parse::() { + Ok(v) => v, + Err(e) => { + warn!( "invalid max_version specified in bootstrap node txt record: {}", e ); - continue; - } - }; - - // Node Id - let node_id_str = &records[3]; - let node_id_key = match DHTKey::try_decode(node_id_str) { - Ok(v) => v, - Err(e) => { - warn!( - "Invalid node id in bootstrap node record {}: {}", - node_id_str, e - ); - continue; - } - }; - - // Hostname - let hostname_str = &records[4]; - - // If this is our own node id, then we skip it for bootstrap, in case we are a bootstrap node - if self.routing_table().node_id() == node_id_key { - continue; - } - - // Resolve each record and store in node dial infos list - let mut bootstrap_record = BootstrapRecord { - min_version, - max_version, - dial_info_details: Vec::new(), - }; - for rec in &records[5..] { - let rec = rec.trim(); - let dial_infos = match DialInfo::try_vec_from_short(rec, hostname_str) { - Ok(dis) => dis, - Err(e) => { - warn!("Couldn't resolve bootstrap node dial info {}: {}", rec, e); continue; } }; - for di in dial_infos { - bootstrap_record.dial_info_details.push(DialInfoDetail { - dial_info: di, - class: DialInfoClass::Direct, - }); + // Node Id + let node_id_str = &records[3]; + let node_id_key = match DHTKey::try_decode(node_id_str) { + Ok(v) => v, + Err(e) => { + warn!( + "Invalid node id in bootstrap node record {}: {}", + node_id_str, e + ); + continue; + } + }; + + // Hostname + let hostname_str = &records[4]; + + // If this is our own node id, then we skip it for bootstrap, in case we are a bootstrap node + if self.routing_table().node_id() == node_id_key { + continue; } + + // Resolve each record and store in node dial infos list + let mut bootstrap_record = BootstrapRecord { + min_version, + max_version, + dial_info_details: Vec::new(), + }; + for rec in &records[5..] { + let rec = rec.trim(); + let dial_infos = match DialInfo::try_vec_from_short(rec, hostname_str) { + Ok(dis) => dis, + Err(e) => { + warn!( + "Couldn't resolve bootstrap node dial info {}: {}", + rec, e + ); + continue; + } + }; + + for di in dial_infos { + bootstrap_record.dial_info_details.push(DialInfoDetail { + dial_info: di, + class: DialInfoClass::Direct, + }); + } + } + bootstrap_records.push((node_id_key, bootstrap_record)); } - bootstrap_records.push((node_id_key, bootstrap_record)); + Some(bootstrap_records) } - Some(bootstrap_records) - }); + .instrument(Span::current()), + ); } let mut bsmap = BootstrapRecordMap::new(); @@ -172,6 +178,7 @@ impl NetworkManager { } // 'direct' bootstrap task routine for systems incapable of resolving TXT records, such as browser WASM + #[instrument(level = "trace", skip(self), err)] pub(super) async fn direct_bootstrap_task_routine( self, stop_token: StopToken, @@ -201,7 +208,8 @@ impl NetworkManager { let routing_table = routing_table.clone(); unord.push( // lets ask bootstrap to find ourselves now - async move { routing_table.reverse_find_node(nr, true).await }, + async move { routing_table.reverse_find_node(nr, true).await } + .instrument(Span::current()), ); } } @@ -300,23 +308,26 @@ impl NetworkManager { ) { // Add this our futures to process in parallel let routing_table = routing_table.clone(); - unord.push(async move { - // Need VALID signed peer info, so ask bootstrap to find_node of itself - // which will ensure it has the bootstrap's signed peer info as part of the response - let _ = routing_table.find_target(nr.clone()).await; + unord.push( + async move { + // Need VALID signed peer info, so ask bootstrap to find_node of itself + // which will ensure it has the bootstrap's signed peer info as part of the response + let _ = routing_table.find_target(nr.clone()).await; - // Ensure we got the signed peer info - if !nr.signed_node_info_has_valid_signature(RoutingDomain::PublicInternet) { - log_net!(warn - "bootstrap at {:?} did not return valid signed node info", - nr - ); - // If this node info is invalid, it will time out after being unpingable - } else { - // otherwise this bootstrap is valid, lets ask it to find ourselves now - routing_table.reverse_find_node(nr, true).await + // Ensure we got the signed peer info + if !nr.signed_node_info_has_valid_signature(RoutingDomain::PublicInternet) { + log_net!(warn + "bootstrap at {:?} did not return valid signed node info", + nr + ); + // If this node info is invalid, it will time out after being unpingable + } else { + // otherwise this bootstrap is valid, lets ask it to find ourselves now + routing_table.reverse_find_node(nr, true).await + } } - }); + .instrument(Span::current()), + ); } } @@ -382,7 +393,11 @@ impl NetworkManager { let nr_filtered = nr.filtered_clone(NodeRefFilter::new().with_dial_info_filter(dif)); log_net!("--> Keepalive ping to {:?}", nr_filtered); - unord.push(async move { rpc.rpc_call_status(nr_filtered).await }.boxed()); + unord.push( + async move { rpc.rpc_call_status(nr_filtered).await } + .instrument(Span::current()) + .boxed(), + ); did_pings = true; } } @@ -392,7 +407,11 @@ impl NetworkManager { // any mapped ports to preserve if !did_pings { let rpc = rpc.clone(); - unord.push(async move { rpc.rpc_call_status(nr).await }.boxed()); + unord.push( + async move { rpc.rpc_call_status(nr).await } + .instrument(Span::current()) + .boxed(), + ); } } @@ -420,7 +439,11 @@ impl NetworkManager { let rpc = rpc.clone(); // Just do a single ping with the best protocol for all the nodes - unord.push(async move { rpc.rpc_call_status(nr).await }.boxed()); + unord.push( + async move { rpc.rpc_call_status(nr).await } + .instrument(Span::current()) + .boxed(), + ); } Ok(()) @@ -479,7 +502,10 @@ impl NetworkManager { ); for nr in noderefs { let routing_table = routing_table.clone(); - ord.push_back(async move { routing_table.reverse_find_node(nr, false).await }); + ord.push_back( + async move { routing_table.reverse_find_node(nr, false).await } + .instrument(Span::current()), + ); } // do peer minimum search in order from fastest to slowest diff --git a/veilid-core/src/network_manager/wasm/protocol/ws.rs b/veilid-core/src/network_manager/wasm/protocol/ws.rs index 1c9df3c3..1b48c59d 100644 --- a/veilid-core/src/network_manager/wasm/protocol/ws.rs +++ b/veilid-core/src/network_manager/wasm/protocol/ws.rs @@ -106,7 +106,7 @@ impl WebsocketNetworkConnection { pub struct WebsocketProtocolHandler {} impl WebsocketProtocolHandler { - #[instrument(level = "trace", err)] + #[instrument(level = "trace", ret, err)] pub async fn connect( dial_info: &DialInfo, timeout_ms: u32, diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 6a6ac7ad..63a36b35 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -147,8 +147,6 @@ pub struct RPCProcessorInner { } pub struct RPCProcessorUnlockedInner { - node_id: DHTKey, - node_id_secret: DHTKeySecret, timeout: u64, queue_size: u32, concurrency: u32, @@ -183,8 +181,6 @@ impl RPCProcessor { ) -> RPCProcessorUnlockedInner { // make local copy of node id for easy access let c = config.get(); - let node_id = c.network.node_id; - let node_id_secret = c.network.node_id_secret; // set up channel let mut concurrency = c.network.rpc.concurrency; @@ -209,8 +205,6 @@ impl RPCProcessor { let validate_dial_info_receipt_time_ms = c.network.dht.validate_dial_info_receipt_time_ms; RPCProcessorUnlockedInner { - node_id, - node_id_secret, timeout, queue_size, concurrency, @@ -947,16 +941,16 @@ impl RPCProcessor { stop_token: StopToken, receiver: flume::Receiver<(Option, RPCMessageEncoded)>, ) { - while let Ok(Ok((_span_id, msg))) = + while let Ok(Ok((span_id, msg))) = receiver.recv_async().timeout_at(stop_token.clone()).await { let rpc_worker_span = span!(parent: None, Level::TRACE, "rpc_worker"); + //let rpc_worker_span = span!(Level::TRACE, "rpc_worker"); // fixme: causes crashes? "Missing otel data span extensions"?? - //rpc_worker_span.follows_from(span_id); - let _enter = rpc_worker_span.enter(); - + rpc_worker_span.follows_from(span_id); let _ = self .process_rpc_message(msg) + .instrument(rpc_worker_span) .await .map_err(logthru_rpc!("couldn't process rpc message")); } diff --git a/veilid-core/src/tests/common/mod.rs b/veilid-core/src/tests/common/mod.rs index eaae93db..537a41a5 100644 --- a/veilid-core/src/tests/common/mod.rs +++ b/veilid-core/src/tests/common/mod.rs @@ -1,3 +1,4 @@ +pub mod test_async_tag_lock; pub mod test_host_interface; pub mod test_protected_store; pub mod test_table_store; diff --git a/veilid-core/src/tests/common/test_async_tag_lock.rs b/veilid-core/src/tests/common/test_async_tag_lock.rs new file mode 100644 index 00000000..f932a457 --- /dev/null +++ b/veilid-core/src/tests/common/test_async_tag_lock.rs @@ -0,0 +1,159 @@ +use crate::xx::*; +use crate::*; + +pub async fn test_simple_no_contention() { + info!("test_simple_no_contention"); + + let table = AsyncTagLockTable::new(); + + let a1 = SocketAddr::new("1.2.3.4".parse().unwrap(), 1234); + let a2 = SocketAddr::new("6.9.6.9".parse().unwrap(), 6969); + + { + let g1 = table.lock_tag(a1).await; + let g2 = table.lock_tag(a2).await; + drop(g2); + drop(g1); + } + + { + let g1 = table.lock_tag(a1).await; + let g2 = table.lock_tag(a2).await; + drop(g1); + drop(g2); + } + + assert_eq!(table.len(), 0); +} + +pub async fn test_simple_single_contention() { + info!("test_simple_single_contention"); + + let table = AsyncTagLockTable::new(); + + let a1 = SocketAddr::new("1.2.3.4".parse().unwrap(), 1234); + + let g1 = table.lock_tag(a1).await; + + println!("locked"); + let t1 = intf::spawn(async move { + // move the guard into the task + let _g1_take = g1; + // hold the guard for a bit + println!("waiting"); + intf::sleep(1000).await; + // release the guard + println!("released"); + }); + + // wait to lock again, will contend until spawned task exits + let _g1_b = table.lock_tag(a1).await; + println!("locked"); + + // Ensure task is joined + t1.await; + + assert_eq!(table.len(), 1); +} + +pub async fn test_simple_double_contention() { + info!("test_simple_double_contention"); + + let table = AsyncTagLockTable::new(); + + let a1 = SocketAddr::new("1.2.3.4".parse().unwrap(), 1234); + let a2 = SocketAddr::new("6.9.6.9".parse().unwrap(), 6969); + + let g1 = table.lock_tag(a1).await; + let g2 = table.lock_tag(a2).await; + + println!("locked"); + let t1 = intf::spawn(async move { + // move the guard into the task + let _g1_take = g1; + // hold the guard for a bit + println!("waiting"); + intf::sleep(1000).await; + // release the guard + println!("released"); + }); + let t2 = intf::spawn(async move { + // move the guard into the task + let _g2_take = g2; + // hold the guard for a bit + println!("waiting"); + intf::sleep(500).await; + // release the guard + println!("released"); + }); + + // wait to lock again, will contend until spawned task exits + let _g1_b = table.lock_tag(a1).await; + // wait to lock again, should complete immediately + let _g2_b = table.lock_tag(a2).await; + + println!("locked"); + + // Ensure tasks are joined + t1.await; + t2.await; + + assert_eq!(table.len(), 2); +} + +pub async fn test_parallel_single_contention() { + info!("test_parallel_single_contention"); + + let table = AsyncTagLockTable::new(); + + let a1 = SocketAddr::new("1.2.3.4".parse().unwrap(), 1234); + + let table1 = table.clone(); + let t1 = intf::spawn(async move { + // lock the tag + let _g = table1.lock_tag(a1).await; + println!("locked t1"); + // hold the guard for a bit + println!("waiting t1"); + intf::sleep(500).await; + // release the guard + println!("released t1"); + }); + + let table2 = table.clone(); + let t2 = intf::spawn(async move { + // lock the tag + let _g = table2.lock_tag(a1).await; + println!("locked t2"); + // hold the guard for a bit + println!("waiting t2"); + intf::sleep(500).await; + // release the guard + println!("released t2"); + }); + + let table3 = table.clone(); + let t3 = intf::spawn(async move { + // lock the tag + let _g = table3.lock_tag(a1).await; + println!("locked t3"); + // hold the guard for a bit + println!("waiting t3"); + intf::sleep(500).await; + // release the guard + println!("released t3"); + }); + + // Ensure tasks are joined + t1.await; + t2.await; + t3.await; + + assert_eq!(table.len(), 0); +} + +pub async fn test_all() { + test_simple_no_contention().await; + test_simple_single_contention().await; + test_parallel_single_contention().await; +} diff --git a/veilid-core/src/tests/native/mod.rs b/veilid-core/src/tests/native/mod.rs index e5ea0e2a..da86b323 100644 --- a/veilid-core/src/tests/native/mod.rs +++ b/veilid-core/src/tests/native/mod.rs @@ -70,6 +70,8 @@ pub fn run_all_tests() { exec_test_crypto(); info!("TEST: exec_test_envelope_receipt"); exec_test_envelope_receipt(); + info!("TEST: exec_test_async_tag_lock"); + exec_test_async_tag_lock(); info!("Finished unit tests"); } @@ -135,6 +137,11 @@ fn exec_test_envelope_receipt() { test_envelope_receipt::test_all().await; }) } +fn exec_test_async_tag_lock() { + block_on(async { + test_async_tag_lock::test_all().await; + }) +} /////////////////////////////////////////////////////////////////////////// cfg_if! { if #[cfg(test)] { @@ -223,5 +230,12 @@ cfg_if! { setup(); exec_test_envelope_receipt(); } + + #[test] + #[serial] + fn run_test_async_tag_lock() { + setup(); + exec_test_async_tag_lock(); + } } } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 0934ff5c..f330ef44 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -1636,6 +1636,13 @@ impl PeerAddress { } } +/// Represents the 5-tuple of an established connection +/// Not used to specify connections to create, that is reserved for DialInfo +/// +/// ConnectionDescriptors should never be from unspecified local addresses for connection oriented protocols +/// If the medium does not allow local addresses, None should have been used or 'new_no_local' +/// If we are specifying only a port, then the socket's 'local_address()' should have been used, since an +/// established connection is always from a real address to another real address. #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct ConnectionDescriptor { remote: PeerAddress, @@ -1644,6 +1651,10 @@ pub struct ConnectionDescriptor { impl ConnectionDescriptor { pub fn new(remote: PeerAddress, local: SocketAddress) -> Self { + assert!( + !remote.protocol_type().is_connection_oriented() || !local.address().is_unspecified() + ); + Self { remote, local: Some(local), diff --git a/veilid-core/src/xx/async_tag_lock.rs b/veilid-core/src/xx/async_tag_lock.rs new file mode 100644 index 00000000..9e865935 --- /dev/null +++ b/veilid-core/src/xx/async_tag_lock.rs @@ -0,0 +1,126 @@ +use super::*; +use core::hash::Hash; + +pub struct AsyncTagLockGuard +where + T: Hash + Eq + Clone, +{ + table: AsyncTagLockTable, + tag: T, + _guard: AsyncMutexGuardArc<()>, +} + +impl Drop for AsyncTagLockGuard +where + T: Hash + Eq + Clone, +{ + fn drop(&mut self) { + let mut inner = self.table.inner.lock(); + // Inform the table we're dropping this guard + let waiters = { + // Get the table entry, it must exist since we have a guard locked + let entry = inner.table.get_mut(&self.tag).unwrap(); + // Decrement the number of waiters + entry.waiters -= 1; + // Return the number of waiters left + entry.waiters + }; + // If there are no waiters left, we remove the tag from the table + if waiters == 0 { + inner.table.remove(&self.tag).unwrap(); + } + // Proceed with releasing _guard, which may cause some concurrent tag lock to acquire + } +} + +#[derive(Clone)] +struct AsyncTagLockTableEntry { + mutex: Arc>, + waiters: usize, +} + +struct AsyncTagLockTableInner +where + T: Hash + Eq + Clone, +{ + table: HashMap, +} + +#[derive(Clone)] +pub struct AsyncTagLockTable +where + T: Hash + Eq + Clone, +{ + inner: Arc>>, +} + +impl fmt::Debug for AsyncTagLockTable +where + T: Hash + Eq + Clone, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AsyncTagLockTable").finish() + } +} + +impl AsyncTagLockTable +where + T: Hash + Eq + Clone, +{ + pub fn new() -> Self { + Self { + inner: Arc::new(Mutex::new(AsyncTagLockTableInner { + table: HashMap::new(), + })), + } + } + + pub fn len(&self) -> usize { + let inner = self.inner.lock(); + inner.table.len() + } + + pub async fn lock_tag(&self, tag: T) -> AsyncTagLockGuard { + // Get or create a tag lock entry + let mutex = { + let mut inner = self.inner.lock(); + + // See if this tag is in the table + // and if not, add a new mutex for this tag + let entry = inner + .table + .entry(tag.clone()) + .or_insert_with(|| AsyncTagLockTableEntry { + mutex: Arc::new(AsyncMutex::new(())), + waiters: 0, + }); + + // Increment the number of waiters + entry.waiters += 1; + + // Return the mutex associated with the tag + entry.mutex.clone() + + // Drop the table guard + }; + + // Lock the tag lock + let guard; + cfg_if! { + if #[cfg(feature="rt-tokio")] { + // tokio version + guard = mutex.lock_owned().await; + } else { + // async_std and wasm async_mutex version + guard = mutex.lock_arc().await; + } + } + + // Return the locked guard + AsyncTagLockGuard { + table: self.clone(), + tag, + _guard: guard, + } + } +} diff --git a/veilid-core/src/xx/mod.rs b/veilid-core/src/xx/mod.rs index 0f72f7b6..d9f48ffd 100644 --- a/veilid-core/src/xx/mod.rs +++ b/veilid-core/src/xx/mod.rs @@ -1,5 +1,6 @@ // mod bump_port; mod async_peek_stream; +mod async_tag_lock; mod clone_stream; mod eventual; mod eventual_base; @@ -68,6 +69,7 @@ cfg_if! { pub use core::ops::{FnOnce, FnMut, Fn}; pub use async_lock::Mutex as AsyncMutex; pub use async_lock::MutexGuard as AsyncMutexGuard; + pub use async_lock::MutexGuardArc as AsyncMutexGuardArc; pub use no_std_net::{ SocketAddr, SocketAddrV4, SocketAddrV6, ToSocketAddrs, IpAddr, Ipv4Addr, Ipv6Addr }; pub use async_executors::JoinHandle as LowLevelJoinHandle; } else { @@ -98,10 +100,12 @@ cfg_if! { if #[cfg(feature="rt-async-std")] { pub use async_std::sync::Mutex as AsyncMutex; pub use async_std::sync::MutexGuard as AsyncMutexGuard; + pub use async_std::sync::MutexGuardArc as AsyncMutexGuardArc; pub use async_std::task::JoinHandle as LowLevelJoinHandle; } else if #[cfg(feature="rt-tokio")] { pub use tokio::sync::Mutex as AsyncMutex; pub use tokio::sync::MutexGuard as AsyncMutexGuard; + pub use tokio::sync::OwnedMutexGuard as AsyncMutexGuardArc; pub use tokio::task::JoinHandle as LowLevelJoinHandle; } else { #[compile_error("must use an executor")] @@ -113,6 +117,7 @@ cfg_if! { // pub use bump_port::*; pub use async_peek_stream::*; +pub use async_tag_lock::*; pub use clone_stream::*; pub use eventual::*; pub use eventual_base::{EventualCommon, EventualResolvedFuture}; diff --git a/veilid-core/tests/node.rs b/veilid-core/tests/node.rs index 547456c3..da301eeb 100644 --- a/veilid-core/tests/node.rs +++ b/veilid-core/tests/node.rs @@ -74,3 +74,10 @@ async fn run_test_envelope_receipt() { test_envelope_receipt::test_all().await; } + +#[wasm_bindgen_test] +async fn run_test_async_tag_lock() { + setup(); + + test_async_tag_lock::test_all().await; +} diff --git a/veilid-core/tests/web.rs b/veilid-core/tests/web.rs index bc93ae4c..a5ac14b7 100644 --- a/veilid-core/tests/web.rs +++ b/veilid-core/tests/web.rs @@ -78,3 +78,10 @@ async fn run_test_envelope_receipt() { test_envelope_receipt::test_all().await; } + +#[wasm_bindgen_test] +async fn run_test_async_tag_lock() { + setup(); + + test_async_tag_lock::test_all().await; +} From 4b2164a54605e948d00dfc73006ed40a8df0218e Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 4 Oct 2022 11:27:38 -0400 Subject: [PATCH 04/67] various fixes, including node_ref last_connection sorting problem --- Cargo.lock | 936 ++++++++++-------- setup_linux.sh | 3 +- setup_macos.sh | 2 +- .../src/network_manager/connection_manager.rs | 38 +- .../src/network_manager/connection_table.rs | 12 +- veilid-core/src/network_manager/mod.rs | 82 +- .../native/network_class_discovery.rs | 13 + .../src/network_manager/native/network_tcp.rs | 10 +- .../src/network_manager/native/network_udp.rs | 2 +- .../native/protocol/sockets.rs | 30 +- .../src/network_manager/network_connection.rs | 9 +- veilid-core/src/receipt_manager.rs | 2 +- veilid-core/src/routing_table/bucket_entry.rs | 70 +- veilid-core/src/routing_table/node_ref.rs | 35 +- veilid-core/src/rpc_processor/mod.rs | 9 +- .../src/rpc_processor/operation_waiter.rs | 4 +- 16 files changed, 740 insertions(+), 517 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66988967..db7beaf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,9 +51,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" dependencies = [ "memchr", ] @@ -108,13 +108,19 @@ dependencies = [ [[package]] name = "android_system_properties" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ed72e1635e121ca3e79420540282af22da58be50de153d36f81ddc6b83aa9e" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "ansi_term" version = "0.12.1" @@ -126,9 +132,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.62" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1485d4d2cc45e7b201ee3767015c96faa5904387c9d87c6efdd0fb511f12d305" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" [[package]] name = "arraydeque" @@ -185,9 +191,9 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5262ed948da60dd8956c6c5aca4d4163593dddb7b32d73267c93dab7b2e98940" +checksum = "0da5b41ee986eed3f524c380e6d64965aea573882a8907682ad100f7859305ca" dependencies = [ "async-channel", "async-executor", @@ -195,15 +201,14 @@ dependencies = [ "async-lock", "blocking", "futures-lite", - "num_cpus", "once_cell", ] [[package]] name = "async-io" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab006897723d9352f63e2b13047177c3982d8d79709d713ce7747a8f19fd1b0" +checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7" dependencies = [ "autocfg", "concurrent-queue", @@ -276,9 +281,9 @@ dependencies = [ [[package]] name = "async-std-resolver" -version = "0.21.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2f8a4a203be3325981310ab243a28e6e4ea55b6519bffce05d41ab60e09ad8" +checksum = "6ba50e24d9ee0a8950d3d03fc6d0dd10aa14b5de3b101949b4e160f7fee7c723" dependencies = [ "async-std", "async-trait", @@ -381,7 +386,7 @@ dependencies = [ "futures-util", "pin-project 1.0.12", "rustc_version", - "tokio 1.20.1", + "tokio 1.21.2", "wasm-bindgen-futures", ] @@ -440,6 +445,51 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e3356844c4d6a6d6467b8da2cffb4a2820be256f50a3a386c9d152bab31043" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes 1.2.1", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite 0.2.9", + "serde", + "sync_wrapper", + "tokio 1.21.2", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f0c0a60006f2a293d82d571f635042a72edf927539b7685bd62d361963839b" +dependencies = [ + "async-trait", + "bytes 1.2.1", + "futures-util", + "http", + "http-body", + "mime", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.66" @@ -539,9 +589,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ "generic-array", ] @@ -591,18 +641,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426" -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bugsalot" version = "0.2.2" @@ -647,15 +685,15 @@ checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" [[package]] name = "capnp" -version = "0.14.8" +version = "0.14.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82efa3b0ab5e7e32b786334b052560ec0094135f906975d7481651b9ecf31a6a" +checksum = "ddf2d33bdc66ac282c34d34ecc9f8d8f1d82d1540994c5f8ed1c36dd0f95aaf3" [[package]] name = "capnp-futures" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a15248c8facb189a3c5fee74fbf1ff3adc134261d27da663b89c7d19ebaf983" +checksum = "9821801cc6f199a9d9c3c793504e800c797b536d2befddaffb15144e40a6e63a" dependencies = [ "capnp", "futures", @@ -771,6 +809,33 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "ciborium" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" + +[[package]] +name = "ciborium-ll" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.3.0" @@ -792,9 +857,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a050e2153c5be08febd6734e29298e844fdb0fa21aeddd63b4eb7baa106c69b" +checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" dependencies = [ "glob", "libc", @@ -803,20 +868,9 @@ dependencies = [ [[package]] name = "clap" -version = "2.34.0" +version = "3.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "bitflags", - "textwrap 0.11.0", - "unicode-width", -] - -[[package]] -name = "clap" -version = "3.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29e724a68d9319343bb3328c9cc2dfde263f4b3142ee1059a9980580171c954b" +checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" dependencies = [ "atty", "bitflags", @@ -824,7 +878,7 @@ dependencies = [ "indexmap", "strsim", "termcolor", - "textwrap 0.15.0", + "textwrap", ] [[package]] @@ -956,24 +1010,25 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc948ebb96241bb40ab73effeb80d9f93afaad49359d159a5e61be51619fe813" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" dependencies = [ "libc", ] [[package]] name = "criterion" -version = "0.3.6" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" dependencies = [ + "anes", "atty", "cast", - "clap 2.34.0", + "ciborium", + "clap", "criterion-plot", - "csv", "itertools", "lazy_static", "num-traits", @@ -982,7 +1037,6 @@ dependencies = [ "rayon", "regex", "serde", - "serde_cbor", "serde_derive", "serde_json", "tinytemplate", @@ -991,9 +1045,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.4.5" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", "itertools", @@ -1022,26 +1076,24 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" +checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" dependencies = [ "autocfg", "cfg-if 1.0.0", "crossbeam-utils", "memoffset", - "once_cell", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" dependencies = [ "cfg-if 1.0.0", - "once_cell", ] [[package]] @@ -1095,28 +1147,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "csv" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" -dependencies = [ - "bstr", - "csv-core", - "itoa 0.4.8", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" -dependencies = [ - "memchr", -] - [[package]] name = "ctor" version = "0.1.23" @@ -1157,7 +1187,7 @@ dependencies = [ "libc", "log", "signal-hook", - "tokio 1.20.1", + "tokio 1.21.2", "unicode-segmentation", "unicode-width", ] @@ -1172,7 +1202,7 @@ dependencies = [ "flexi_logger", "lazy_static", "log", - "time 0.3.9", + "time 0.3.14", "unicode-width", ] @@ -1201,8 +1231,8 @@ dependencies = [ "log", "num", "owning_ref", - "time 0.3.9", - "tokio 1.20.1", + "time 0.3.14", + "tokio 1.21.2", "toml", "unicode-segmentation", "unicode-width", @@ -1239,7 +1269,7 @@ checksum = "1c359b7249347e46fb28804470d071c921156ad62b3eef5d34e2ba867533dec8" dependencies = [ "byteorder", "digest 0.9.0", - "rand_core 0.6.3", + "rand_core 0.6.4", "subtle-ng", "zeroize", ] @@ -1260,8 +1290,18 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.13.4", + "darling_macro 0.13.4", +] + +[[package]] +name = "darling" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02" +dependencies = [ + "darling_core 0.14.1", + "darling_macro 0.14.1", ] [[package]] @@ -1278,26 +1318,51 @@ dependencies = [ "syn", ] +[[package]] +name = "darling_core" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "darling_macro" version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "darling_core", + "darling_core 0.13.4", + "quote", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5" +dependencies = [ + "darling_core 0.14.1", "quote", "syn", ] [[package]] name = "dashmap" -version = "5.3.4" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ "cfg-if 1.0.0", "hashbrown", "lock_api", + "once_cell", "parking_lot_core 0.9.3", ] @@ -1329,11 +1394,11 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ - "block-buffer 0.10.2", + "block-buffer 0.10.3", "crypto-common", ] @@ -1393,11 +1458,11 @@ checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "enum-as-inner" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro2", "quote", "syn", @@ -1446,9 +1511,9 @@ dependencies = [ [[package]] name = "enumset" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4799cdb24d48f1f8a7a98d06b7fde65a85a2d1e42b25a889f5406aa1fbefe074" +checksum = "19be8061a06ab6f3a6cf21106c873578bf01bd42ad15e0311a9c76161cb1c753" dependencies = [ "enumset_derive", "serde", @@ -1456,11 +1521,11 @@ dependencies = [ [[package]] name = "enumset_derive" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea83a3fbdc1d999ccfbcbee717eab36f8edf2d71693a23ce0d7cca19e085304c" +checksum = "03e7b551eba279bf0fa88b83a46330168c1560a52a94f5126f892f0b364ab3e0" dependencies = [ - "darling", + "darling 0.14.1", "proc-macro2", "quote", "syn", @@ -1468,9 +1533,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" dependencies = [ "log", "regex", @@ -1492,9 +1557,9 @@ dependencies = [ [[package]] name = "ethbloom" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +checksum = "c22d4b5885b6aa2fe5e8b9329fb8d232bf739e434e6b87347c63bdd00c120f60" dependencies = [ "crunchy", "fixed-hash", @@ -1505,9 +1570,9 @@ dependencies = [ [[package]] name = "ethereum-types" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +checksum = "81224dc661606574f5a0f28c9947d0ee1d93ff11c5f1c4e7272f52e8c0b5483c" dependencies = [ "ethbloom", "fixed-hash", @@ -1588,9 +1653,9 @@ dependencies = [ [[package]] name = "fixed-hash" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", "rand 0.8.5", @@ -1606,9 +1671,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flexi_logger" -version = "0.23.0" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8790f70905b203171c21060222f18f1df5cba07317860215b7880b32aaef290" +checksum = "f4a12e3b5a8775259ee83ac38aea8cdf9c3a1667c02178d207378c0837808fa9" dependencies = [ "ansi_term", "atty", @@ -1619,7 +1684,7 @@ dependencies = [ "regex", "rustversion", "thiserror", - "time 0.3.9", + "time 0.3.14", ] [[package]] @@ -1643,11 +1708,10 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" dependencies = [ - "matches", "percent-encoding", ] @@ -1685,9 +1749,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab30e97ab6aacfe635fad58f22c2bb06c8b685f7421eb1e064a729e2a5f481fa" +checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" dependencies = [ "futures-channel", "futures-core", @@ -1700,9 +1764,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bfc52cbddcfd745bf1740338492bb0bd83d76c67b445f91c5fb29fae29ecaa1" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" dependencies = [ "futures-core", "futures-sink", @@ -1710,15 +1774,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2acedae88d38235936c3922476b10fced7b2b68136f5e3c03c2d5be348a1115" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" [[package]] name = "futures-executor" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d11aa21b5b587a64682c0094c2bdd4df0076c5324961a40cc3abd7f37930528" +checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" dependencies = [ "futures-core", "futures-task", @@ -1727,9 +1791,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93a66fc6d035a26a3ae255a6d2bca35eda63ae4c5512bef54449113f7a1228e5" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" [[package]] name = "futures-lite" @@ -1748,9 +1812,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0db9cce532b0eae2ccf2766ab246f114b56b9cf6d445e00c2549fbc100ca045d" +checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" dependencies = [ "proc-macro2", "quote", @@ -1759,15 +1823,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0bae1fe9752cf7fd9b0064c674ae63f97b37bc714d745cbde0afb7ec4e6765" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" [[package]] name = "futures-task" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "842fc63b931f4056a24d59de13fb1272134ce261816e063e634ad0c15cdc5306" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" [[package]] name = "futures-timer" @@ -1781,9 +1845,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0828a5471e340229c11c77ca80017937ce3c58cb788a17e5f1c2d5c485a9577" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" dependencies = [ "futures-channel", "futures-core", @@ -1899,8 +1963,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.20.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util", "tracing", ] @@ -1929,22 +1993,13 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452c155cb93fecdfb02a73dd57b5d8e442c2063bd7aac72f1bc5e4263a43086" +checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" dependencies = [ "hashbrown", ] -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.0" @@ -2005,7 +2060,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes 1.2.1", "fnv", - "itoa 1.0.3", + "itoa", ] [[package]] @@ -2020,10 +2075,16 @@ dependencies = [ ] [[package]] -name = "httparse" -version = "1.7.1" +name = "http-range-header" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" @@ -2046,10 +2107,10 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.3", + "itoa", "pin-project-lite 0.2.9", "socket2", - "tokio 1.20.1", + "tokio 1.21.2", "tower-service", "tracing", "want", @@ -2063,15 +2124,15 @@ checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ "hyper", "pin-project-lite 0.2.9", - "tokio 1.20.1", + "tokio 1.21.2", "tokio-io-timeout", ] [[package]] name = "iana-time-zone" -version = "0.1.46" +version = "0.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +checksum = "fd911b35d940d2bd0bea0f9100068e5b97b51a1cbe13d13382f132e0365257a0" dependencies = [ "android_system_properties", "core-foundation-sys 0.8.3", @@ -2097,6 +2158,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "ifstructs" version = "0.1.1" @@ -2121,7 +2192,7 @@ dependencies = [ "simplelog 0.9.0", "tokio 0.2.25", "tokio 0.3.7", - "tokio 1.20.1", + "tokio 1.21.2", "url", "xmltree", ] @@ -2146,9 +2217,9 @@ dependencies = [ [[package]] name = "impl-serde" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +checksum = "ebc88fc67028ae3db0c853baa36269d398d5f45b6982f95549ff5def78c935cd" dependencies = [ "serde", ] @@ -2239,19 +2310,13 @@ checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "itertools" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] -[[package]] -name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - [[package]] name = "itoa" version = "1.0.3" @@ -2280,9 +2345,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "js-sys" -version = "0.3.59" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ "wasm-bindgen", ] @@ -2306,9 +2371,9 @@ dependencies = [ [[package]] name = "keccak-hash" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82bc5d5ca345b067619615f62ac6f93e7daa67eb82d080bc380ed480708ec9e3" +checksum = "4b286e6b663fb926e1eeb68528e69cb70ed46c6d65871a21b2215ae8154c6d3c" dependencies = [ "primitive-types", "tiny-keccak", @@ -2343,7 +2408,7 @@ dependencies = [ "backtrace", "byteorder", "cfg-if 1.0.0", - "clap 3.2.17", + "clap", "core-foundation 0.9.3", "core-foundation-sys 0.8.3", "directories", @@ -2453,9 +2518,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.132" +version = "0.2.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" [[package]] name = "libloading" @@ -2498,9 +2563,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "lock_api" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -2518,9 +2583,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.7.8" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909" dependencies = [ "hashbrown", ] @@ -2561,6 +2626,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + [[package]] name = "memchr" version = "2.5.0" @@ -2582,6 +2653,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2590,9 +2667,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" dependencies = [ "adler", ] @@ -2625,7 +2702,7 @@ dependencies = [ "libc", "log", "miow 0.3.7", - "ntapi", + "ntapi 0.3.7", "winapi 0.3.9", ] @@ -2754,7 +2831,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0df7ac00c4672f9d5aece54ee3347520b7e20f158656c7db2e6de01902eb7a6c" dependencies = [ - "darling", + "darling 0.13.4", "proc-macro-crate 1.2.1", "proc-macro2", "quote", @@ -2831,7 +2908,7 @@ dependencies = [ "log", "netlink-packet-core", "netlink-sys", - "tokio 1.20.1", + "tokio 1.21.2", ] [[package]] @@ -2843,7 +2920,7 @@ dependencies = [ "futures", "libc", "log", - "tokio 1.20.1", + "tokio 1.21.2", ] [[package]] @@ -2910,6 +2987,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "ntapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "num" version = "0.4.0" @@ -3037,9 +3123,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] name = "oorandom" @@ -3055,31 +3141,19 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opentelemetry" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" dependencies = [ - "async-std", - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "js-sys", - "lazy_static", - "percent-encoding", - "pin-project 1.0.12", - "rand 0.8.5", - "thiserror", - "tokio 1.20.1", - "tokio-stream", + "opentelemetry_api", + "opentelemetry_sdk", ] [[package]] name = "opentelemetry-otlp" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a" +checksum = "d1c928609d087790fc936a1067bdc310ae702bdf3b090c3f281b713622c8bbde" dependencies = [ "async-trait", "futures", @@ -3087,23 +3161,78 @@ dependencies = [ "grpcio", "http", "opentelemetry", + "opentelemetry-proto", "prost", "protobuf", "thiserror", - "tokio 1.20.1", + "tokio 1.21.2", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61a2f56df5574508dd86aaca016c917489e589ece4141df1b5e349af8d66c28" +dependencies = [ + "futures", + "futures-util", + "grpcio", + "opentelemetry", + "prost", + "protobuf", "tonic", "tonic-build", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" +checksum = "9b02e0230abb0ab6636d18e2ba8fa02903ea63772281340ccac18e0af3ec9eeb" dependencies = [ "opentelemetry", ] +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap", + "js-sys", + "once_cell", + "pin-project-lite 0.2.9", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-std", + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand 0.8.5", + "thiserror", + "tokio 1.21.2", + "tokio-stream", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -3137,9 +3266,9 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "parity-scale-codec" -version = "3.1.5" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9182e4a71cae089267ab03e67c99368db7cd877baf50f931e5d6d4b71e195ac0" +checksum = "366e44391a8af4cfd6002ef6ba072bae071a96aafca98d7d448a34c5dca38b6a" dependencies = [ "arrayvec", "bitvec", @@ -3217,9 +3346,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9423e2b32f7a043629287a536f21951e8c6a82482d0acb1eeebfc90bc2225b22" +checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" [[package]] name = "pathdiff" @@ -3235,15 +3364,15 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0560d531d1febc25a3c9398a62a71256c0178f2e3443baedd9ad4bb8c9deb4" +checksum = "dbc7bc69c062e492337d74d59b120c274fd3d261b6bf6d3207d499b4b379c41a" dependencies = [ "thiserror", "ucd-trie", @@ -3251,9 +3380,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905708f7f674518498c1f8d644481440f476d39ca6ecae83319bba7c6c12da91" +checksum = "60b75706b9642ebcb34dab3bc7750f811609a0eb1dd8b88c2d15bf628c1c65b2" dependencies = [ "pest", "pest_generator", @@ -3261,9 +3390,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5803d8284a629cc999094ecd630f55e91b561a1d1ba75e233b00ae13b91a69ad" +checksum = "f4f9272122f5979a6511a749af9db9bfc810393f63119970d7085fed1c4ea0db" dependencies = [ "pest", "pest_meta", @@ -3274,13 +3403,13 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1538eb784f07615c6d9a8ab061089c6c54a344c5b4301db51990ca1c241e8c04" +checksum = "4c8717927f9b79515e565a64fe46c38b8cd0427e64c40680b14a7365ab09ac8d" dependencies = [ "once_cell", "pest", - "sha-1 0.10.0", + "sha1", ] [[package]] @@ -3369,9 +3498,9 @@ checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "plotters" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "716b4eeb6c4a1d3ecc956f75b43ec2e8e8ba80026413e70a3f41fd3313d3492b" +checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" dependencies = [ "num-traits", "plotters-backend", @@ -3427,10 +3556,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] -name = "primitive-types" -version = "0.11.1" +name = "prettyplease" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +checksum = "a49e86d2c26a24059894a3afa13fd17d063419b05dfb83f06d9c3566060c3f5a" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primitive-types" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cfd65aea0c5fa0bfcc7c9e7ca828c921ef778f43d325325ec84bda371bfa75a" dependencies = [ "fixed-hash", "impl-codec", @@ -3485,18 +3624,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.43" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" +checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +checksum = "399c3c31cdec40583bb68f0b18403400d01ec4289c383aa047560439952c4dd7" dependencies = [ "bytes 1.2.1", "prost-derive", @@ -3504,12 +3643,12 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.9.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb" dependencies = [ "bytes 1.2.1", - "heck 0.3.3", + "heck", "itertools", "lazy_static", "log", @@ -3524,9 +3663,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" dependencies = [ "anyhow", "itertools", @@ -3537,9 +3676,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.9.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +checksum = "4dfaa718ad76a44b3415e6c4d53b17c8f99160dcb3a99b10470fce8ad43f6e3e" dependencies = [ "bytes 1.2.1", "prost", @@ -3547,9 +3686,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.27.1" +version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "quick-error" @@ -3593,7 +3732,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha 0.3.1", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -3613,7 +3752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -3627,9 +3766,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom 0.2.7", ] @@ -3810,7 +3949,7 @@ dependencies = [ "netlink-proto", "nix 0.22.3", "thiserror", - "tokio 1.20.1", + "tokio 1.21.2", ] [[package]] @@ -3822,7 +3961,7 @@ dependencies = [ "bitflags", "fallible-iterator", "fallible-streaming-iterator", - "hashlink 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hashlink 0.8.1", "libsqlite3-sys", "smallvec", ] @@ -4002,9 +4141,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711" +checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "send_wrapper" @@ -4029,9 +4168,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.144" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" dependencies = [ "serde_derive", ] @@ -4057,9 +4196,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.144" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" dependencies = [ "proc-macro2", "quote", @@ -4072,7 +4211,7 @@ version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" dependencies = [ - "itoa 1.0.3", + "itoa", "ryu", "serde", ] @@ -4090,21 +4229,21 @@ dependencies = [ [[package]] name = "serde_test" -version = "1.0.144" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c7f3621491f256177206a7c2152c17f322c0d0b30af05359088172437d29e25" +checksum = "9c17d2112159132660b4c5399e274f676fb75a2f8d70b7468f18f045b71138ed" dependencies = [ "serde", ] [[package]] name = "serde_yaml" -version = "0.9.10" +version = "0.9.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a09f551ccc8210268ef848f0bab37b306e87b85b2e017b899e7fb815f5aed62" +checksum = "8613d593412a0deb7bbd8de9d908efff5a0cb9ccd8f62c641e7b2ed2f57291d1" dependencies = [ "indexmap", - "itoa 1.0.3", + "itoa", "ryu", "serde", "unsafe-libyaml", @@ -4157,7 +4296,18 @@ checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.3", + "digest 0.10.5", +] + +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.5", ] [[package]] @@ -4232,9 +4382,9 @@ dependencies = [ [[package]] name = "signature" -version = "1.6.0" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0ea32af43239f0d353a7dd75a22d94c329c8cdaafdcb4c1c1335aa10c298a4a" +checksum = "deb766570a2825fa972bceff0d195727876a9cdf2460ab2e52d455dc2de47fd9" [[package]] name = "simplelog" @@ -4255,7 +4405,7 @@ checksum = "48dfff04aade74dd495b007c831cd6f4e0cee19c344dd9dc0884c0289b70a786" dependencies = [ "log", "termcolor", - "time 0.3.9", + "time 0.3.14", ] [[package]] @@ -4269,9 +4419,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "snailquote" @@ -4285,9 +4435,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10c98bba371b9b22a71a9414e420f92ddeb2369239af08200816169d5e2dd7aa" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" dependencies = [ "libc", "winapi 0.3.9", @@ -4352,15 +4502,21 @@ checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" [[package]] name = "syn" -version = "1.0.99" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" +checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + [[package]] name = "synstructure" version = "0.12.6" @@ -4375,14 +4531,14 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.25.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71eb43e528fdc239f08717ec2a378fdb017dddbc3412de15fff527554591a66c" +checksum = "7890fff842b8db56f2033ebee8f6efe1921475c3830c115995552914fb967580" dependencies = [ "cfg-if 1.0.0", "core-foundation-sys 0.8.3", "libc", - "ntapi", + "ntapi 0.4.0", "once_cell", "rayon", "winapi 0.3.9", @@ -4419,33 +4575,24 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.11.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "textwrap" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" [[package]] name = "thiserror" -version = "1.0.32" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5f6586b7f764adc0231f4c79be7b920e766bb2f3e51b3661cdb263828f19994" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.32" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bafc5b54507e0149cdf1b145a5d80ab80a90bcd9275df43d4fff68460f6c21" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" dependencies = [ "proc-macro2", "quote", @@ -4474,11 +4621,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" dependencies = [ - "itoa 1.0.3", + "itoa", "libc", "num_threads", "time-macros", @@ -4554,9 +4701,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.20.1" +version = "1.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" dependencies = [ "autocfg", "bytes 1.2.1", @@ -4564,7 +4711,6 @@ dependencies = [ "memchr", "mio 0.8.4", "num_cpus", - "once_cell", "parking_lot 0.12.1", "pin-project-lite 0.2.9", "signal-hook-registry", @@ -4580,7 +4726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ "pin-project-lite 0.2.9", - "tokio 1.20.1", + "tokio 1.21.2", ] [[package]] @@ -4596,41 +4742,27 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" +checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" dependencies = [ "futures-core", "pin-project-lite 0.2.9", - "tokio 1.20.1", + "tokio 1.21.2", ] [[package]] name = "tokio-util" -version = "0.6.10" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes 1.2.1", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.9", - "tokio 1.20.1", -] - -[[package]] -name = "tokio-util" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ "bytes 1.2.1", "futures-core", "futures-io", "futures-sink", "pin-project-lite 0.2.9", - "tokio 1.20.1", + "tokio 1.21.2", "tracing", ] @@ -4645,12 +4777,13 @@ dependencies = [ [[package]] name = "tonic" -version = "0.6.2" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" +checksum = "55b9af819e54b8f33d453655bef9b9acc171568fb49523078d0cc4e7484200ec" dependencies = [ "async-stream", "async-trait", + "axum", "base64 0.13.0", "bytes 1.2.1", "futures-core", @@ -4664,9 +4797,9 @@ dependencies = [ "pin-project 1.0.12", "prost", "prost-derive", - "tokio 1.20.1", + "tokio 1.21.2", "tokio-stream", - "tokio-util 0.6.10", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -4676,10 +4809,11 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.6.2" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +checksum = "48c6fd7c2581e36d63388a9e04c350c21beb7a8b059580b2e93993c526899ddc" dependencies = [ + "prettyplease", "proc-macro2", "prost-build", "quote", @@ -4699,13 +4833,32 @@ dependencies = [ "pin-project-lite 0.2.9", "rand 0.8.5", "slab", - "tokio 1.20.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "tower-http" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +dependencies = [ + "bitflags", + "bytes 1.2.1", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite 0.2.9", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.1" @@ -4749,7 +4902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.9", + "time 0.3.14", "tracing-subscriber", ] @@ -4818,9 +4971,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.17.4" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" dependencies = [ "once_cell", "opentelemetry", @@ -4861,9 +5014,9 @@ dependencies = [ [[package]] name = "trust-dns-proto" -version = "0.21.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" +checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" dependencies = [ "async-trait", "cfg-if 1.0.0", @@ -4872,35 +5025,35 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna", + "idna 0.2.3", "ipnet", "lazy_static", - "log", "rand 0.8.5", "smallvec", "thiserror", "tinyvec", - "tokio 1.20.1", + "tokio 1.21.2", + "tracing", "url", ] [[package]] name = "trust-dns-resolver" -version = "0.21.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" +checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" dependencies = [ "cfg-if 1.0.0", "futures-util", "ipconfig", "lazy_static", - "log", "lru-cache", "parking_lot 0.12.1", "resolv-conf", "smallvec", "thiserror", - "tokio 1.20.1", + "tokio 1.21.2", + "tracing", "trust-dns-proto", ] @@ -4956,15 +5109,15 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "ucd-trie" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89570599c4fe5585de2b388aab47e99f7fa4e9238a1399f707a02e356058141c" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" [[package]] name = "uint" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f03af7ccf01dd611cc450a0d10dbc9b745770d096473e2faf0ca6e2d66d1e0" +checksum = "a45526d29728d135c2900b0d30573fe3ee79fceb12ef534c7bb30e810a91b601" dependencies = [ "byteorder", "crunchy", @@ -4980,36 +5133,36 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" +checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" [[package]] name = "unicode-normalization" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" [[package]] name = "unicode-width" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unicode_categories" @@ -5029,9 +5182,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "931179334a56395bcf64ba5e0ff56781381c1a5832178280c7d7f91d1679aeb0" +checksum = "c1e5fa573d8ac5f1a856f8d7be41d390ee973daf97c806b2c1a465e4e1406e68" [[package]] name = "untrusted" @@ -5041,13 +5194,12 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna", - "matches", + "idna 0.3.0", "percent-encoding", ] @@ -5090,7 +5242,7 @@ dependencies = [ "capnp-rpc", "capnpc", "cfg-if 1.0.0", - "clap 3.2.17", + "clap", "config", "crossbeam-channel", "cursive", @@ -5107,8 +5259,8 @@ dependencies = [ "serde_derive", "serial_test", "thiserror", - "tokio 1.20.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util", "veilid-core", ] @@ -5187,9 +5339,9 @@ dependencies = [ "static_assertions", "stop-token", "thiserror", - "tokio 1.20.1", + "tokio 1.21.2", "tokio-stream", - "tokio-util 0.7.3", + "tokio-util", "tracing", "tracing-android", "tracing-error", @@ -5202,7 +5354,7 @@ dependencies = [ "wasm-logger", "web-sys", "webpki 0.22.0", - "webpki-roots 0.22.4", + "webpki-roots 0.22.5", "wee_alloc", "winapi 0.3.9", "windows", @@ -5231,9 +5383,9 @@ dependencies = [ "parking_lot 0.12.1", "serde", "serde_json", - "tokio 1.20.1", + "tokio 1.21.2", "tokio-stream", - "tokio-util 0.7.3", + "tokio-util", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -5252,7 +5404,7 @@ dependencies = [ "capnp-rpc", "capnpc", "cfg-if 1.0.0", - "clap 3.2.17", + "clap", "color-eyre", "config", "ctrlc", @@ -5275,9 +5427,9 @@ dependencies = [ "signal-hook", "signal-hook-async-std", "stop-token", - "tokio 1.20.1", + "tokio 1.21.2", "tokio-stream", - "tokio-util 0.7.3", + "tokio-util", "tracing", "tracing-appender", "tracing-journald", @@ -5364,9 +5516,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if 1.0.0", "serde", @@ -5376,9 +5528,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ "bumpalo", "log", @@ -5391,9 +5543,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -5403,9 +5555,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5413,9 +5565,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", @@ -5426,15 +5578,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" [[package]] name = "wasm-bindgen-test" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513df541345bb9fcc07417775f3d51bbb677daf307d8035c0afafd87dc2e6599" +checksum = "09d2fff962180c3fadf677438054b1db62bee4aa32af26a45388af07d1287e1d" dependencies = [ "console_error_panic_hook", "js-sys", @@ -5446,9 +5598,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6150d36a03e90a3cf6c12650be10626a9902d70c5270fd47d7a47e5389a10d56" +checksum = "4683da3dfc016f704c9f82cf401520c4f1cb3ee440f7f52b3d6ac29506a49ca7" dependencies = [ "proc-macro2", "quote", @@ -5467,9 +5619,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.59" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" dependencies = [ "js-sys", "wasm-bindgen", @@ -5506,9 +5658,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" +checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" dependencies = [ "webpki 0.22.0", ] @@ -5536,13 +5688,13 @@ dependencies = [ [[package]] name = "which" -version = "4.2.5" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" dependencies = [ "either", - "lazy_static", "libc", + "once_cell", ] [[package]] @@ -5768,7 +5920,7 @@ checksum = "bf7074de8999662970c3c4c8f7f30925028dd8f4ca31ad4c055efa9cdf2ec326" dependencies = [ "curve25519-dalek-ng", "rand 0.8.5", - "rand_core 0.6.3", + "rand_core 0.6.4", "zeroize", ] diff --git a/setup_linux.sh b/setup_linux.sh index cf6c4cd9..59677a64 100755 --- a/setup_linux.sh +++ b/setup_linux.sh @@ -77,4 +77,5 @@ rustup target add aarch64-linux-android armv7-linux-androideabi i686-linux-andro cargo install wasm-bindgen-cli # Ensure packages are installed -sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk llvm wabt capnproto +sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk llvm wabt capnproto protobuf-compiler + diff --git a/setup_macos.sh b/setup_macos.sh index fc24a87e..501d08b2 100755 --- a/setup_macos.sh +++ b/setup_macos.sh @@ -114,5 +114,5 @@ if [ "$BREW_USER" == "" ]; then BREW_USER=`whoami` fi fi -sudo -H -u $BREW_USER brew install capnp cmake wabt llvm +sudo -H -u $BREW_USER brew install capnp cmake wabt llvm protobuf diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index 2bceb171..7e2923b7 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -26,6 +26,7 @@ struct ConnectionManagerArc { connection_initial_timeout_ms: u32, connection_inactivity_timeout_ms: u32, connection_table: ConnectionTable, + address_lock_table: AsyncTagLockTable, inner: Mutex>, } impl core::fmt::Debug for ConnectionManagerArc { @@ -69,6 +70,7 @@ impl ConnectionManager { connection_initial_timeout_ms, connection_inactivity_timeout_ms, connection_table: ConnectionTable::new(config), + address_lock_table: AsyncTagLockTable::new(), inner: Mutex::new(None), } } @@ -196,7 +198,7 @@ impl ConnectionManager { } // Returns a network connection if one already is established - #[instrument(level = "trace", skip(self), ret)] + //#[instrument(level = "trace", skip(self), ret)] pub fn get_connection(&self, descriptor: ConnectionDescriptor) -> Option { self.arc .connection_table @@ -236,11 +238,6 @@ impl ConnectionManager { did_kill } - /// Locak remote address - // async fn lock_remote_address(&self, remote_addr: SocketAddr) -> { - - // } - /// Called when we want to create a new connection or get the current one that already exists /// This will kill off any connections that are in conflict with the new connection to be made /// in order to make room for the new connection in the system's connection table @@ -251,18 +248,17 @@ impl ConnectionManager { local_addr: Option, dial_info: DialInfo, ) -> EyreResult> { - warn!( + // Async lock on the remote address for atomicity per remote + let peer_address = dial_info.to_peer_address(); + let remote_addr = peer_address.to_socket_addr(); + let _lock_guard = self.arc.address_lock_table.lock_tag(remote_addr); + + log_net!( "== get_or_create_connection local_addr={:?} dial_info={:?}", local_addr.green(), dial_info.green() ); - // Make a connection descriptor for this dialinfo - let peer_address = dial_info.to_peer_address(); - - // Async lock on the remote address for atomicity - //let _lock_guard = self.lock_remote_address(peer_address.to_socket_addr()); - // Kill off any possibly conflicting connections let did_kill = self.kill_off_colliding_connections(&dial_info).await; let mut retry_count = if did_kill { 2 } else { 0 }; @@ -299,6 +295,22 @@ impl ConnectionManager { } Err(e) => { if retry_count == 0 { + // Try one last time to return a connection from the table, in case + // an 'accept' happened at literally the same time as our connect + if let Some(conn) = self + .arc + .connection_table + .get_last_connection_by_remote(peer_address) + { + log_net!( + "== Returning existing connection in race local_addr={:?} peer_address={:?}", + local_addr.green(), + peer_address.green() + ); + + return Ok(NetworkResult::Value(conn)); + } + return Err(e).wrap_err("failed to connect"); } } diff --git a/veilid-core/src/network_manager/connection_table.rs b/veilid-core/src/network_manager/connection_table.rs index c16214f3..82d13ae3 100644 --- a/veilid-core/src/network_manager/connection_table.rs +++ b/veilid-core/src/network_manager/connection_table.rs @@ -144,7 +144,7 @@ impl ConnectionTable { let mut out_conn = None; if inner.conn_by_id[protocol_index].len() > inner.max_connections[protocol_index] { if let Some((lruk, lru_conn)) = inner.conn_by_id[protocol_index].remove_lru() { - debug!("connection lru out: {:?}", lru_conn); + log_net!(debug "connection lru out: {:?}", lru_conn); out_conn = Some(lru_conn); Self::remove_connection_records(&mut *inner, lruk); } @@ -158,7 +158,8 @@ impl ConnectionTable { Ok(out_conn) } - #[instrument(level = "trace", skip(self), ret)] + //#[instrument(level = "trace", skip(self), ret)] + #[allow(dead_code)] pub fn get_connection_by_id(&self, id: NetworkConnectionId) -> Option { let mut inner = self.inner.lock(); let protocol_index = *inner.protocol_index_by_id.get(&id)?; @@ -166,7 +167,7 @@ impl ConnectionTable { Some(out.get_handle()) } - #[instrument(level = "trace", skip(self), ret)] + //#[instrument(level = "trace", skip(self), ret)] pub fn get_connection_by_descriptor( &self, descriptor: ConnectionDescriptor, @@ -179,7 +180,7 @@ impl ConnectionTable { Some(out.get_handle()) } - #[instrument(level = "trace", skip(self), ret)] + //#[instrument(level = "trace", skip(self), ret)] pub fn get_last_connection_by_remote(&self, remote: PeerAddress) -> Option { let mut inner = self.inner.lock(); @@ -189,7 +190,8 @@ impl ConnectionTable { Some(out.get_handle()) } - #[instrument(level = "trace", skip(self), ret)] + //#[instrument(level = "trace", skip(self), ret)] + #[allow(dead_code)] pub fn get_connection_ids_by_remote(&self, remote: PeerAddress) -> Vec { let inner = self.inner.lock(); inner diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 9b2d9f18..1a132f38 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1949,47 +1949,53 @@ impl NetworkManager { .clone() .unlocked_inner .node_info_update_single_future - .single_spawn(async move { - // Only update if we actually have valid signed node info for this routing domain - if !this.routing_table().has_valid_own_node_info(routing_domain) { - trace!( + .single_spawn( + async move { + // Only update if we actually have valid signed node info for this routing domain + if !this.routing_table().has_valid_own_node_info(routing_domain) { + trace!( "not sending node info update because our network class is not yet valid" ); - return; + return; + } + + // Get the list of refs to all nodes to update + let cur_ts = intf::get_timestamp(); + let node_refs = + this.routing_table() + .get_nodes_needing_updates(routing_domain, cur_ts, all); + + // Send the updates + log_net!(debug "Sending node info updates to {} nodes", node_refs.len()); + let mut unord = FuturesUnordered::new(); + for nr in node_refs { + let rpc = this.rpc_processor(); + unord.push( + async move { + // Update the node + if let Err(e) = rpc + .rpc_call_node_info_update(nr.clone(), routing_domain) + .await + { + // Not fatal, but we should be able to see if this is happening + trace!("failed to send node info update to {:?}: {}", nr, e); + return; + } + + // Mark the node as having seen our node info + nr.set_seen_our_node_info(routing_domain); + } + .instrument(Span::current()), + ); + } + + // Wait for futures to complete + while unord.next().await.is_some() {} + + log_rtab!(debug "Finished sending node updates"); } - - // Get the list of refs to all nodes to update - let cur_ts = intf::get_timestamp(); - let node_refs = - this.routing_table() - .get_nodes_needing_updates(routing_domain, cur_ts, all); - - // Send the updates - log_net!(debug "Sending node info updates to {} nodes", node_refs.len()); - let mut unord = FuturesUnordered::new(); - for nr in node_refs { - let rpc = this.rpc_processor(); - unord.push(async move { - // Update the node - if let Err(e) = rpc - .rpc_call_node_info_update(nr.clone(), routing_domain) - .await - { - // Not fatal, but we should be able to see if this is happening - trace!("failed to send node info update to {:?}: {}", nr, e); - return; - } - - // Mark the node as having seen our node info - nr.set_seen_our_node_info(routing_domain); - }); - } - - // Wait for futures to complete - while unord.next().await.is_some() {} - - log_rtab!(debug "Finished sending node updates"); - }) + .instrument(Span::current()), + ) .await; } } diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/network_class_discovery.rs index b4c1568d..ed95e0d1 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/network_class_discovery.rs @@ -255,6 +255,13 @@ impl DiscoveryContext { { return Some(external_mapped_dial_info); } else { + warn!("UPNP port mapping succeeded but port {}/{} is still unreachable.\nYou may need to add a local firewall allowed port on this machine.\n", + local_port, match llpt { + LowLevelProtocolType::UDP => "udp", + LowLevelProtocolType::TCP => "tcp", + } + ); + // release the mapping if we're still unreachable let _ = self .net @@ -628,6 +635,7 @@ impl Network { } Some(vec![udpv4_context]) } + .instrument(trace_span!("do_public_dial_info_check UDPv4")) .boxed(), ); } @@ -647,6 +655,7 @@ impl Network { } Some(vec![udpv6_context]) } + .instrument(trace_span!("do_public_dial_info_check UDPv6")) .boxed(), ); } @@ -669,6 +678,7 @@ impl Network { } Some(vec![tcpv4_context]) } + .instrument(trace_span!("do_public_dial_info_check TCPv4")) .boxed(), ); } @@ -688,6 +698,7 @@ impl Network { } Some(vec![wsv4_context]) } + .instrument(trace_span!("do_public_dial_info_check WSv4")) .boxed(), ); } @@ -710,6 +721,7 @@ impl Network { } Some(vec![tcpv6_context]) } + .instrument(trace_span!("do_public_dial_info_check TCPv6")) .boxed(), ); } @@ -729,6 +741,7 @@ impl Network { } Some(vec![wsv6_context]) } + .instrument(trace_span!("do_public_dial_info_check WSv6")) .boxed(), ); } diff --git a/veilid-core/src/network_manager/native/network_tcp.rs b/veilid-core/src/network_manager/native/network_tcp.rs index 806abc2c..ebca4378 100644 --- a/veilid-core/src/network_manager/native/network_tcp.rs +++ b/veilid-core/src/network_manager/native/network_tcp.rs @@ -109,11 +109,11 @@ impl Network { }; // XXX - warn!( - "DEBUGACCEPT: local={} remote={}", - tcp_stream.local_addr().unwrap(), - tcp_stream.peer_addr().unwrap(), - ); + // warn!( + // "DEBUGACCEPT: local={} remote={}", + // tcp_stream.local_addr().unwrap(), + // tcp_stream.peer_addr().unwrap(), + // ); let listener_state = listener_state.clone(); let connection_manager = connection_manager.clone(); diff --git a/veilid-core/src/network_manager/native/network_udp.rs b/veilid-core/src/network_manager/native/network_udp.rs index b00bf643..66c491bb 100644 --- a/veilid-core/src/network_manager/native/network_udp.rs +++ b/veilid-core/src/network_manager/native/network_udp.rs @@ -84,7 +84,7 @@ impl Network { } } } - }; + }.instrument(Span::current()); protocol_handlers_unordered.push(ph_future); } diff --git a/veilid-core/src/network_manager/native/protocol/sockets.rs b/veilid-core/src/network_manager/native/protocol/sockets.rs index 0650d686..1c7b9d26 100644 --- a/veilid-core/src/network_manager/native/protocol/sockets.rs +++ b/veilid-core/src/network_manager/native/protocol/sockets.rs @@ -185,7 +185,7 @@ pub async fn nonblocking_connect( let socket2_addr = socket2::SockAddr::from(addr); // XXX - let bind_local_addr = socket.local_addr().unwrap().as_socket().unwrap(); + //let bind_local_addr = socket.local_addr().unwrap().as_socket().unwrap(); // Connect to the remote address match socket.connect(&socket2_addr) { @@ -197,24 +197,24 @@ pub async fn nonblocking_connect( } .map_err(|e| { // XXX - warn!( - "DEBUGCONNECT XXXFAILXXX: bind={} local={} remote={}\nbacktrace={:?}", - bind_local_addr, - socket.local_addr().unwrap().as_socket().unwrap(), - addr, - backtrace::Backtrace::new(), - ); + // warn!( + // "DEBUGCONNECT XXXFAILXXX: bind={} local={} remote={}\nbacktrace={:?}", + // bind_local_addr, + // socket.local_addr().unwrap().as_socket().unwrap(), + // addr, + // backtrace::Backtrace::new(), + // ); e })?; // XXX - warn!( - "DEBUGCONNECT: bind={} local={} remote={}\nbacktrace={:?}", - bind_local_addr, - socket.local_addr().unwrap().as_socket().unwrap(), - addr, - backtrace::Backtrace::new(), - ); + // warn!( + // "DEBUGCONNECT: bind={} local={} remote={}\nbacktrace={:?}", + // bind_local_addr, + // socket.local_addr().unwrap().as_socket().unwrap(), + // addr, + // backtrace::Backtrace::new(), + // ); let async_stream = Async::new(std::net::TcpStream::from(socket))?; diff --git a/veilid-core/src/network_manager/network_connection.rs b/veilid-core/src/network_manager/network_connection.rs index b9658b4a..cffc91ae 100644 --- a/veilid-core/src/network_manager/network_connection.rs +++ b/veilid-core/src/network_manager/network_connection.rs @@ -210,11 +210,13 @@ impl NetworkConnection { Ok(NetworkResult::Value(out)) } + #[allow(dead_code)] pub fn stats(&self) -> NetworkConnectionStats { let stats = self.stats.lock(); stats.clone() } + #[allow(dead_code)] pub fn established_time(&self) -> u64 { self.established_time } @@ -260,10 +262,11 @@ impl NetworkConnection { need_sender = false; let sender_fut = receiver.recv_async().then(|res| async { match res { - Ok((span_id, message)) => { + Ok((_span_id, message)) => { - let recv_span = span!(parent: None, Level::TRACE, "process_connection recv"); - recv_span.follows_from(span_id); + let recv_span = span!(Level::TRACE, "process_connection recv"); + // xxx: causes crash (Missing otel data span extensions) + // recv_span.follows_from(span_id); // send the packet if let Err(e) = Self::send_internal( diff --git a/veilid-core/src/receipt_manager.rs b/veilid-core/src/receipt_manager.rs index b3d76139..4ad56db9 100644 --- a/veilid-core/src/receipt_manager.rs +++ b/veilid-core/src/receipt_manager.rs @@ -246,7 +246,7 @@ impl ReceiptManager { if let Some(callback) = Self::perform_callback(ReceiptEvent::Expired, &mut expired_record_mut) { - callbacks.push(callback) + callbacks.push(callback.instrument(Span::current())) } } diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 89fa46c1..25bb62e1 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -264,32 +264,62 @@ impl BucketEntryInner { self.last_connections.clear(); } - // Gets the best 'last connection' that matches a set of routing domain, protocol types and address types - pub(super) fn last_connection( + // Gets the 'last connection' that matches a specific connection key + // pub(super) fn last_connection( + // &self, + // protocol_type: ProtocolType, + // address_type: AddressType, + // ) -> Option<(ConnectionDescriptor, u64)> { + // let key = LastConnectionKey(protocol_type, address_type); + // self.last_connections.get(&key).cloned() + // } + + // Gets all the 'last connections' that match a particular filter + pub(super) fn last_connections( &self, routing_table_inner: &RoutingTableInner, - node_ref_filter: Option, - ) -> Option<(ConnectionDescriptor, u64)> { - // Iterate peer scopes and protocol types and address type in order to ensure we pick the preferred protocols if all else is the same - let nrf = node_ref_filter.unwrap_or_default(); - for pt in nrf.dial_info_filter.protocol_type_set { - for at in nrf.dial_info_filter.address_type_set { - let key = LastConnectionKey(pt, at); - if let Some(v) = self.last_connections.get(&key) { - // Verify this connection could be in the filtered routing domain - let address = v.0.remote_address().address(); - if let Some(rd) = - RoutingTable::routing_domain_for_address_inner(routing_table_inner, address) - { - if nrf.routing_domain_set.contains(rd) { - return Some(*v); + filter: Option, + ) -> Vec<(ConnectionDescriptor, u64)> { + let mut out: Vec<(ConnectionDescriptor, u64)> = self + .last_connections + .iter() + .filter_map(|(k, v)| { + let include = if let Some(filter) = &filter { + let remote_address = v.0.remote_address().address(); + if let Some(routing_domain) = RoutingTable::routing_domain_for_address_inner( + routing_table_inner, + remote_address, + ) { + if filter.routing_domain_set.contains(routing_domain) + && filter.dial_info_filter.protocol_type_set.contains(k.0) + && filter.dial_info_filter.address_type_set.contains(k.1) + { + // matches filter + true + } else { + // does not match filter + false } + } else { + // no valid routing domain + false } + } else { + // no filter + true + }; + if include { + Some(v.clone()) + } else { + None } - } - } - None + }) + .collect(); + // Sort with newest timestamps first + out.sort_by(|a, b| b.1.cmp(&a.1)); + out } + pub fn set_min_max_version(&mut self, min_max_version: (u8, u8)) { self.min_max_version = Some(min_max_version); } diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 0b64de14..96b90393 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -318,24 +318,29 @@ impl NodeRef { } pub fn last_connection(&self) -> Option { - // Get the last connection and the last time we saw anything with this connection - let (last_connection, last_seen) = - self.operate(|rti, e| e.last_connection(rti, self.filter.clone()))?; + // Get the last connections and the last time we saw anything with this connection + // Filtered first and then sorted by most recent + let last_connections = self.operate(|rti, e| e.last_connections(rti, self.filter.clone())); - // Should we check the connection table? - if last_connection.protocol_type().is_connection_oriented() { - // Look the connection up in the connection manager and see if it's still there - let connection_manager = self.routing_table.network_manager().connection_manager(); - connection_manager.get_connection(last_connection)?; - } else { - // If this is not connection oriented, then we check our last seen time - // to see if this mapping has expired (beyond our timeout) - let cur_ts = intf::get_timestamp(); - if (last_seen + (CONNECTIONLESS_TIMEOUT_SECS as u64 * 1_000_000u64)) < cur_ts { - return None; + // Do some checks to ensure these are possibly still 'live' + for (last_connection, last_seen) in last_connections { + // Should we check the connection table? + if last_connection.protocol_type().is_connection_oriented() { + // Look the connection up in the connection manager and see if it's still there + let connection_manager = self.routing_table.network_manager().connection_manager(); + if connection_manager.get_connection(last_connection).is_some() { + return Some(last_connection); + } + } else { + // If this is not connection oriented, then we check our last seen time + // to see if this mapping has expired (beyond our timeout) + let cur_ts = intf::get_timestamp(); + if (last_seen + (CONNECTIONLESS_TIMEOUT_SECS as u64 * 1_000_000u64)) >= cur_ts { + return Some(last_connection); + } } } - Some(last_connection) + None } pub fn clear_last_connections(&self) { diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 63a36b35..9de635db 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -941,13 +941,12 @@ impl RPCProcessor { stop_token: StopToken, receiver: flume::Receiver<(Option, RPCMessageEncoded)>, ) { - while let Ok(Ok((span_id, msg))) = + while let Ok(Ok((_span_id, msg))) = receiver.recv_async().timeout_at(stop_token.clone()).await { - let rpc_worker_span = span!(parent: None, Level::TRACE, "rpc_worker"); - //let rpc_worker_span = span!(Level::TRACE, "rpc_worker"); - // fixme: causes crashes? "Missing otel data span extensions"?? - rpc_worker_span.follows_from(span_id); + let rpc_worker_span = span!(parent: None, Level::TRACE, "rpc_worker recv"); + // xxx: causes crash (Missing otel data span extensions) + // rpc_worker_span.follows_from(span_id); let _ = self .process_rpc_message(msg) .instrument(rpc_worker_span) diff --git a/veilid-core/src/rpc_processor/operation_waiter.rs b/veilid-core/src/rpc_processor/operation_waiter.rs index cb5d4afa..4dc71332 100644 --- a/veilid-core/src/rpc_processor/operation_waiter.rs +++ b/veilid-core/src/rpc_processor/operation_waiter.rs @@ -127,8 +127,8 @@ where let (_span_id, ret) = res.take_value().unwrap(); let end_ts = intf::get_timestamp(); - // fixme: causes crashes? "Missing otel data span extensions"?? - //Span::current().follows_from(span_id); + //xxx: causes crash (Missing otel data span extensions) + // Span::current().follows_from(span_id); (ret, end_ts - start_ts) })) From 7ed6b44d21334ead8a4672a01020bf28e8666f0e Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 4 Oct 2022 13:09:03 -0400 Subject: [PATCH 05/67] better race condition handling --- .../src/network_manager/connection_manager.rs | 27 +++++----- veilid-core/src/network_manager/mod.rs | 53 ++++++++++++++++--- veilid-core/src/network_manager/native/mod.rs | 12 +++-- .../src/network_manager/native/network_udp.rs | 2 +- .../native/protocol/sockets.rs | 29 ++-------- veilid-core/src/routing_table/mod.rs | 17 +++++- veilid-core/src/veilid_api/debug.rs | 3 ++ veilid-core/src/veilid_api/routing_context.rs | 5 +- veilid-core/src/xx/network_result.rs | 28 +++++++++- 9 files changed, 120 insertions(+), 56 deletions(-) diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index 7e2923b7..f535ea5d 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -289,28 +289,29 @@ impl ConnectionManager { .await; match result_net_res { Ok(net_res) => { - if net_res.is_value() || retry_count == 0 { - break net_res; - } - } - Err(e) => { - if retry_count == 0 { - // Try one last time to return a connection from the table, in case - // an 'accept' happened at literally the same time as our connect + // If the connection 'already exists', then try one last time to return a connection from the table, in case + // an 'accept' happened at literally the same time as our connect + if net_res.is_already_exists() { if let Some(conn) = self .arc .connection_table .get_last_connection_by_remote(peer_address) { log_net!( - "== Returning existing connection in race local_addr={:?} peer_address={:?}", - local_addr.green(), - peer_address.green() - ); + "== Returning existing connection in race local_addr={:?} peer_address={:?}", + local_addr.green(), + peer_address.green() + ); return Ok(NetworkResult::Value(conn)); } - + } + if net_res.is_value() || retry_count == 0 { + break net_res; + } + } + Err(e) => { + if retry_count == 0 { return Err(e).wrap_err("failed to connect"); } } diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 1a132f38..80301d6c 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -223,7 +223,15 @@ impl NetworkManager { this.unlocked_inner .rolling_transfers_task .set_routine(move |s, l, t| { - Box::pin(this2.clone().rolling_transfers_task_routine(s, l, t)) + Box::pin( + this2 + .clone() + .rolling_transfers_task_routine(s, l, t) + .instrument(trace_span!( + parent: None, + "NetworkManager rolling transfers task routine" + )), + ) }); } // Set relay management tick task @@ -232,7 +240,12 @@ impl NetworkManager { this.unlocked_inner .relay_management_task .set_routine(move |s, l, t| { - Box::pin(this2.clone().relay_management_task_routine(s, l, t)) + Box::pin( + this2 + .clone() + .relay_management_task_routine(s, l, t) + .instrument(trace_span!(parent: None, "relay management task routine")), + ) }); } // Set bootstrap tick task @@ -240,7 +253,14 @@ impl NetworkManager { let this2 = this.clone(); this.unlocked_inner .bootstrap_task - .set_routine(move |s, _l, _t| Box::pin(this2.clone().bootstrap_task_routine(s))); + .set_routine(move |s, _l, _t| { + Box::pin( + this2 + .clone() + .bootstrap_task_routine(s) + .instrument(trace_span!(parent: None, "bootstrap task routine")), + ) + }); } // Set peer minimum refresh tick task { @@ -248,7 +268,15 @@ impl NetworkManager { this.unlocked_inner .peer_minimum_refresh_task .set_routine(move |s, _l, _t| { - Box::pin(this2.clone().peer_minimum_refresh_task_routine(s)) + Box::pin( + this2 + .clone() + .peer_minimum_refresh_task_routine(s) + .instrument(trace_span!( + parent: None, + "peer minimum refresh task routine" + )), + ) }); } // Set ping validator tick task @@ -257,7 +285,12 @@ impl NetworkManager { this.unlocked_inner .ping_validator_task .set_routine(move |s, l, t| { - Box::pin(this2.clone().ping_validator_task_routine(s, l, t)) + Box::pin( + this2 + .clone() + .ping_validator_task_routine(s, l, t) + .instrument(trace_span!(parent: None, "ping validator task routine")), + ) }); } // Set public address check task @@ -266,7 +299,15 @@ impl NetworkManager { this.unlocked_inner .public_address_check_task .set_routine(move |s, l, t| { - Box::pin(this2.clone().public_address_check_task_routine(s, l, t)) + Box::pin( + this2 + .clone() + .public_address_check_task_routine(s, l, t) + .instrument(trace_span!( + parent: None, + "public address check task routine" + )), + ) }); } this diff --git a/veilid-core/src/network_manager/native/mod.rs b/veilid-core/src/network_manager/native/mod.rs index 868561fa..1a58218f 100644 --- a/veilid-core/src/network_manager/native/mod.rs +++ b/veilid-core/src/network_manager/native/mod.rs @@ -462,11 +462,13 @@ impl Network { // receive single response let mut out = vec![0u8; MAX_MESSAGE_SIZE]; - let (recv_len, recv_addr) = - network_result_try!(timeout(timeout_ms, h.recv_message(&mut out)) - .await - .into_network_result()) - .wrap_err("recv_message failure")?; + let (recv_len, recv_addr) = network_result_try!(timeout( + timeout_ms, + h.recv_message(&mut out).instrument(Span::current()) + ) + .await + .into_network_result()) + .wrap_err("recv_message failure")?; let recv_socket_addr = recv_addr.remote_address().to_socket_addr(); self.network_manager() diff --git a/veilid-core/src/network_manager/native/network_udp.rs b/veilid-core/src/network_manager/native/network_udp.rs index 66c491bb..0d906f16 100644 --- a/veilid-core/src/network_manager/native/network_udp.rs +++ b/veilid-core/src/network_manager/native/network_udp.rs @@ -54,7 +54,7 @@ impl Network { loop { match ph - .recv_message(&mut data) + .recv_message(&mut data).instrument(Span::current()) .timeout_at(stop_token.clone()) .await { diff --git a/veilid-core/src/network_manager/native/protocol/sockets.rs b/veilid-core/src/network_manager/native/protocol/sockets.rs index 1c7b9d26..c8918e33 100644 --- a/veilid-core/src/network_manager/native/protocol/sockets.rs +++ b/veilid-core/src/network_manager/native/protocol/sockets.rs @@ -172,7 +172,8 @@ pub fn new_bound_first_tcp_socket(local_address: SocketAddr) -> io::Result Ok(()), @@ -194,28 +192,7 @@ pub async fn nonblocking_connect( Err(err) if err.raw_os_error() == Some(libc::EINPROGRESS) => Ok(()), Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => Ok(()), Err(e) => Err(e), - } - .map_err(|e| { - // XXX - // warn!( - // "DEBUGCONNECT XXXFAILXXX: bind={} local={} remote={}\nbacktrace={:?}", - // bind_local_addr, - // socket.local_addr().unwrap().as_socket().unwrap(), - // addr, - // backtrace::Backtrace::new(), - // ); - e - })?; - - // XXX - // warn!( - // "DEBUGCONNECT: bind={} local={} remote={}\nbacktrace={:?}", - // bind_local_addr, - // socket.local_addr().unwrap().as_socket().unwrap(), - // addr, - // backtrace::Backtrace::new(), - // ); - + }?; let async_stream = Async::new(std::net::TcpStream::from(socket))?; // The stream becomes writable when connected diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 6a9cb6a8..cc831641 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -125,7 +125,15 @@ impl RoutingTable { this.unlocked_inner .rolling_transfers_task .set_routine(move |s, l, t| { - Box::pin(this2.clone().rolling_transfers_task_routine(s, l, t)) + Box::pin( + this2 + .clone() + .rolling_transfers_task_routine(s, l, t) + .instrument(trace_span!( + parent: None, + "RoutingTable rolling transfers task routine" + )), + ) }); } @@ -135,7 +143,12 @@ impl RoutingTable { this.unlocked_inner .kick_buckets_task .set_routine(move |s, l, t| { - Box::pin(this2.clone().kick_buckets_task_routine(s, l, t)) + Box::pin( + this2 + .clone() + .kick_buckets_task_routine(s, l, t) + .instrument(trace_span!(parent: None, "kick buckets task routine")), + ) }); } this diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 39b128ab..16b7b6b7 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -406,6 +406,9 @@ impl VeilidAPI { NetworkResult::NoConnection(e) => { return Ok(format!("NoConnection({})", e)); } + NetworkResult::AlreadyExists(e) => { + return Ok(format!("AlreadyExists({})", e)); + } NetworkResult::InvalidMessage(e) => { return Ok(format!("InvalidMessage({})", e)); } diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index 5cfac250..b095fd3a 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -130,11 +130,12 @@ impl RoutingContext { let answer = match rpc_processor.rpc_call_app_call(dest, request).await { Ok(NetworkResult::Value(v)) => v, Ok(NetworkResult::Timeout) => return Err(VeilidAPIError::Timeout), - Ok(NetworkResult::NoConnection(e)) => { + Ok(NetworkResult::NoConnection(e)) | Ok(NetworkResult::AlreadyExists(e)) => { return Err(VeilidAPIError::NoConnection { message: e.to_string(), }) } + Ok(NetworkResult::InvalidMessage(message)) => { return Err(VeilidAPIError::Generic { message }) } @@ -159,7 +160,7 @@ impl RoutingContext { match rpc_processor.rpc_call_app_message(dest, message).await { Ok(NetworkResult::Value(())) => {} Ok(NetworkResult::Timeout) => return Err(VeilidAPIError::Timeout), - Ok(NetworkResult::NoConnection(e)) => { + Ok(NetworkResult::NoConnection(e)) | Ok(NetworkResult::AlreadyExists(e)) => { return Err(VeilidAPIError::NoConnection { message: e.to_string(), }) diff --git a/veilid-core/src/xx/network_result.rs b/veilid-core/src/xx/network_result.rs index 579a2a80..e4c72ba1 100644 --- a/veilid-core/src/xx/network_result.rs +++ b/veilid-core/src/xx/network_result.rs @@ -35,6 +35,7 @@ impl IoNetworkResultExt for io::Result { | io::ErrorKind::ConnectionReset | io::ErrorKind::HostUnreachable | io::ErrorKind::NetworkUnreachable => Ok(NetworkResult::NoConnection(e)), + io::ErrorKind::AddrNotAvailable => Ok(NetworkResult::AlreadyExists(e)), _ => Err(e), }, #[cfg(not(feature = "io_error_more"))] @@ -50,6 +51,7 @@ impl IoNetworkResultExt for io::Result { io::ErrorKind::ConnectionAborted | io::ErrorKind::ConnectionRefused | io::ErrorKind::ConnectionReset => Ok(NetworkResult::NoConnection(e)), + io::ErrorKind::AddrNotAvailable => Ok(NetworkResult::AlreadyExists(e)), _ => Err(e), } } @@ -66,6 +68,7 @@ impl NetworkResultResultExt for NetworkResult> { match self { NetworkResult::Timeout => Ok(NetworkResult::::Timeout), NetworkResult::NoConnection(e) => Ok(NetworkResult::::NoConnection(e)), + NetworkResult::AlreadyExists(e) => Ok(NetworkResult::::AlreadyExists(e)), NetworkResult::InvalidMessage(s) => Ok(NetworkResult::::InvalidMessage(s)), NetworkResult::Value(Ok(v)) => Ok(NetworkResult::::Value(v)), NetworkResult::Value(Err(e)) => Err(e), @@ -90,6 +93,7 @@ impl FoldedNetworkResultExt for io::Result> { | io::ErrorKind::ConnectionReset | io::ErrorKind::HostUnreachable | io::ErrorKind::NetworkUnreachable => Ok(NetworkResult::NoConnection(e)), + io::ErrorKind::AddrNotAvailable => Ok(NetworkResult::AlreadyExists(e)), _ => Err(e), }, #[cfg(not(feature = "io_error_more"))] @@ -105,6 +109,7 @@ impl FoldedNetworkResultExt for io::Result> { io::ErrorKind::ConnectionAborted | io::ErrorKind::ConnectionRefused | io::ErrorKind::ConnectionReset => Ok(NetworkResult::NoConnection(e)), + io::ErrorKind::AddrNotAvailable => Ok(NetworkResult::AlreadyExists(e)), _ => Err(e), } } @@ -124,6 +129,7 @@ impl FoldedNetworkResultExt for io::Result> { | io::ErrorKind::ConnectionReset | io::ErrorKind::HostUnreachable | io::ErrorKind::NetworkUnreachable => Ok(NetworkResult::NoConnection(e)), + io::ErrorKind::AddrNotAvailable => Ok(NetworkResult::AlreadyExists(e)), _ => Err(e), }, #[cfg(not(feature = "io_error_more"))] @@ -139,6 +145,7 @@ impl FoldedNetworkResultExt for io::Result> { io::ErrorKind::ConnectionAborted | io::ErrorKind::ConnectionRefused | io::ErrorKind::ConnectionReset => Ok(NetworkResult::NoConnection(e)), + io::ErrorKind::AddrNotAvailable => Ok(NetworkResult::AlreadyExists(e)), _ => Err(e), } } @@ -153,6 +160,7 @@ impl FoldedNetworkResultExt for io::Result> { pub enum NetworkResult { Timeout, NoConnection(io::Error), + AlreadyExists(io::Error), InvalidMessage(String), Value(T), } @@ -170,7 +178,9 @@ impl NetworkResult { pub fn invalid_message(s: S) -> Self { Self::InvalidMessage(s.to_string()) } - + pub fn already_exists(e: io::Error) -> Self { + Self::AlreadyExists(e) + } pub fn value(value: T) -> Self { Self::Value(value) } @@ -181,6 +191,9 @@ impl NetworkResult { pub fn is_no_connection(&self) -> bool { matches!(self, Self::NoConnection(_)) } + pub fn is_already_exists(&self) -> bool { + matches!(self, Self::AlreadyExists(_)) + } pub fn is_value(&self) -> bool { matches!(self, Self::Value(_)) } @@ -188,6 +201,7 @@ impl NetworkResult { match self { Self::Timeout => NetworkResult::::Timeout, Self::NoConnection(e) => NetworkResult::::NoConnection(e), + Self::AlreadyExists(e) => NetworkResult::::AlreadyExists(e), Self::InvalidMessage(s) => NetworkResult::::InvalidMessage(s), Self::Value(v) => NetworkResult::::Value(f(v)), } @@ -196,6 +210,7 @@ impl NetworkResult { match self { Self::Timeout => Err(io::Error::new(io::ErrorKind::TimedOut, "Timed out")), Self::NoConnection(e) => Err(e), + Self::AlreadyExists(e) => Err(e), Self::InvalidMessage(s) => Err(io::Error::new( io::ErrorKind::InvalidData, format!("Invalid message: {}", s), @@ -230,6 +245,7 @@ impl Debug for NetworkResult { match self { Self::Timeout => write!(f, "Timeout"), Self::NoConnection(e) => f.debug_tuple("NoConnection").field(e).finish(), + Self::AlreadyExists(e) => f.debug_tuple("AlreadyExists").field(e).finish(), Self::InvalidMessage(s) => f.debug_tuple("InvalidMessage").field(s).finish(), Self::Value(v) => f.debug_tuple("Value").field(v).finish(), } @@ -241,6 +257,7 @@ impl Display for NetworkResult { match self { Self::Timeout => write!(f, "Timeout"), Self::NoConnection(e) => write!(f, "NoConnection({})", e.kind()), + Self::AlreadyExists(e) => write!(f, "AlreadyExists({})", e.kind()), Self::InvalidMessage(s) => write!(f, "InvalidMessage({})", s), Self::Value(_) => write!(f, "Value"), } @@ -258,6 +275,7 @@ macro_rules! network_result_try { match $r { NetworkResult::Timeout => return Ok(NetworkResult::Timeout), NetworkResult::NoConnection(e) => return Ok(NetworkResult::NoConnection(e)), + NetworkResult::AlreadyExists(e) => return Ok(NetworkResult::AlreadyExists(e)), NetworkResult::InvalidMessage(s) => return Ok(NetworkResult::InvalidMessage(s)), NetworkResult::Value(v) => v, } @@ -272,6 +290,10 @@ macro_rules! network_result_try { $f; return Ok(NetworkResult::NoConnection(e)); } + NetworkResult::AlreadyExists(e) => { + $f; + return Ok(NetworkResult::AlreadyExists(e)); + } NetworkResult::InvalidMessage(s) => { $f; return Ok(NetworkResult::InvalidMessage(s)); @@ -293,6 +315,10 @@ macro_rules! network_result_value_or_log { log_net!($level "{}({}) at {}@{}:{}", "No connection".green(), e.to_string(), file!(), line!(), column!()); $f } + NetworkResult::AlreadyExists(e) => { + log_net!($level "{}({}) at {}@{}:{}", "Already exists".green(), e.to_string(), file!(), line!(), column!()); + $f + } NetworkResult::InvalidMessage(s) => { log_net!($level "{}({}) at {}@{}:{}", "Invalid message".green(), s, file!(), line!(), column!()); $f From 98f90154f38d010d98395f65d35d6724ac48886b Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 4 Oct 2022 20:09:32 -0400 Subject: [PATCH 06/67] instrumentation and network fixes --- veilid-core/src/network_manager/mod.rs | 5 ++- .../native/network_class_discovery.rs | 35 ++++++++++++------- .../src/network_manager/native/network_udp.rs | 4 +-- .../network_manager/native/protocol/tcp.rs | 4 +-- .../network_manager/native/protocol/udp.rs | 10 +++--- .../src/network_manager/native/protocol/ws.rs | 4 +-- .../src/network_manager/wasm/protocol/ws.rs | 4 +-- veilid-core/src/routing_table/node_ref.rs | 2 +- veilid-core/src/xx/must_join_single_future.rs | 4 +-- veilid-core/src/xx/tick_task.rs | 27 +++++++++----- 10 files changed, 62 insertions(+), 37 deletions(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 80301d6c..4e3d0593 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -610,12 +610,15 @@ impl NetworkManager { let c = self.config.get(); c.network.dht.min_peer_count as usize }; + // If none, then add the bootstrap nodes to it if live_public_internet_entry_count == 0 { self.unlocked_inner.bootstrap_task.tick().await?; } // If we still don't have enough peers, find nodes until we do - else if live_public_internet_entry_count < min_peer_count { + else if !self.unlocked_inner.bootstrap_task.is_running() + && live_public_internet_entry_count < min_peer_count + { self.unlocked_inner.peer_minimum_refresh_task.tick().await?; } diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/network_class_discovery.rs index ed95e0d1..b53fefad 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/network_class_discovery.rs @@ -382,17 +382,6 @@ impl DiscoveryContext { // If we know we are behind NAT check what kind #[instrument(level = "trace", skip(self), ret, err)] pub async fn protocol_process_nat(&self) -> EyreResult { - let (node_1, external_1_dial_info, external_1_address, protocol_type, address_type) = { - let inner = self.inner.lock(); - ( - inner.node_1.as_ref().unwrap().clone(), - inner.external_1_dial_info.as_ref().unwrap().clone(), - inner.external_1_address.unwrap(), - inner.protocol_type.unwrap(), - inner.address_type.unwrap(), - ) - }; - // Attempt a port mapping via all available and enabled mechanisms // Try this before the direct mapping in the event that we are restarting // and may not have recorded a mapping created the last time @@ -404,8 +393,30 @@ impl DiscoveryContext { // No more retries return Ok(true); } + + // XXX: is this necessary? + // Redo our external_1 dial info detection because a failed port mapping attempt + // may cause it to become invalid + // Get our external address from some fast node, call it node 1 + // if !self.protocol_get_external_address_1().await { + // // If we couldn't get an external address, then we should just try the whole network class detection again later + // return Ok(false); + // } + + // Get the external dial info for our use here + let (node_1, external_1_dial_info, external_1_address, protocol_type, address_type) = { + let inner = self.inner.lock(); + ( + inner.node_1.as_ref().unwrap().clone(), + inner.external_1_dial_info.as_ref().unwrap().clone(), + inner.external_1_address.unwrap(), + inner.protocol_type.unwrap(), + inner.address_type.unwrap(), + ) + }; + // Do a validate_dial_info on the external address from a redirected node - else if self + if self .validate_dial_info(node_1.clone(), external_1_dial_info.clone(), true) .await { diff --git a/veilid-core/src/network_manager/native/network_udp.rs b/veilid-core/src/network_manager/native/network_udp.rs index 0d906f16..b00bf643 100644 --- a/veilid-core/src/network_manager/native/network_udp.rs +++ b/veilid-core/src/network_manager/native/network_udp.rs @@ -54,7 +54,7 @@ impl Network { loop { match ph - .recv_message(&mut data).instrument(Span::current()) + .recv_message(&mut data) .timeout_at(stop_token.clone()) .await { @@ -84,7 +84,7 @@ impl Network { } } } - }.instrument(Span::current()); + }; protocol_handlers_unordered.push(ph_future); } diff --git a/veilid-core/src/network_manager/native/protocol/tcp.rs b/veilid-core/src/network_manager/native/protocol/tcp.rs index 91167b69..847c8608 100644 --- a/veilid-core/src/network_manager/native/protocol/tcp.rs +++ b/veilid-core/src/network_manager/native/protocol/tcp.rs @@ -87,11 +87,11 @@ impl RawTcpNetworkConnection { Ok(NetworkResult::Value(out)) } - #[instrument(level = "trace", err, skip(self), fields(network_result))] + // #[instrument(level = "trace", err, skip(self), fields(network_result))] pub async fn recv(&self) -> io::Result>> { let mut stream = self.stream.clone(); let out = Self::recv_internal(&mut stream).await?; - tracing::Span::current().record("network_result", &tracing::field::display(&out)); + //tracing::Span::current().record("network_result", &tracing::field::display(&out)); Ok(out) } } diff --git a/veilid-core/src/network_manager/native/protocol/udp.rs b/veilid-core/src/network_manager/native/protocol/udp.rs index 8dc39b2e..6af66b1d 100644 --- a/veilid-core/src/network_manager/native/protocol/udp.rs +++ b/veilid-core/src/network_manager/native/protocol/udp.rs @@ -11,7 +11,7 @@ impl RawUdpProtocolHandler { Self { socket } } - #[instrument(level = "trace", err, skip(self, data), fields(data.len = data.len(), ret.len, ret.from))] + // #[instrument(level = "trace", err, skip(self, data), fields(data.len = data.len(), ret.len, ret.descriptor))] pub async fn recv_message(&self, data: &mut [u8]) -> io::Result<(usize, ConnectionDescriptor)> { let (size, descriptor) = loop { let (size, remote_addr) = network_result_value_or_log!(debug self.socket.recv_from(data).await.into_network_result()? => continue); @@ -33,12 +33,12 @@ impl RawUdpProtocolHandler { break (size, descriptor); }; - tracing::Span::current().record("ret.len", &size); - tracing::Span::current().record("ret.from", &format!("{:?}", descriptor).as_str()); + // tracing::Span::current().record("ret.len", &size); + // tracing::Span::current().record("ret.descriptor", &format!("{:?}", descriptor).as_str()); Ok((size, descriptor)) } - #[instrument(level = "trace", err, skip(self, data), fields(data.len = data.len(), ret.len, ret.from))] + #[instrument(level = "trace", err, skip(self, data), fields(data.len = data.len(), ret.len, ret.descriptor))] pub async fn send_message( &self, data: Vec, @@ -67,6 +67,8 @@ impl RawUdpProtocolHandler { bail_io_error_other!("UDP partial send") } + tracing::Span::current().record("ret.len", &len); + tracing::Span::current().record("ret.descriptor", &format!("{:?}", descriptor).as_str()); Ok(NetworkResult::value(descriptor)) } diff --git a/veilid-core/src/network_manager/native/protocol/ws.rs b/veilid-core/src/network_manager/native/protocol/ws.rs index 520cd344..89e3c46d 100644 --- a/veilid-core/src/network_manager/native/protocol/ws.rs +++ b/veilid-core/src/network_manager/native/protocol/ws.rs @@ -93,7 +93,7 @@ where Ok(out) } - #[instrument(level = "trace", err, skip(self), fields(network_result, ret.len))] + // #[instrument(level = "trace", err, skip(self), fields(network_result, ret.len))] pub async fn recv(&self) -> io::Result>> { let out = match self.stream.clone().next().await { Some(Ok(Message::Binary(v))) => { @@ -120,7 +120,7 @@ where )), }; - tracing::Span::current().record("network_result", &tracing::field::display(&out)); + // tracing::Span::current().record("network_result", &tracing::field::display(&out)); Ok(out) } } diff --git a/veilid-core/src/network_manager/wasm/protocol/ws.rs b/veilid-core/src/network_manager/wasm/protocol/ws.rs index 1b48c59d..f9adebb8 100644 --- a/veilid-core/src/network_manager/wasm/protocol/ws.rs +++ b/veilid-core/src/network_manager/wasm/protocol/ws.rs @@ -75,7 +75,7 @@ impl WebsocketNetworkConnection { Ok(out) } - #[instrument(level = "trace", err, skip(self), fields(network_result, ret.len))] + // #[instrument(level = "trace", err, skip(self), fields(network_result, ret.len))] pub async fn recv(&self) -> io::Result>> { let out = match SendWrapper::new(self.inner.ws_stream.clone().next()).await { Some(WsMessage::Binary(v)) => { @@ -95,7 +95,7 @@ impl WebsocketNetworkConnection { bail_io_error_other!("WS stream closed"); } }; - tracing::Span::current().record("network_result", &tracing::field::display(&out)); + // tracing::Span::current().record("network_result", &tracing::field::display(&out)); Ok(out) } } diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 96b90393..d686e84d 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -90,7 +90,7 @@ impl NodeRef { node_id, entry, filter, - reliable: true, + reliable: false, #[cfg(feature = "tracking")] track_id: entry.track(), } diff --git a/veilid-core/src/xx/must_join_single_future.rs b/veilid-core/src/xx/must_join_single_future.rs index 8830c1d3..ffebeaae 100644 --- a/veilid-core/src/xx/must_join_single_future.rs +++ b/veilid-core/src/xx/must_join_single_future.rs @@ -63,7 +63,7 @@ where inner.join_handle = jh; } - // Check the result + /// Check the result and take it if there is one pub async fn check(&self) -> Result, ()> { let mut out: Option = None; @@ -96,7 +96,7 @@ where Ok(out) } - // Wait for the result + /// Wait for the result and take it pub async fn join(&self) -> Result, ()> { let mut out: Option = None; diff --git a/veilid-core/src/xx/tick_task.rs b/veilid-core/src/xx/tick_task.rs index 9ac821d1..56ea442e 100644 --- a/veilid-core/src/xx/tick_task.rs +++ b/veilid-core/src/xx/tick_task.rs @@ -15,6 +15,7 @@ pub struct TickTask { routine: OnceCell>>, stop_source: AsyncMutex>, single_future: MustJoinSingleFuture>, + running: Arc, } impl TickTask { @@ -25,6 +26,7 @@ impl TickTask { routine: OnceCell::new(), stop_source: AsyncMutex::new(None), single_future: MustJoinSingleFuture::new(), + running: Arc::new(AtomicBool::new(false)), } } pub fn new_ms(tick_period_ms: u32) -> Self { @@ -34,6 +36,7 @@ impl TickTask { routine: OnceCell::new(), stop_source: AsyncMutex::new(None), single_future: MustJoinSingleFuture::new(), + running: Arc::new(AtomicBool::new(false)), } } pub fn new(tick_period_sec: u32) -> Self { @@ -43,6 +46,7 @@ impl TickTask { routine: OnceCell::new(), stop_source: AsyncMutex::new(None), single_future: MustJoinSingleFuture::new(), + running: Arc::new(AtomicBool::new(false)), } } @@ -53,6 +57,10 @@ impl TickTask { self.routine.set(Box::new(routine)).map_err(drop).unwrap(); } + pub fn is_running(&self) -> bool { + self.running.load(core::sync::atomic::Ordering::Relaxed) + } + pub async fn stop(&self) -> Result<(), E> { // drop the stop source if we have one let opt_stop_source = &mut *self.stop_source.lock().await; @@ -107,15 +115,16 @@ impl TickTask { // Run the singlefuture let stop_source = StopSource::new(); - match self - .single_future - .single_spawn(self.routine.get().unwrap()( - stop_source.token(), - last_timestamp_us, - now, - )) - .await - { + let stop_token = stop_source.token(); + let running = self.running.clone(); + let routine = self.routine.get().unwrap()(stop_token, last_timestamp_us, now); + let wrapped_routine = Box::pin(async move { + running.store(true, core::sync::atomic::Ordering::Relaxed); + let out = routine.await; + running.store(false, core::sync::atomic::Ordering::Relaxed); + out + }); + match self.single_future.single_spawn(wrapped_routine).await { // We should have already consumed the result of the last run, or there was none // and we should definitely have run, because the prior 'check()' operation // should have ensured the singlefuture was ready to run From b1cc0d803c1a07c90ea00a122e79dea1341e45a4 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 5 Oct 2022 19:12:10 -0400 Subject: [PATCH 07/67] xfer --- Cargo.lock | 14 ++++++++ veilid-core/src/network_manager/wasm/mod.rs | 37 +++++++++++++++------ veilid-wasm/Cargo.toml | 1 + veilid-wasm/src/lib.rs | 3 +- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db7beaf2..8c0adabc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1919,6 +1919,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gloo-utils" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40913a05c8297adca04392f707b1e73b12ba7b8eab7244a4961580b1fd34063c" +dependencies = [ + "js-sys", + "serde", + "serde_json", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "grpcio" version = "0.9.1" @@ -5448,6 +5461,7 @@ dependencies = [ "console_error_panic_hook", "data-encoding", "futures-util", + "gloo-utils", "js-sys", "lazy_static", "send_wrapper 0.6.0", diff --git a/veilid-core/src/network_manager/wasm/mod.rs b/veilid-core/src/network_manager/wasm/mod.rs index 448c1e9e..c36454e5 100644 --- a/veilid-core/src/network_manager/wasm/mod.rs +++ b/veilid-core/src/network_manager/wasm/mod.rs @@ -10,28 +10,37 @@ use std::io; ///////////////////////////////////////////////////////////////// struct NetworkInner { - network_manager: NetworkManager, network_started: bool, network_needs_restart: bool, protocol_config: Option, } +struct NetworkUnlockedInner { + network_manager: NetworkManager, +} + #[derive(Clone)] pub struct Network { config: VeilidConfig, inner: Arc>, + unlocked_inner: Arc, } impl Network { - fn new_inner(network_manager: NetworkManager) -> NetworkInner { + fn new_inner() -> NetworkInner { NetworkInner { - network_manager, network_started: false, network_needs_restart: false, protocol_config: None, //join_handle: None, } } + fn new_unlocked_inner(network_manager: NetworkManager) -> NetworkUnlockedInner { + NetworkUnlockedInner { + network_manager + } + } + pub fn new( network_manager: NetworkManager, routing_table: RoutingTable, @@ -39,15 +48,16 @@ impl Network { ) -> Self { Self { config: network_manager.config(), - inner: Arc::new(Mutex::new(Self::new_inner(network_manager))), + inner: Arc::new(Mutex::new(Self::new_inner())), + unlocked_inner: Arc::new(Self::new_unlocked_inner(network_manager)) } } fn network_manager(&self) -> NetworkManager { - self.inner.lock().network_manager.clone() + self.unlocked_inner.network_manager.clone() } fn connection_manager(&self) -> ConnectionManager { - self.inner.lock().network_manager.connection_manager() + self.unlocked_inner.network_manager.connection_manager() } ///////////////////////////////////////////////////////////////// @@ -269,15 +279,22 @@ impl Network { trace!("stopping network"); // Reset state - let network_manager = self.inner.lock().network_manager.clone(); + let network_manager = self.network_manager(); let routing_table = network_manager.routing_table(); // Drop all dial info - routing_table.clear_dial_info_details(RoutingDomain::PublicInternet); - routing_table.clear_dial_info_details(RoutingDomain::LocalNetwork); + let mut editor = routing_table.edit_routing_domain(RoutingDomain::PublicInternet); + editor.disable_node_info_updates(); + editor.clear_dial_info_details(); + editor.commit().await; + + let mut editor = routing_table.edit_routing_domain(RoutingDomain::LocalNetwork); + editor.disable_node_info_updates(); + editor.clear_dial_info_details(); + editor.commit().await; // Cancels all async background tasks by dropping join handles - *self.inner.lock() = Self::new_inner(network_manager); + *self.inner.lock() = Self::new_inner(); trace!("network stopped"); } diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index 7ca4e42f..758f4f88 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -27,6 +27,7 @@ lazy_static = "^1" send_wrapper = "^0" futures-util = { version = "^0", default_features = false, features = ["alloc"] } data-encoding = { version = "^2" } +gloo-utils = { version = "^0", features = ["serde"] } [dev-dependencies] wasm-bindgen-test = "^0" diff --git a/veilid-wasm/src/lib.rs b/veilid-wasm/src/lib.rs index 6baf1e80..e49bb8e6 100644 --- a/veilid-wasm/src/lib.rs +++ b/veilid-wasm/src/lib.rs @@ -10,6 +10,7 @@ use alloc::*; use core::any::{Any, TypeId}; use core::cell::RefCell; use futures_util::FutureExt; +use gloo_utils::format::JsValueSerdeExt; use js_sys::*; use lazy_static::*; use send_wrapper::*; @@ -306,5 +307,5 @@ pub fn veilid_version() -> JsValue { minor, patch, }; - JsValue::from_serde(&vv).unwrap() + ::from_serde(&vv).unwrap() } From e77577ba66d9c25263b392c464403a522277c026 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 6 Oct 2022 11:40:55 -0400 Subject: [PATCH 08/67] wasm fixes --- setup_linux.sh | 2 +- setup_macos.sh | 8 +--- .../src/network_manager/connection_manager.rs | 5 ++- veilid-core/src/network_manager/tasks.rs | 10 +++-- .../tests/test_connection_table.rs | 2 + veilid-core/src/network_manager/wasm/mod.rs | 25 ++++++++---- .../src/network_manager/wasm/protocol/ws.rs | 2 +- veilid-core/src/routing_table/find_nodes.rs | 11 +++-- veilid-core/src/rpc_processor/rpc_status.rs | 4 ++ .../src/tests/common/test_async_tag_lock.rs | 40 +++++++++---------- veilid-core/src/xx/async_tag_lock.rs | 37 +++++++++++------ 11 files changed, 87 insertions(+), 59 deletions(-) diff --git a/setup_linux.sh b/setup_linux.sh index 59677a64..68c17c16 100755 --- a/setup_linux.sh +++ b/setup_linux.sh @@ -74,7 +74,7 @@ fi rustup target add aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android wasm32-unknown-unknown # install cargo packages -cargo install wasm-bindgen-cli +cargo install wasm-bindgen-cli wasm-pack # Ensure packages are installed sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk llvm wabt capnproto protobuf-compiler diff --git a/setup_macos.sh b/setup_macos.sh index 501d08b2..91d3c4e9 100755 --- a/setup_macos.sh +++ b/setup_macos.sh @@ -90,13 +90,7 @@ fi rustup target add aarch64-apple-darwin aarch64-apple-ios x86_64-apple-darwin x86_64-apple-ios wasm32-unknown-unknown aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android # install cargo packages -cargo install wasm-bindgen-cli - -# install bitcode compatible ios toolchain -# echo Manual Step: -# echo install +ios-arm64-1.57.0 toolchain for bitcode from https://github.com/getditto/rust-bitcode/releases/latest and unzip -# echo xattr -d -r com.apple.quarantine . -# echo ./install.sh +cargo install wasm-bindgen-cli wasm-pack # ensure we have command line tools xcode-select --install diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index f535ea5d..d6350612 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -251,7 +251,8 @@ impl ConnectionManager { // Async lock on the remote address for atomicity per remote let peer_address = dial_info.to_peer_address(); let remote_addr = peer_address.to_socket_addr(); - let _lock_guard = self.arc.address_lock_table.lock_tag(remote_addr); + + let _lock_guard = self.arc.address_lock_table.lock_tag(remote_addr).await; log_net!( "== get_or_create_connection local_addr={:?} dial_info={:?}", @@ -369,7 +370,7 @@ impl ConnectionManager { // Called by low-level network when any connection-oriented protocol connection appears // either from incoming connections. - #[cfg_attr(target_os = "wasm32", allow(dead_code))] + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] pub(super) async fn on_accepted_protocol_network_connection( &self, protocol_connection: ProtocolNetworkConnection, diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index 30014fca..55396fe3 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -353,7 +353,7 @@ impl NetworkManager { let node_refs = routing_table.get_nodes_needing_ping(RoutingDomain::PublicInternet, cur_ts); // Look up any NAT mappings we may need to try to preserve with keepalives - let mut mapped_port_info = routing_table.get_mapped_port_info(); + let mut mapped_port_info = routing_table.get_low_level_port_info(); // Get the PublicInternet relay if we are using one let opt_relay_nr = routing_table.relay_node(RoutingDomain::PublicInternet); @@ -558,7 +558,8 @@ impl NetworkManager { // Do we need a relay? if !has_relay && node_info.requires_relay() { - // Do we need an outbound relay? + // Do we want an outbound relay? + let mut got_outbound_relay = false; if network_class.outbound_wants_relay() { // The outbound relay is the host of the PWA if let Some(outbound_relay_peerinfo) = intf::get_outbound_relay_peer().await { @@ -571,10 +572,11 @@ impl NetworkManager { ) { info!("Outbound relay node selected: {}", nr); editor.set_relay_node(nr); + got_outbound_relay = true; } } - // Otherwise we must need an inbound relay - } else { + } + if !got_outbound_relay { // Find a node in our routing table that is an acceptable inbound relay if let Some(nr) = routing_table.find_inbound_relay(RoutingDomain::PublicInternet, cur_ts) diff --git a/veilid-core/src/network_manager/tests/test_connection_table.rs b/veilid-core/src/network_manager/tests/test_connection_table.rs index 06991423..79306051 100644 --- a/veilid-core/src/network_manager/tests/test_connection_table.rs +++ b/veilid-core/src/network_manager/tests/test_connection_table.rs @@ -52,6 +52,7 @@ pub async fn test_add_get_remove() { ); let c1 = NetworkConnection::dummy(1, a1); + let c1b = NetworkConnection::dummy(10, a1); let c1h = c1.get_handle(); let c2 = NetworkConnection::dummy(2, a2); let c3 = NetworkConnection::dummy(3, a3); @@ -65,6 +66,7 @@ pub async fn test_add_get_remove() { assert_eq!(table.connection_count(), 0); assert_eq!(table.get_connection_by_descriptor(a1), None); table.add_connection(c1).unwrap(); + assert!(table.add_connection(c1b).is_err()); assert_eq!(table.connection_count(), 1); assert!(table.remove_connection_by_id(4).is_none()); diff --git a/veilid-core/src/network_manager/wasm/mod.rs b/veilid-core/src/network_manager/wasm/mod.rs index c36454e5..c35167a9 100644 --- a/veilid-core/src/network_manager/wasm/mod.rs +++ b/veilid-core/src/network_manager/wasm/mod.rs @@ -16,7 +16,10 @@ struct NetworkInner { } struct NetworkUnlockedInner { + // Accessors + routing_table: RoutingTable, network_manager: NetworkManager, + connection_manager: ConnectionManager, } #[derive(Clone)] @@ -35,9 +38,15 @@ impl Network { } } - fn new_unlocked_inner(network_manager: NetworkManager) -> NetworkUnlockedInner { + fn new_unlocked_inner( + network_manager: NetworkManager, + routing_table: RoutingTable, + connection_manager: ConnectionManager, + ) -> NetworkUnlockedInner { NetworkUnlockedInner { - network_manager + network_manager, + routing_table, + connection_manager } } @@ -49,15 +58,18 @@ impl Network { Self { config: network_manager.config(), inner: Arc::new(Mutex::new(Self::new_inner())), - unlocked_inner: Arc::new(Self::new_unlocked_inner(network_manager)) + unlocked_inner: Arc::new(Self::new_unlocked_inner(network_manager, routing_table, connection_manager)) } } fn network_manager(&self) -> NetworkManager { self.unlocked_inner.network_manager.clone() } + fn routing_table(&self) -> RoutingTable { + self.unlocked_inner.routing_table.clone() + } fn connection_manager(&self) -> ConnectionManager { - self.unlocked_inner.network_manager.connection_manager() + self.unlocked_inner.connection_manager.clone() } ///////////////////////////////////////////////////////////////// @@ -279,8 +291,7 @@ impl Network { trace!("stopping network"); // Reset state - let network_manager = self.network_manager(); - let routing_table = network_manager.routing_table(); + let routing_table = self.routing_table(); // Drop all dial info let mut editor = routing_table.edit_routing_domain(RoutingDomain::PublicInternet); @@ -299,7 +310,7 @@ impl Network { trace!("network stopped"); } - pub fn is_usable_interface_address(&self, addr: IpAddr) -> bool { + pub fn is_usable_interface_address(&self, _addr: IpAddr) -> bool { false } diff --git a/veilid-core/src/network_manager/wasm/protocol/ws.rs b/veilid-core/src/network_manager/wasm/protocol/ws.rs index f9adebb8..0315d3e5 100644 --- a/veilid-core/src/network_manager/wasm/protocol/ws.rs +++ b/veilid-core/src/network_manager/wasm/protocol/ws.rs @@ -112,7 +112,7 @@ impl WebsocketProtocolHandler { timeout_ms: u32, ) -> io::Result> { // Split dial info up - let (tls, scheme) = match dial_info { + let (_tls, scheme) = match dial_info { DialInfo::WS(_) => (false, "ws"), DialInfo::WSS(_) => (true, "wss"), _ => panic!("invalid dialinfo for WS/WSS protocol"), diff --git a/veilid-core/src/routing_table/find_nodes.rs b/veilid-core/src/routing_table/find_nodes.rs index ca9841a2..feeb6f6f 100644 --- a/veilid-core/src/routing_table/find_nodes.rs +++ b/veilid-core/src/routing_table/find_nodes.rs @@ -7,7 +7,7 @@ use crate::*; pub type LowLevelProtocolPorts = BTreeSet<(LowLevelProtocolType, AddressType, u16)>; pub type ProtocolToPortMapping = BTreeMap<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>; #[derive(Clone, Debug)] -pub struct MappedPortInfo { +pub struct LowLevelPortInfo { pub low_level_protocol_ports: LowLevelProtocolPorts, pub protocol_to_port: ProtocolToPortMapping, } @@ -389,7 +389,7 @@ impl RoutingTable { // Only one protocol per low level protocol/port combination is required // For example, if WS/WSS and TCP protocols are on the same low-level TCP port, only TCP keepalives will be required // and we do not need to do WS/WSS keepalive as well. If they are on different ports, then we will need WS/WSS keepalives too. - pub fn get_mapped_port_info(&self) -> MappedPortInfo { + pub fn get_low_level_port_info(&self) -> LowLevelPortInfo { let mut low_level_protocol_ports = BTreeSet::<(LowLevelProtocolType, AddressType, u16)>::new(); let mut protocol_to_port = @@ -412,7 +412,7 @@ impl RoutingTable { ), ); } - MappedPortInfo { + LowLevelPortInfo { low_level_protocol_ports, protocol_to_port, } @@ -423,7 +423,7 @@ impl RoutingTable { let outbound_dif = self .network_manager() .get_outbound_dial_info_filter(RoutingDomain::PublicInternet); - let mapped_port_info = self.get_mapped_port_info(); + let mapped_port_info = self.get_low_level_port_info(); move |e: &BucketEntryInner| { // Ensure this node is not on the local network @@ -435,6 +435,9 @@ impl RoutingTable { // as we need to be able to use the relay for keepalives for all nat mappings let mut low_level_protocol_ports = mapped_port_info.low_level_protocol_ports.clone(); + info!("outbound_dif: {:?}", outbound_dif); + info!("low_level_protocol_ports: {:?}", low_level_protocol_ports); + let can_serve_as_relay = e .node_info(RoutingDomain::PublicInternet) .map(|n| { diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 079665d4..31f56a7f 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -8,6 +8,7 @@ impl RPCProcessor { self, peer: NodeRef, ) -> Result>, RPCError> { + info!("ping to {:?}", peer); let routing_domain = match peer.best_routing_domain() { Some(rd) => rd, None => { @@ -43,6 +44,7 @@ impl RPCProcessor { }, _ => return Err(RPCError::invalid_format("not an answer")), }; + info!("qwer"); // Ensure the returned node status is the kind for the routing domain we asked for match routing_domain { @@ -62,6 +64,8 @@ impl RPCProcessor { } } + info!("zxzxv"); + // Update latest node status in routing table peer.update_node_status(status_a.node_status); diff --git a/veilid-core/src/tests/common/test_async_tag_lock.rs b/veilid-core/src/tests/common/test_async_tag_lock.rs index f932a457..e590d4df 100644 --- a/veilid-core/src/tests/common/test_async_tag_lock.rs +++ b/veilid-core/src/tests/common/test_async_tag_lock.rs @@ -35,20 +35,20 @@ pub async fn test_simple_single_contention() { let g1 = table.lock_tag(a1).await; - println!("locked"); + info!("locked"); let t1 = intf::spawn(async move { // move the guard into the task let _g1_take = g1; // hold the guard for a bit - println!("waiting"); + info!("waiting"); intf::sleep(1000).await; // release the guard - println!("released"); + info!("released"); }); // wait to lock again, will contend until spawned task exits let _g1_b = table.lock_tag(a1).await; - println!("locked"); + info!("locked"); // Ensure task is joined t1.await; @@ -67,24 +67,24 @@ pub async fn test_simple_double_contention() { let g1 = table.lock_tag(a1).await; let g2 = table.lock_tag(a2).await; - println!("locked"); + info!("locked"); let t1 = intf::spawn(async move { - // move the guard into the task + // move the guard into the tas let _g1_take = g1; // hold the guard for a bit - println!("waiting"); + info!("waiting"); intf::sleep(1000).await; // release the guard - println!("released"); + info!("released"); }); let t2 = intf::spawn(async move { // move the guard into the task let _g2_take = g2; // hold the guard for a bit - println!("waiting"); + info!("waiting"); intf::sleep(500).await; // release the guard - println!("released"); + info!("released"); }); // wait to lock again, will contend until spawned task exits @@ -92,7 +92,7 @@ pub async fn test_simple_double_contention() { // wait to lock again, should complete immediately let _g2_b = table.lock_tag(a2).await; - println!("locked"); + info!("locked"); // Ensure tasks are joined t1.await; @@ -112,36 +112,36 @@ pub async fn test_parallel_single_contention() { let t1 = intf::spawn(async move { // lock the tag let _g = table1.lock_tag(a1).await; - println!("locked t1"); + info!("locked t1"); // hold the guard for a bit - println!("waiting t1"); + info!("waiting t1"); intf::sleep(500).await; // release the guard - println!("released t1"); + info!("released t1"); }); let table2 = table.clone(); let t2 = intf::spawn(async move { // lock the tag let _g = table2.lock_tag(a1).await; - println!("locked t2"); + info!("locked t2"); // hold the guard for a bit - println!("waiting t2"); + info!("waiting t2"); intf::sleep(500).await; // release the guard - println!("released t2"); + info!("released t2"); }); let table3 = table.clone(); let t3 = intf::spawn(async move { // lock the tag let _g = table3.lock_tag(a1).await; - println!("locked t3"); + info!("locked t3"); // hold the guard for a bit - println!("waiting t3"); + info!("waiting t3"); intf::sleep(500).await; // release the guard - println!("released t3"); + info!("released t3"); }); // Ensure tasks are joined diff --git a/veilid-core/src/xx/async_tag_lock.rs b/veilid-core/src/xx/async_tag_lock.rs index 9e865935..5f0623c1 100644 --- a/veilid-core/src/xx/async_tag_lock.rs +++ b/veilid-core/src/xx/async_tag_lock.rs @@ -1,18 +1,33 @@ use super::*; +use core::fmt::Debug; use core::hash::Hash; +#[derive(Debug)] pub struct AsyncTagLockGuard where - T: Hash + Eq + Clone, + T: Hash + Eq + Clone + Debug, { table: AsyncTagLockTable, tag: T, _guard: AsyncMutexGuardArc<()>, } +impl AsyncTagLockGuard +where + T: Hash + Eq + Clone + Debug, +{ + fn new(table: AsyncTagLockTable, tag: T, guard: AsyncMutexGuardArc<()>) -> Self { + Self { + table, + tag, + _guard: guard, + } + } +} + impl Drop for AsyncTagLockGuard where - T: Hash + Eq + Clone, + T: Hash + Eq + Clone + Debug, { fn drop(&mut self) { let mut inner = self.table.inner.lock(); @@ -33,7 +48,7 @@ where } } -#[derive(Clone)] +#[derive(Clone, Debug)] struct AsyncTagLockTableEntry { mutex: Arc>, waiters: usize, @@ -41,7 +56,7 @@ struct AsyncTagLockTableEntry { struct AsyncTagLockTableInner where - T: Hash + Eq + Clone, + T: Hash + Eq + Clone + Debug, { table: HashMap, } @@ -49,14 +64,14 @@ where #[derive(Clone)] pub struct AsyncTagLockTable where - T: Hash + Eq + Clone, + T: Hash + Eq + Clone + Debug, { inner: Arc>>, } impl fmt::Debug for AsyncTagLockTable where - T: Hash + Eq + Clone, + T: Hash + Eq + Clone + Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("AsyncTagLockTable").finish() @@ -65,7 +80,7 @@ where impl AsyncTagLockTable where - T: Hash + Eq + Clone, + T: Hash + Eq + Clone + Debug, { pub fn new() -> Self { Self { @@ -111,16 +126,12 @@ where // tokio version guard = mutex.lock_owned().await; } else { - // async_std and wasm async_mutex version + // async-std and wasm async-lock version guard = mutex.lock_arc().await; } } // Return the locked guard - AsyncTagLockGuard { - table: self.clone(), - tag, - _guard: guard, - } + AsyncTagLockGuard::new(self.clone(), tag, guard) } } From 1fdcd5ae455b1ab7017da923316ae5f8a00af59b Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 6 Oct 2022 12:39:30 -0400 Subject: [PATCH 09/67] wasm fixes --- veilid-core/src/routing_table/bucket_entry.rs | 30 ++++++++++++------- veilid-core/src/routing_table/find_nodes.rs | 3 -- veilid-core/src/rpc_processor/rpc_status.rs | 4 --- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 25bb62e1..f3f665e8 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -439,14 +439,17 @@ impl BucketEntryInner { } } - fn needs_constant_ping(&self, cur_ts: u64, interval: u64) -> bool { - // If we have not either seen the node, nor asked it a question in the last 'interval' - // then we should ping it - let latest_contact_time = self - .peer_stats + /// Return the last time we either saw a node, or asked it a question + fn latest_contact_time(&self) -> Option { + self.peer_stats .rpc_stats .last_seen_ts - .max(self.peer_stats.rpc_stats.last_question); + .max(self.peer_stats.rpc_stats.last_question) + } + + fn needs_constant_ping(&self, cur_ts: u64, interval: u64) -> bool { + // If we have not either seen the node in the last 'interval' then we should ping it + let latest_contact_time = self.latest_contact_time(); match latest_contact_time { None => true, @@ -468,14 +471,19 @@ impl BucketEntryInner { return self.needs_constant_ping(cur_ts, KEEPALIVE_PING_INTERVAL_SECS as u64); } + // If we don't have node status for this node, then we should ping it to get some node status + for routing_domain in RoutingDomainSet::all() { + if self.has_node_info(routing_domain.into()) { + if self.node_status(routing_domain).is_none() { + return true; + } + } + } + match state { BucketEntryState::Reliable => { // If we are in a reliable state, we need a ping on an exponential scale - let latest_contact_time = self - .peer_stats - .rpc_stats - .last_seen_ts - .max(self.peer_stats.rpc_stats.last_question); + let latest_contact_time = self.latest_contact_time(); match latest_contact_time { None => { diff --git a/veilid-core/src/routing_table/find_nodes.rs b/veilid-core/src/routing_table/find_nodes.rs index feeb6f6f..4485b510 100644 --- a/veilid-core/src/routing_table/find_nodes.rs +++ b/veilid-core/src/routing_table/find_nodes.rs @@ -435,9 +435,6 @@ impl RoutingTable { // as we need to be able to use the relay for keepalives for all nat mappings let mut low_level_protocol_ports = mapped_port_info.low_level_protocol_ports.clone(); - info!("outbound_dif: {:?}", outbound_dif); - info!("low_level_protocol_ports: {:?}", low_level_protocol_ports); - let can_serve_as_relay = e .node_info(RoutingDomain::PublicInternet) .map(|n| { diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 31f56a7f..079665d4 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -8,7 +8,6 @@ impl RPCProcessor { self, peer: NodeRef, ) -> Result>, RPCError> { - info!("ping to {:?}", peer); let routing_domain = match peer.best_routing_domain() { Some(rd) => rd, None => { @@ -44,7 +43,6 @@ impl RPCProcessor { }, _ => return Err(RPCError::invalid_format("not an answer")), }; - info!("qwer"); // Ensure the returned node status is the kind for the routing domain we asked for match routing_domain { @@ -64,8 +62,6 @@ impl RPCProcessor { } } - info!("zxzxv"); - // Update latest node status in routing table peer.update_node_status(status_a.node_status); From 338dc6b39d329bcb4addf4802b28427d981906ea Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 9 Oct 2022 14:59:01 -0400 Subject: [PATCH 10/67] refactor checkpoint --- veilid-core/src/dht/key.rs | 6 + veilid-core/src/network_manager/mod.rs | 155 ++------ veilid-core/src/network_manager/native/mod.rs | 44 +-- .../native/network_class_discovery.rs | 16 +- veilid-core/src/network_manager/tasks.rs | 34 +- veilid-core/src/network_manager/wasm/mod.rs | 10 +- veilid-core/src/routing_table/bucket.rs | 17 +- veilid-core/src/routing_table/bucket_entry.rs | 40 ++- veilid-core/src/routing_table/debug.rs | 8 +- veilid-core/src/routing_table/find_nodes.rs | 129 +++---- veilid-core/src/routing_table/mod.rs | 260 +++++++++----- veilid-core/src/routing_table/node_ref.rs | 4 +- .../src/routing_table/route_spec_store.rs | 334 ++++++++++++++++++ .../routing_table/routing_domain_editor.rs | 121 ++++++- .../src/routing_table/routing_domains.rs | 242 ++++++++++--- veilid-core/src/routing_table/tasks.rs | 9 +- veilid-core/src/rpc_processor/destination.rs | 142 ++------ veilid-core/src/rpc_processor/mod.rs | 3 +- .../src/rpc_processor/rpc_find_node.rs | 26 +- .../rpc_processor/rpc_validate_dial_info.rs | 2 +- veilid-core/src/veilid_api/mod.rs | 19 - veilid-core/src/veilid_api/privacy.rs | 24 +- veilid-core/src/veilid_api/routing_context.rs | 26 +- .../src/veilid_api/serialize_helpers.rs | 15 + 24 files changed, 1122 insertions(+), 564 deletions(-) create mode 100644 veilid-core/src/routing_table/route_spec_store.rs diff --git a/veilid-core/src/dht/key.rs b/veilid-core/src/dht/key.rs index 8a020a38..c0c1934f 100644 --- a/veilid-core/src/dht/key.rs +++ b/veilid-core/src/dht/key.rs @@ -16,15 +16,21 @@ use serde::{Deserialize, Serialize}; ////////////////////////////////////////////////////////////////////// +/// Length of a DHT key in bytes #[allow(dead_code)] pub const DHT_KEY_LENGTH: usize = 32; +/// Length of a DHT key in bytes after encoding to base64url #[allow(dead_code)] pub const DHT_KEY_LENGTH_ENCODED: usize = 43; +/// Length of a DHT secret in bytes #[allow(dead_code)] pub const DHT_KEY_SECRET_LENGTH: usize = 32; +/// Length of a DHT secret in bytes after encoding to base64url #[allow(dead_code)] pub const DHT_KEY_SECRET_LENGTH_ENCODED: usize = 43; +/// Length of a DHT signature in bytes #[allow(dead_code)] +/// Length of a DHT signature in bytes after encoding to base64url pub const DHT_SIGNATURE_LENGTH: usize = 64; #[allow(dead_code)] pub const DHT_SIGNATURE_LENGTH_ENCODED: usize = 86; diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 4e3d0593..455d35fd 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -38,6 +38,7 @@ use xx::*; //////////////////////////////////////////////////////////////////////////////////////// pub const RELAY_MANAGEMENT_INTERVAL_SECS: u32 = 1; +pub const PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS: u32 = 1; pub const MAX_MESSAGE_SIZE: usize = MAX_ENVELOPE_SIZE; pub const IPADDR_TABLE_SIZE: usize = 1024; pub const IPADDR_MAX_INACTIVE_DURATION_US: u64 = 300_000_000u64; // 5 minutes @@ -148,11 +149,6 @@ struct NetworkManagerInner { BTreeMap>, public_address_inconsistencies_table: BTreeMap>, - protocol_config: Option, - public_inbound_dial_info_filter: Option, - local_inbound_dial_info_filter: Option, - public_outbound_dial_info_filter: Option, - local_outbound_dial_info_filter: Option, } struct NetworkManagerUnlockedInner { @@ -163,6 +159,7 @@ struct NetworkManagerUnlockedInner { // Background processes rolling_transfers_task: TickTask, relay_management_task: TickTask, + private_route_management_task: TickTask, bootstrap_task: TickTask, peer_minimum_refresh_task: TickTask, ping_validator_task: TickTask, @@ -186,11 +183,6 @@ impl NetworkManager { client_whitelist: LruCache::new_unbounded(), public_address_check_cache: BTreeMap::new(), public_address_inconsistencies_table: BTreeMap::new(), - protocol_config: None, - public_inbound_dial_info_filter: None, - local_inbound_dial_info_filter: None, - public_outbound_dial_info_filter: None, - local_outbound_dial_info_filter: None, } } fn new_unlocked_inner(config: VeilidConfig) -> NetworkManagerUnlockedInner { @@ -201,6 +193,7 @@ impl NetworkManager { update_callback: RwLock::new(None), rolling_transfers_task: TickTask::new(ROLLING_TRANSFERS_INTERVAL_SECS), relay_management_task: TickTask::new(RELAY_MANAGEMENT_INTERVAL_SECS), + private_route_management_task: TickTask::new(PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS), bootstrap_task: TickTask::new(1), peer_minimum_refresh_task: TickTask::new_ms(c.network.dht.min_peer_refresh_time_ms), ping_validator_task: TickTask::new(1), @@ -248,6 +241,23 @@ impl NetworkManager { ) }); } + // Set private route management tick task + { + let this2 = this.clone(); + this.unlocked_inner + .private_route_management_task + .set_routine(move |s, l, t| { + Box::pin( + this2 + .clone() + .private_route_management_task_routine(s, l, t) + .instrument(trace_span!( + parent: None, + "private route management task routine" + )), + ) + }); + } // Set bootstrap tick task { let this2 = this.clone(); @@ -434,41 +444,6 @@ impl NetworkManager { return Err(e); } - // Store copy of protocol config and dial info filters - { - let pc = self.net().get_protocol_config().unwrap(); - - let mut inner = self.inner.lock(); - - inner.public_inbound_dial_info_filter = Some( - DialInfoFilter::all() - .with_protocol_type_set(pc.inbound) - .with_address_type_set(pc.family_global), - ); - inner.local_inbound_dial_info_filter = Some( - DialInfoFilter::all() - .with_protocol_type_set(pc.inbound) - .with_address_type_set(pc.family_local), - ); - inner.public_outbound_dial_info_filter = Some( - DialInfoFilter::all() - .with_protocol_type_set(pc.outbound) - .with_address_type_set(pc.family_global), - ); - inner.local_outbound_dial_info_filter = Some( - DialInfoFilter::all() - .with_protocol_type_set(pc.outbound) - .with_address_type_set(pc.family_local), - ); - - inner.protocol_config = Some(pc); - } - - // Inform routing table entries that our dial info has changed - for rd in RoutingDomain::all() { - self.send_node_info_updates(rd, true).await; - } - // Inform api clients that things have changed self.send_network_update(); @@ -527,12 +502,7 @@ impl NetworkManager { // reset the state debug!("resetting network manager state"); { - let mut inner = self.inner.lock(); - inner.public_inbound_dial_info_filter = None; - inner.local_inbound_dial_info_filter = None; - inner.public_outbound_dial_info_filter = None; - inner.local_outbound_dial_info_filter = None; - inner.protocol_config = None; + *self.inner.lock() = NetworkManager::new_inner(); } // send update @@ -640,15 +610,6 @@ impl NetworkManager { Ok(()) } - // Return what network class we are in - pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option { - if let Some(components) = self.unlocked_inner.components.read().as_ref() { - components.net.get_network_class(routing_domain) - } else { - None - } - } - // Get our node's capabilities fn generate_public_internet_node_status(&self) -> PublicInternetNodeStatus { let node_info = self @@ -694,58 +655,6 @@ impl NetworkManager { } } - // Return what protocols we have enabled - pub fn get_protocol_config(&self) -> ProtocolConfig { - let inner = self.inner.lock(); - inner.protocol_config.as_ref().unwrap().clone() - } - - // Return a dial info filter for what we can receive - pub fn get_inbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { - let inner = self.inner.lock(); - match routing_domain { - RoutingDomain::PublicInternet => inner - .public_inbound_dial_info_filter - .as_ref() - .unwrap() - .clone(), - RoutingDomain::LocalNetwork => inner - .local_inbound_dial_info_filter - .as_ref() - .unwrap() - .clone(), - } - } - pub fn get_inbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { - let dif = self.get_inbound_dial_info_filter(routing_domain); - NodeRefFilter::new() - .with_routing_domain(routing_domain) - .with_dial_info_filter(dif) - } - - // Return a dial info filter for what we can send out - pub fn get_outbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { - let inner = self.inner.lock(); - match routing_domain { - RoutingDomain::PublicInternet => inner - .public_outbound_dial_info_filter - .as_ref() - .unwrap() - .clone(), - RoutingDomain::LocalNetwork => inner - .local_outbound_dial_info_filter - .as_ref() - .unwrap() - .clone(), - } - } - pub fn get_outbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { - let dif = self.get_outbound_dial_info_filter(routing_domain); - NodeRefFilter::new() - .with_routing_domain(routing_domain) - .with_dial_info_filter(dif) - } - // Generates a multi-shot/normal receipt #[instrument(level = "trace", skip(self, extra_data, callback), err)] pub fn generate_receipt>( @@ -890,7 +799,7 @@ impl NetworkManager { }; // Get the udp direct dialinfo for the hole punch - let outbound_nrf = self + let outbound_nrf = routing_table .get_outbound_node_ref_filter(RoutingDomain::PublicInternet) .with_protocol_type(ProtocolType::UDP); peer_nr.set_filter(Some(outbound_nrf)); @@ -1027,7 +936,10 @@ impl NetworkManager { #[instrument(level = "trace", skip(self), ret)] fn get_contact_method_public(&self, target_node_ref: NodeRef) -> ContactMethod { // Scope noderef down to protocols we can do outbound - let public_outbound_nrf = self.get_outbound_node_ref_filter(RoutingDomain::PublicInternet); + let routing_table = self.routing_table(); + + let public_outbound_nrf = + routing_table.get_outbound_node_ref_filter(RoutingDomain::PublicInternet); let target_node_ref = target_node_ref.filtered_clone(public_outbound_nrf.clone()); // Get the best match internet dial info if we have it @@ -1047,16 +959,14 @@ impl NetworkManager { // Can we reach the inbound relay? if inbound_relay_nr.first_filtered_dial_info_detail().is_some() { // Can we receive anything inbound ever? - let our_network_class = self + let our_network_class = routing_table .get_network_class(RoutingDomain::PublicInternet) .unwrap_or(NetworkClass::Invalid); if matches!(our_network_class, NetworkClass::InboundCapable) { - let routing_table = self.routing_table(); - ///////// Reverse connection // Get the best match dial info for an reverse inbound connection - let reverse_dif = self + let reverse_dif = routing_table .get_inbound_dial_info_filter(RoutingDomain::PublicInternet) .filtered( &target_node_ref @@ -1090,7 +1000,7 @@ impl NetworkManager { udp_target_nr.first_filtered_dial_info_detail() { // Does the self node have a direct udp dialinfo the target can reach? - let inbound_udp_dif = self + let inbound_udp_dif = routing_table .get_inbound_dial_info_filter(RoutingDomain::PublicInternet) .filtered( &target_node_ref @@ -1151,7 +1061,10 @@ impl NetworkManager { #[instrument(level = "trace", skip(self), ret)] fn get_contact_method_local(&self, target_node_ref: NodeRef) -> ContactMethod { // Scope noderef down to protocols we can do outbound - let local_outbound_nrf = self.get_outbound_node_ref_filter(RoutingDomain::LocalNetwork); + let routing_table = self.routing_table(); + + let local_outbound_nrf = + routing_table.get_outbound_node_ref_filter(RoutingDomain::LocalNetwork); let target_node_ref = target_node_ref.filtered_clone(local_outbound_nrf); // Get the best matching local direct dial info if we have it @@ -1865,7 +1778,7 @@ impl NetworkManager { let mut bad_public_address_detection_punishment: Option< Box, > = None; - let public_internet_network_class = net + let public_internet_network_class = routing_table .get_network_class(RoutingDomain::PublicInternet) .unwrap_or(NetworkClass::Invalid); let needs_public_address_detection = diff --git a/veilid-core/src/network_manager/native/mod.rs b/veilid-core/src/network_manager/native/mod.rs index 1a58218f..7994a752 100644 --- a/veilid-core/src/network_manager/native/mod.rs +++ b/veilid-core/src/network_manager/native/mod.rs @@ -40,11 +40,9 @@ struct NetworkInner { /// such as dhcp release or change of address or interfaces being added or removed network_needs_restart: bool, /// the calculated protocol configuration for inbound/outbound protocols - protocol_config: Option, + protocol_config: ProtocolConfig, /// set of statically configured protocols with public dialinfo static_public_dialinfo: ProtocolTypeSet, - /// network class per routing domain - network_class: [Option; RoutingDomain::count()], /// join handles for all the low level network background tasks join_handles: Vec>, /// stop source for shutting down the low level network background tasks @@ -120,9 +118,8 @@ impl Network { needs_public_dial_info_check: false, doing_public_dial_info_check: false, public_dial_info_check_punishment: None, - protocol_config: None, + protocol_config: Default::default(), static_public_dialinfo: ProtocolTypeSet::empty(), - network_class: [None, None], join_handles: Vec::new(), stop_source: None, udp_port: 0u16, @@ -620,7 +617,7 @@ impl Network { ///////////////////////////////////////////////////////////////// - pub fn get_protocol_config(&self) -> Option { + pub fn get_protocol_config(&self) -> ProtocolConfig { self.inner.lock().protocol_config } @@ -734,7 +731,8 @@ impl Network { family_local, } }; - inner.protocol_config = Some(protocol_config); + inner.protocol_config = protocol_config; + protocol_config }; @@ -771,27 +769,37 @@ impl Network { // that we have ports available to us self.free_bound_first_ports(); - // If we have static public dialinfo, upgrade our network class + // set up the routing table's network config + // if we have static public dialinfo, upgrade our network class + + editor_public_internet.setup_network( + protocol_config.inbound, + protocol_config.outbound, + protocol_config.family_global, + ); + editor_local_network.setup_network( + protocol_config.inbound, + protocol_config.outbound, + protocol_config.family_local, + ); let detect_address_changes = { let c = self.config.get(); c.network.detect_address_changes }; - if !detect_address_changes { - let mut inner = self.inner.lock(); + let inner = self.inner.lock(); if !inner.static_public_dialinfo.is_empty() { - inner.network_class[RoutingDomain::PublicInternet as usize] = - Some(NetworkClass::InboundCapable); + editor_public_internet.set_network_class(Some(NetworkClass::InboundCapable)); } } - info!("network started"); - self.inner.lock().network_started = true; - // commit routing table edits editor_public_internet.commit().await; editor_local_network.commit().await; + info!("network started"); + self.inner.lock().network_started = true; + Ok(()) } @@ -873,11 +881,6 @@ impl Network { inner.doing_public_dial_info_check } - pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option { - let inner = self.inner.lock(); - inner.network_class[routing_domain as usize] - } - ////////////////////////////////////////// #[instrument(level = "trace", skip(self), err)] @@ -939,6 +942,7 @@ impl Network { // If we need to figure out our network class, tick the task for it if detect_address_changes { let public_internet_network_class = self + .routing_table() .get_network_class(RoutingDomain::PublicInternet) .unwrap_or(NetworkClass::Invalid); let needs_public_dial_info_check = self.needs_public_dial_info_check(); diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/network_class_discovery.rs index b53fefad..aff9bd28 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/network_class_discovery.rs @@ -125,7 +125,7 @@ impl DiscoveryContext { RoutingDomain::PublicInternet, dial_info_filter.clone(), ); - let disallow_relays_filter = move |e: &BucketEntryInner| { + let disallow_relays_filter = move |_rti, e: &BucketEntryInner| { if let Some(n) = e.node_info(RoutingDomain::PublicInternet) { n.relay_peer_info.is_none() } else { @@ -610,12 +610,14 @@ impl Network { _l: u64, _t: u64, ) -> EyreResult<()> { + let routing_table = self.routing_table(); + // Figure out if we can optimize TCP/WS checking since they are often on the same port let (protocol_config, existing_network_class, tcp_same_port) = { let inner = self.inner.lock(); - let protocol_config = inner.protocol_config.unwrap_or_default(); + let protocol_config = inner.protocol_config; let existing_network_class = - inner.network_class[RoutingDomain::PublicInternet as usize]; + routing_table.get_network_class(RoutingDomain::PublicInternet); let tcp_same_port = if protocol_config.inbound.contains(ProtocolType::TCP) && protocol_config.inbound.contains(ProtocolType::WS) { @@ -625,7 +627,6 @@ impl Network { }; (protocol_config, existing_network_class, tcp_same_port) }; - let routing_table = self.routing_table(); // Process all protocol and address combinations let mut futures = FuturesUnordered::new(); @@ -849,17 +850,16 @@ impl Network { // Is the network class different? if existing_network_class != new_network_class { - self.inner.lock().network_class[RoutingDomain::PublicInternet as usize] = - new_network_class; + editor.set_network_class(new_network_class); changed = true; log_net!(debug "PublicInternet network class changed to {:?}", new_network_class); } } else if existing_network_class.is_some() { // Network class could not be determined editor.clear_dial_info_details(); - self.inner.lock().network_class[RoutingDomain::PublicInternet as usize] = None; + editor.set_network_class(None); changed = true; - log_net!(debug "network class cleared"); + log_net!(debug "PublicInternet network class cleared"); } // Punish nodes that told us our public address had changed when it didn't diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index 55396fe3..e41c8c06 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -495,8 +495,8 @@ impl NetworkManager { // even the unreliable ones, and ask them to find nodes close to our node too let noderefs = routing_table.find_fastest_nodes( min_peer_count, - |_k, _v| true, - |k: DHTKey, v: Option>| { + |_rti, _k, _v| true, + |_rti, k: DHTKey, v: Option>| { NodeRef::new(routing_table.clone(), k, v.unwrap().clone(), None) }, ); @@ -525,7 +525,7 @@ impl NetworkManager { // Get our node's current node info and network class and do the right thing let routing_table = self.routing_table(); let node_info = routing_table.get_own_node_info(RoutingDomain::PublicInternet); - let network_class = self.get_network_class(RoutingDomain::PublicInternet); + let network_class = routing_table.get_network_class(RoutingDomain::PublicInternet); // Get routing domain editor let mut editor = routing_table.edit_routing_domain(RoutingDomain::PublicInternet); @@ -594,6 +594,34 @@ impl NetworkManager { Ok(()) } + // Keep private routes assigned and accessible + #[instrument(level = "trace", skip(self), err)] + pub(super) async fn private_route_management_task_routine( + self, + _stop_token: StopToken, + _last_ts: u64, + cur_ts: u64, + ) -> EyreResult<()> { + // Get our node's current node info and network class and do the right thing + let routing_table = self.routing_table(); + let node_info = routing_table.get_own_node_info(RoutingDomain::PublicInternet); + let network_class = routing_table.get_network_class(RoutingDomain::PublicInternet); + + // Get routing domain editor + let mut editor = routing_table.edit_routing_domain(RoutingDomain::PublicInternet); + + // Do we know our network class yet? + if let Some(network_class) = network_class { + + // see if we have any routes that need extending + } + + // Commit the changes + editor.commit().await; + + Ok(()) + } + // Compute transfer statistics for the low level network #[instrument(level = "trace", skip(self), err)] pub(super) async fn rolling_transfers_task_routine( diff --git a/veilid-core/src/network_manager/wasm/mod.rs b/veilid-core/src/network_manager/wasm/mod.rs index c35167a9..d630f598 100644 --- a/veilid-core/src/network_manager/wasm/mod.rs +++ b/veilid-core/src/network_manager/wasm/mod.rs @@ -12,7 +12,7 @@ use std::io; struct NetworkInner { network_started: bool, network_needs_restart: bool, - protocol_config: Option, + protocol_config: ProtocolConfig, } struct NetworkUnlockedInner { @@ -34,7 +34,7 @@ impl Network { NetworkInner { network_started: false, network_needs_restart: false, - protocol_config: None, //join_handle: None, + protocol_config: Default::default(), } } @@ -247,7 +247,7 @@ impl Network { pub async fn startup(&self) -> EyreResult<()> { // get protocol config - self.inner.lock().protocol_config = Some({ + self.inner.lock().protocol_config = { let c = self.config.get(); let inbound = ProtocolTypeSet::new(); let mut outbound = ProtocolTypeSet::new(); @@ -269,7 +269,7 @@ impl Network { family_global, family_local, } - }); + }; self.inner.lock().network_started = true; Ok(()) @@ -337,7 +337,7 @@ impl Network { }; } - pub fn get_protocol_config(&self) -> Option { + pub fn get_protocol_config(&self) -> ProtocolConfig { self.inner.lock().protocol_config.clone() } diff --git a/veilid-core/src/routing_table/bucket.rs b/veilid-core/src/routing_table/bucket.rs index 8312038d..88051c0c 100644 --- a/veilid-core/src/routing_table/bucket.rs +++ b/veilid-core/src/routing_table/bucket.rs @@ -48,13 +48,6 @@ impl Bucket { // newest_entry is updated by kick_bucket() } - pub(super) fn roll_transfers(&self, last_ts: u64, cur_ts: u64) { - // Called every ROLLING_TRANSFERS_INTERVAL_SECS - for (_k, v) in &self.entries { - v.with_mut(|e| e.roll_transfers(last_ts, cur_ts)); - } - } - pub(super) fn entry(&self, key: &DHTKey) -> Option> { self.entries.get(key).cloned() } @@ -63,7 +56,11 @@ impl Bucket { self.entries.iter() } - pub(super) fn kick(&mut self, bucket_depth: usize) -> Option> { + pub(super) fn kick( + &self, + inner: &mut RoutingTableInner, + bucket_depth: usize, + ) -> Option> { // Get number of entries to attempt to purge from bucket let bucket_len = self.entries.len(); @@ -87,8 +84,8 @@ impl Bucket { if a.0 == b.0 { return core::cmp::Ordering::Equal; } - a.1.with(|ea| { - b.1.with(|eb| { + a.1.with(inner, |rti, ea| { + b.1.with(rti, |_rti, eb| { let astate = state_ordering(ea.state(cur_ts)); let bstate = state_ordering(eb.state(cur_ts)); // first kick dead nodes, then unreliable nodes diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index f3f665e8..cb60d8ec 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -132,6 +132,28 @@ impl BucketEntryInner { } } + // Less is more reliable then older + pub fn cmp_oldest_reliable(cur_ts: u64, e1: &Self, e2: &Self) -> std::cmp::Ordering { + // Reverse compare so most reliable is at front + let ret = e2.state(cur_ts).cmp(&e1.state(cur_ts)); + if ret != std::cmp::Ordering::Equal { + return ret; + } + + // Lower timestamp to the front, recent or no timestamp is at the end + if let Some(e1_ts) = &e1.peer_stats.rpc_stats.first_consecutive_seen_ts { + if let Some(e2_ts) = &e2.peer_stats.rpc_stats.first_consecutive_seen_ts { + e1_ts.cmp(&e2_ts) + } else { + std::cmp::Ordering::Less + } + } else if e2.peer_stats.rpc_stats.first_consecutive_seen_ts.is_some() { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + } + pub fn sort_fastest_reliable_fn(cur_ts: u64) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering { move |e1, e2| Self::cmp_fastest_reliable(cur_ts, e1, e2) } @@ -645,20 +667,26 @@ impl BucketEntry { } } - pub(super) fn with(&self, f: F) -> R + // Note, that this requires -also- holding the RoutingTable read lock, as an + // immutable reference to RoutingTableInner must be passed in to get this + // This ensures that an operation on the routing table can not change entries + // while it is being read from + pub(super) fn with(&self, rti: &RoutingTableInner, f: F) -> R where - F: FnOnce(&BucketEntryInner) -> R, + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> R, { let inner = self.inner.read(); - f(&*inner) + f(rti, &*inner) } - pub(super) fn with_mut(&self, f: F) -> R + // Note, that this requires -also- holding the RoutingTable write lock, as a + // mutable reference to RoutingTableInner must be passed in to get this + pub(super) fn with_mut(&self, rti: &mut RoutingTableInner, f: F) -> R where - F: FnOnce(&mut BucketEntryInner) -> R, + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> R, { let mut inner = self.inner.write(); - f(&mut *inner) + f(rti, &mut *inner) } } diff --git a/veilid-core/src/routing_table/debug.rs b/veilid-core/src/routing_table/debug.rs index 05f8bdb8..3fa6fbc7 100644 --- a/veilid-core/src/routing_table/debug.rs +++ b/veilid-core/src/routing_table/debug.rs @@ -102,6 +102,7 @@ impl RoutingTable { pub fn debug_info_entries(&self, limit: usize, min_state: BucketEntryState) -> String { let inner = self.inner.read(); + let inner = &*inner; let cur_ts = intf::get_timestamp(); let mut out = String::new(); @@ -114,14 +115,14 @@ impl RoutingTable { let filtered_entries: Vec<(&DHTKey, &Arc)> = inner.buckets[b] .entries() .filter(|e| { - let state = e.1.with(|e| e.state(cur_ts)); + let state = e.1.with(inner, |_rti, e| e.state(cur_ts)); state >= min_state }) .collect(); if !filtered_entries.is_empty() { out += &format!(" Bucket #{}:\n", b); for e in filtered_entries { - let state = e.1.with(|e| e.state(cur_ts)); + let state = e.1.with(inner, |_rti, e| e.state(cur_ts)); out += &format!( " {} [{}]\n", e.0.encode(), @@ -161,6 +162,7 @@ impl RoutingTable { pub fn debug_info_buckets(&self, min_state: BucketEntryState) -> String { let inner = self.inner.read(); + let inner = &*inner; let cur_ts = intf::get_timestamp(); let mut out = String::new(); @@ -175,7 +177,7 @@ impl RoutingTable { while c < COLS { let mut cnt = 0; for e in inner.buckets[b].entries() { - if e.1.with(|e| e.state(cur_ts) >= min_state) { + if e.1.with(inner, |_rti, e| e.state(cur_ts) >= min_state) { cnt += 1; } } diff --git a/veilid-core/src/routing_table/find_nodes.rs b/veilid-core/src/routing_table/find_nodes.rs index 4485b510..3e3866d4 100644 --- a/veilid-core/src/routing_table/find_nodes.rs +++ b/veilid-core/src/routing_table/find_nodes.rs @@ -17,9 +17,9 @@ impl RoutingTable { pub fn make_inbound_dial_info_entry_filter( routing_domain: RoutingDomain, dial_info_filter: DialInfoFilter, - ) -> impl FnMut(&BucketEntryInner) -> bool { + ) -> impl FnMut(&RoutingTableInner, &BucketEntryInner) -> bool { // does it have matching public dial info? - move |e| { + move |_rti, e| { if let Some(ni) = e.node_info(routing_domain) { if ni .first_filtered_dial_info_detail(DialInfoDetail::NO_SORT, |did| { @@ -35,12 +35,12 @@ impl RoutingTable { } // Makes a filter that finds nodes capable of dialing a particular outbound dialinfo - pub fn make_outbound_dial_info_entry_filter( + pub fn make_outbound_dial_info_entry_filter<'s>( routing_domain: RoutingDomain, dial_info: DialInfo, - ) -> impl FnMut(&BucketEntryInner) -> bool { + ) -> impl FnMut(&RoutingTableInner, &'s BucketEntryInner) -> bool { // does the node's outbound capabilities match the dialinfo? - move |e| { + move |_rti, e| { if let Some(ni) = e.node_info(routing_domain) { let dif = DialInfoFilter::all() .with_protocol_type_set(ni.outbound_protocols) @@ -54,19 +54,19 @@ impl RoutingTable { } // Make a filter that wraps another filter - pub fn combine_entry_filters( + pub fn combine_entry_filters<'a, 'b, F, G>( mut f1: F, mut f2: G, - ) -> impl FnMut(&BucketEntryInner) -> bool + ) -> impl FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool where - F: FnMut(&BucketEntryInner) -> bool, - G: FnMut(&BucketEntryInner) -> bool, + F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, + G: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, { - move |e| { - if !f1(e) { + move |rti, e| { + if !f1(rti, e) { return false; } - if !f2(e) { + if !f2(rti, e) { return false; } true @@ -74,21 +74,21 @@ impl RoutingTable { } // Retrieve the fastest nodes in the routing table matching an entry filter - pub fn find_fast_public_nodes_filtered( + pub fn find_fast_public_nodes_filtered<'r, 'e, F>( &self, node_count: usize, mut entry_filter: F, ) -> Vec where - F: FnMut(&BucketEntryInner) -> bool, + F: FnMut(&'r RoutingTableInner, &'e BucketEntryInner) -> bool, { self.find_fastest_nodes( // count node_count, // filter - |_k: DHTKey, v: Option>| { + |rti, _k: DHTKey, v: Option>| { let entry = v.unwrap(); - entry.with(|e| { + entry.with(rti, |rti, e| { // skip nodes on local network if e.node_info(RoutingDomain::LocalNetwork).is_some() { return false; @@ -98,11 +98,11 @@ impl RoutingTable { return false; } // skip nodes that dont match entry filter - entry_filter(e) + entry_filter(rti, e) }) }, // transform - |k: DHTKey, v: Option>| { + |_rti, k: DHTKey, v: Option>| { NodeRef::new(self.clone(), k, v.unwrap().clone(), None) }, ) @@ -123,9 +123,9 @@ impl RoutingTable { // count protocol_types.len() * 2 * max_per_type, // filter - move |_k: DHTKey, v: Option>| { + move |rti, _k: DHTKey, v: Option>| { let entry = v.unwrap(); - entry.with(|e| { + entry.with(rti, |_rti, e| { // skip nodes on our local network here if e.has_node_info(RoutingDomain::LocalNetwork.into()) { return false; @@ -164,20 +164,21 @@ impl RoutingTable { }) }, // transform - |k: DHTKey, v: Option>| { + |_rti, k: DHTKey, v: Option>| { NodeRef::new(self.clone(), k, v.unwrap().clone(), None) }, ) } - pub fn filter_has_valid_signed_node_info( - &self, + pub fn filter_has_valid_signed_node_info_inner( + inner: &RoutingTableInner, routing_domain: RoutingDomain, + has_valid_own_node_info: bool, v: Option>, ) -> bool { match v { - None => self.has_valid_own_node_info(routing_domain), - Some(entry) => entry.with(|e| { + None => has_valid_own_node_info, + Some(entry) => entry.with(inner, |_rti, e| { e.signed_node_info(routing_domain.into()) .map(|sni| sni.has_valid_signature()) .unwrap_or(false) @@ -185,15 +186,18 @@ impl RoutingTable { } } - pub fn transform_to_peer_info( - &self, + pub fn transform_to_peer_info_inner( + inner: &RoutingTableInner, routing_domain: RoutingDomain, + own_peer_info: PeerInfo, k: DHTKey, v: Option>, ) -> PeerInfo { match v { - None => self.get_own_peer_info(routing_domain), - Some(entry) => entry.with(|e| e.make_peer_info(k, routing_domain).unwrap()), + None => own_peer_info, + Some(entry) => entry.with(inner, |_rti, e| { + e.make_peer_info(k, routing_domain).unwrap() + }), } } @@ -206,14 +210,16 @@ impl RoutingTable { mut transform: T, ) -> Vec where - F: FnMut(DHTKey, Option>) -> bool, + F: FnMut(&RoutingTableInner, DHTKey, Option>) -> bool, C: FnMut( + &RoutingTableInner, &(DHTKey, Option>), &(DHTKey, Option>), ) -> core::cmp::Ordering, - T: FnMut(DHTKey, Option>) -> O, + T: FnMut(&RoutingTableInner, DHTKey, Option>) -> O, { let inner = self.inner.read(); + let inner = &*inner; let self_node_id = self.unlocked_inner.node_id; // collect all the nodes for sorting @@ -221,27 +227,32 @@ impl RoutingTable { Vec::<(DHTKey, Option>)>::with_capacity(inner.bucket_entry_count + 1); // add our own node (only one of there with the None entry) - if filter(self_node_id, None) { + if filter(inner, self_node_id, None) { nodes.push((self_node_id, None)); } // add all nodes from buckets - Self::with_entries(&*inner, cur_ts, BucketEntryState::Unreliable, |k, v| { - // Apply filter - if filter(k, Some(v.clone())) { - nodes.push((k, Some(v.clone()))); - } - Option::<()>::None - }); + Self::with_entries( + &*inner, + cur_ts, + BucketEntryState::Unreliable, + |rti, k, v| { + // Apply filter + if filter(rti, k, Some(v.clone())) { + nodes.push((k, Some(v.clone()))); + } + Option::<()>::None + }, + ); // sort by preference for returning nodes - nodes.sort_by(compare); + nodes.sort_by(|a, b| compare(inner, a, b)); // return transformed vector for filtered+sorted nodes let cnt = usize::min(node_count, nodes.len()); let mut out = Vec::::with_capacity(cnt); for node in nodes { - let val = transform(node.0, node.1); + let val = transform(inner, node.0, node.1); out.push(val); } @@ -255,21 +266,21 @@ impl RoutingTable { transform: T, ) -> Vec where - F: FnMut(DHTKey, Option>) -> bool, - T: FnMut(DHTKey, Option>) -> O, + F: FnMut(&RoutingTableInner, DHTKey, Option>) -> bool, + T: FnMut(&RoutingTableInner, DHTKey, Option>) -> O, { let cur_ts = intf::get_timestamp(); let out = self.find_peers_with_sort_and_filter( node_count, cur_ts, // filter - |k, v| { + |rti, k, v| { if let Some(entry) = &v { // always filter out dead nodes - if entry.with(|e| e.state(cur_ts) == BucketEntryState::Dead) { + if entry.with(rti, |_rti, e| e.state(cur_ts) == BucketEntryState::Dead) { false } else { - filter(k, v) + filter(rti, k, v) } } else { // always filter out self peer, as it is irrelevant to the 'fastest nodes' search @@ -277,7 +288,7 @@ impl RoutingTable { } }, // sort - |(a_key, a_entry), (b_key, b_entry)| { + |rti, (a_key, a_entry), (b_key, b_entry)| { // same nodes are always the same if a_key == b_key { return core::cmp::Ordering::Equal; @@ -292,8 +303,8 @@ impl RoutingTable { // reliable nodes come first let ae = a_entry.as_ref().unwrap(); let be = b_entry.as_ref().unwrap(); - ae.with(|ae| { - be.with(|be| { + ae.with(rti, |rti, ae| { + be.with(rti, |_rti, be| { let ra = ae.check_reliable(cur_ts); let rb = be.check_reliable(cur_ts); if ra != rb { @@ -337,8 +348,8 @@ impl RoutingTable { mut transform: T, ) -> Vec where - F: FnMut(DHTKey, Option>) -> bool, - T: FnMut(DHTKey, Option>) -> O, + F: FnMut(&RoutingTableInner, DHTKey, Option>) -> bool, + T: FnMut(&RoutingTableInner, DHTKey, Option>) -> O, { let cur_ts = intf::get_timestamp(); let node_count = { @@ -351,7 +362,7 @@ impl RoutingTable { // filter filter, // sort - |(a_key, a_entry), (b_key, b_entry)| { + |rti, (a_key, a_entry), (b_key, b_entry)| { // same nodes are always the same if a_key == b_key { return core::cmp::Ordering::Equal; @@ -360,10 +371,10 @@ impl RoutingTable { // reliable nodes come first, pessimistically treating our own node as unreliable let ra = a_entry .as_ref() - .map_or(false, |x| x.with(|x| x.check_reliable(cur_ts))); + .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); let rb = b_entry .as_ref() - .map_or(false, |x| x.with(|x| x.check_reliable(cur_ts))); + .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); if ra != rb { if ra { return core::cmp::Ordering::Less; @@ -420,9 +431,7 @@ impl RoutingTable { fn make_public_internet_relay_node_filter(&self) -> impl Fn(&BucketEntryInner) -> bool { // Get all our outbound protocol/address types - let outbound_dif = self - .network_manager() - .get_outbound_dial_info_filter(RoutingDomain::PublicInternet); + let outbound_dif = self.get_outbound_dial_info_filter(RoutingDomain::PublicInternet); let mapped_port_info = self.get_low_level_port_info(); move |e: &BucketEntryInner| { @@ -481,9 +490,9 @@ impl RoutingTable { let mut best_inbound_relay: Option<(DHTKey, Arc)> = None; // Iterate all known nodes for candidates - Self::with_entries(inner, cur_ts, BucketEntryState::Unreliable, |k, v| { + Self::with_entries(inner, cur_ts, BucketEntryState::Unreliable, |rti, k, v| { let v2 = v.clone(); - v.with(|e| { + v.with(rti, |rti, e| { // Ensure we have the node's status if let Some(node_status) = e.node_status(routing_domain) { // Ensure the node will relay @@ -491,7 +500,7 @@ impl RoutingTable { // Compare against previous candidate if let Some(best_inbound_relay) = best_inbound_relay.as_mut() { // Less is faster - let better = best_inbound_relay.1.with(|best| { + let better = best_inbound_relay.1.with(rti, |_rti, best| { BucketEntryInner::cmp_fastest_reliable(cur_ts, e, best) == std::cmp::Ordering::Less }); diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index cc831641..1e86c2d6 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -3,6 +3,7 @@ mod bucket_entry; mod debug; mod find_nodes; mod node_ref; +mod route_spec_store; mod routing_domain_editor; mod routing_domains; mod stats_accounting; @@ -19,6 +20,7 @@ pub use debug::*; pub use find_nodes::*; use hashlink::LruCache; pub use node_ref::*; +pub use route_spec_store::*; pub use routing_domain_editor::*; pub use routing_domains::*; pub use stats_accounting::*; @@ -41,7 +43,7 @@ struct RoutingTableInner { /// The public internet routing domain public_internet_routing_domain: PublicInternetRoutingDomainDetail, /// The dial info we use on the local network - local_network_routing_domain: LocalInternetRoutingDomainDetail, + local_network_routing_domain: LocalNetworkRoutingDomainDetail, /// Interim accounting mechanism for this node's RPC latency to any other node self_latency_stats_accounting: LatencyStatsAccounting, /// Interim accounting mechanism for the total bandwidth to/from this node @@ -50,6 +52,8 @@ struct RoutingTableInner { self_transfer_stats: TransferStatsDownUp, /// Peers we have recently communicated with recent_peers: LruCache, + /// Storage for private/safety RouteSpecs + route_spec_store: RouteSpecStore, } #[derive(Clone, Debug, Default)] @@ -90,12 +94,13 @@ impl RoutingTable { RoutingTableInner { buckets: Vec::new(), public_internet_routing_domain: PublicInternetRoutingDomainDetail::default(), - local_network_routing_domain: LocalInternetRoutingDomainDetail::default(), + local_network_routing_domain: LocalNetworkRoutingDomainDetail::default(), bucket_entry_count: 0, self_latency_stats_accounting: LatencyStatsAccounting::new(), self_transfer_stats_accounting: TransferStatsAccounting::new(), self_transfer_stats: TransferStatsDownUp::default(), recent_peers: LruCache::new(RECENT_PEERS_TABLE_SIZE), + route_spec_store: RouteSpecStore::new(), } } fn new_unlocked_inner( @@ -214,17 +219,21 @@ impl RoutingTable { pub fn relay_node(&self, domain: RoutingDomain) -> Option { let inner = self.inner.read(); - Self::with_routing_domain(&*inner, domain, |rd| rd.relay_node()) + Self::with_routing_domain(&*inner, domain, |rd| rd.common().relay_node()) } pub fn has_dial_info(&self, domain: RoutingDomain) -> bool { let inner = self.inner.read(); - Self::with_routing_domain(&*inner, domain, |rd| !rd.dial_info_details().is_empty()) + Self::with_routing_domain(&*inner, domain, |rd| { + !rd.common().dial_info_details().is_empty() + }) } pub fn dial_info_details(&self, domain: RoutingDomain) -> Vec { let inner = self.inner.read(); - Self::with_routing_domain(&*inner, domain, |rd| rd.dial_info_details().clone()) + Self::with_routing_domain(&*inner, domain, |rd| { + rd.common().dial_info_details().clone() + }) } pub fn first_filtered_dial_info_detail( @@ -235,7 +244,7 @@ impl RoutingTable { let inner = self.inner.read(); for routing_domain in routing_domain_set { let did = Self::with_routing_domain(&*inner, routing_domain, |rd| { - for did in rd.dial_info_details() { + for did in rd.common().dial_info_details() { if did.matches_filter(filter) { return Some(did.clone()); } @@ -258,7 +267,7 @@ impl RoutingTable { let mut ret = Vec::new(); for routing_domain in routing_domain_set { Self::with_routing_domain(&*inner, routing_domain, |rd| { - for did in rd.dial_info_details() { + for did in rd.common().dial_info_details() { if did.matches_filter(filter) { ret.push(did.clone()); } @@ -321,8 +330,8 @@ impl RoutingTable { fn reset_all_seen_our_node_info(inner: &mut RoutingTableInner, routing_domain: RoutingDomain) { let cur_ts = intf::get_timestamp(); - Self::with_entries(&*inner, cur_ts, BucketEntryState::Dead, |_, v| { - v.with_mut(|e| { + Self::with_entries_mut(inner, cur_ts, BucketEntryState::Dead, |rti, _, v| { + v.with_mut(rti, |_rti, e| { e.set_seen_our_node_info(routing_domain, false); }); Option::<()>::None @@ -331,50 +340,83 @@ impl RoutingTable { fn reset_all_updated_since_last_network_change(inner: &mut RoutingTableInner) { let cur_ts = intf::get_timestamp(); - Self::with_entries(&*inner, cur_ts, BucketEntryState::Dead, |_, v| { - v.with_mut(|e| e.set_updated_since_last_network_change(false)); + Self::with_entries_mut(inner, cur_ts, BucketEntryState::Dead, |rti, _, v| { + v.with_mut(rti, |_rti, e| { + e.set_updated_since_last_network_change(false) + }); Option::<()>::None }); } + /// Return a copy of our node's peerinfo pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo { - PeerInfo::new( - NodeId::new(self.node_id()), - self.get_own_signed_node_info(routing_domain), - ) + let inner = &*self.inner.read(); + Self::with_routing_domain(inner, routing_domain, |rdd| { + rdd.common().with_peer_info(|pi| pi.clone()) + }) } + /// Return a copy of our node's signednodeinfo pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedNodeInfo { - let node_id = NodeId::new(self.node_id()); - let secret = self.node_id_secret(); - SignedNodeInfo::with_secret(self.get_own_node_info(routing_domain), node_id, &secret) - .unwrap() + let inner = &*self.inner.read(); + Self::with_routing_domain(inner, routing_domain, |rdd| { + rdd.common() + .with_peer_info(|pi| pi.signed_node_info.clone()) + }) } + /// Return a copy of our node's nodeinfo pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo { - let netman = self.network_manager(); - let relay_node = self.relay_node(routing_domain); - let pc = netman.get_protocol_config(); - NodeInfo { - network_class: netman - .get_network_class(routing_domain) - .unwrap_or(NetworkClass::Invalid), - outbound_protocols: pc.outbound, - address_types: pc.family_global, - min_version: MIN_VERSION, - max_version: MAX_VERSION, - dial_info_detail_list: self.dial_info_details(routing_domain), - relay_peer_info: relay_node - .and_then(|rn| rn.make_peer_info(routing_domain).map(Box::new)), - } + let inner = &*self.inner.read(); + Self::with_routing_domain(inner, routing_domain, |rdd| { + rdd.common() + .with_peer_info(|pi| pi.signed_node_info.node_info.clone()) + }) } + /// Return our currently registered network class pub fn has_valid_own_node_info(&self, routing_domain: RoutingDomain) -> bool { - let netman = self.network_manager(); - let nc = netman - .get_network_class(routing_domain) - .unwrap_or(NetworkClass::Invalid); - !matches!(nc, NetworkClass::Invalid) + let inner = &*self.inner.read(); + Self::with_routing_domain(inner, routing_domain, |rdd| { + rdd.common().has_valid_own_node_info() + }) + } + + /// Return the domain's currently registered network class + pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option { + let inner = &*self.inner.read(); + Self::with_routing_domain(inner, routing_domain, |rdd| rdd.common().network_class()) + } + + /// Return the domain's filter for what we can receivein the form of a dial info filter + pub fn get_inbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { + let inner = &*self.inner.read(); + Self::with_routing_domain(inner, routing_domain, |rdd| { + rdd.common().inbound_dial_info_filter() + }) + } + + /// Return the domain's filter for what we can receive in the form of a node ref filter + pub fn get_inbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { + let dif = self.get_inbound_dial_info_filter(routing_domain); + NodeRefFilter::new() + .with_routing_domain(routing_domain) + .with_dial_info_filter(dif) + } + + /// Return the domain's filter for what we can send out in the form of a dial info filter + pub fn get_outbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { + let inner = &*self.inner.read(); + Self::with_routing_domain(inner, routing_domain, |rdd| { + rdd.common().outbound_dial_info_filter() + }) + } + /// Return the domain's filter for what we can receive in the form of a node ref filter + pub fn get_outbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { + let dif = self.get_outbound_dial_info_filter(routing_domain); + NodeRefFilter::new() + .with_routing_domain(routing_domain) + .with_dial_info_filter(dif) } fn bucket_depth(index: usize) -> usize { @@ -434,8 +476,8 @@ impl RoutingTable { // If the local network topology has changed, nuke the existing local node info and let new local discovery happen if changed { let cur_ts = intf::get_timestamp(); - Self::with_entries(&*inner, cur_ts, BucketEntryState::Dead, |_rti, e| { - e.with_mut(|e| { + Self::with_entries_mut(&mut *inner, cur_ts, BucketEntryState::Dead, |rti, _, e| { + e.with_mut(rti, |_rti, e| { e.clear_signed_node_info(RoutingDomain::LocalNetwork); e.set_seen_our_node_info(RoutingDomain::LocalNetwork, false); e.set_updated_since_last_network_change(false); @@ -449,12 +491,13 @@ impl RoutingTable { // should only be performed when there are no node_refs (detached) pub fn purge_buckets(&self) { let mut inner = self.inner.write(); + let inner = &mut *inner; log_rtab!( "Starting routing table buckets purge. Table currently has {} nodes", inner.bucket_entry_count ); - for bucket in &mut inner.buckets { - bucket.kick(0); + for bucket in &inner.buckets { + bucket.kick(inner, 0); } log_rtab!(debug "Routing table buckets purge complete. Routing table now has {} nodes", @@ -465,13 +508,14 @@ impl RoutingTable { // Attempt to remove last_connections from entries pub fn purge_last_connections(&self) { let mut inner = self.inner.write(); + let inner = &mut *inner; log_rtab!( "Starting routing table last_connections purge. Table currently has {} nodes", inner.bucket_entry_count ); - for bucket in &mut inner.buckets { + for bucket in &inner.buckets { for entry in bucket.entries() { - entry.1.with_mut(|e| { + entry.1.with_mut(inner, |_rti, e| { e.clear_last_connections(); }); } @@ -488,7 +532,7 @@ impl RoutingTable { let bucket = &mut inner.buckets[idx]; let bucket_depth = Self::bucket_depth(idx); - if let Some(dead_node_ids) = bucket.kick(bucket_depth) { + if let Some(dead_node_ids) = bucket.kick(inner, bucket_depth) { // Remove counts inner.bucket_entry_count -= dead_node_ids.len(); log_rtab!(debug "Routing table now has {} nodes", inner.bucket_entry_count); @@ -524,8 +568,8 @@ impl RoutingTable { ) -> usize { let mut count = 0usize; let cur_ts = intf::get_timestamp(); - Self::with_entries(inner, cur_ts, min_state, |_, e| { - if e.with(|e| e.best_routing_domain(routing_domain_set)) + Self::with_entries(inner, cur_ts, min_state, |rti, _, e| { + if e.with(rti, |_rti, e| e.best_routing_domain(routing_domain_set)) .is_some() { count += 1; @@ -535,7 +579,7 @@ impl RoutingTable { count } - fn with_entries) -> Option>( + fn with_entries) -> Option>( inner: &RoutingTableInner, cur_ts: u64, min_state: BucketEntryState, @@ -543,8 +587,29 @@ impl RoutingTable { ) -> Option { for bucket in &inner.buckets { for entry in bucket.entries() { - if entry.1.with(|e| e.state(cur_ts) >= min_state) { - if let Some(out) = f(*entry.0, entry.1.clone()) { + if entry.1.with(inner, |_rti, e| e.state(cur_ts) >= min_state) { + if let Some(out) = f(inner, *entry.0, entry.1.clone()) { + return Some(out); + } + } + } + } + None + } + + fn with_entries_mut< + T, + F: FnMut(&mut RoutingTableInner, DHTKey, Arc) -> Option, + >( + inner: &mut RoutingTableInner, + cur_ts: u64, + min_state: BucketEntryState, + mut f: F, + ) -> Option { + for bucket in &inner.buckets { + for entry in bucket.entries() { + if entry.1.with(inner, |_rti, e| e.state(cur_ts) >= min_state) { + if let Some(out) = f(inner, *entry.0, entry.1.clone()) { return Some(out); } } @@ -561,18 +626,23 @@ impl RoutingTable { ) -> Vec { let inner = self.inner.read(); let mut node_refs = Vec::::with_capacity(inner.bucket_entry_count); - Self::with_entries(&*inner, cur_ts, BucketEntryState::Unreliable, |k, v| { - // Only update nodes that haven't seen our node info yet - if all || !v.with(|e| e.has_seen_our_node_info(routing_domain)) { - node_refs.push(NodeRef::new( - self.clone(), - k, - v, - Some(NodeRefFilter::new().with_routing_domain(routing_domain)), - )); - } - Option::<()>::None - }); + Self::with_entries( + &*inner, + cur_ts, + BucketEntryState::Unreliable, + |rti, k, v| { + // Only update nodes that haven't seen our node info yet + if all || !v.with(rti, |_rti, e| e.has_seen_our_node_info(routing_domain)) { + node_refs.push(NodeRef::new( + self.clone(), + k, + v, + Some(NodeRefFilter::new().with_routing_domain(routing_domain)), + )); + } + Option::<()>::None + }, + ); node_refs } @@ -585,35 +655,45 @@ impl RoutingTable { // Collect relay nodes let opt_relay_id = Self::with_routing_domain(&*inner, routing_domain, |rd| { - rd.relay_node().map(|rn| rn.node_id()) + rd.common().relay_node().map(|rn| rn.node_id()) }); // Collect all entries that are 'needs_ping' and have some node info making them reachable somehow let mut node_refs = Vec::::with_capacity(inner.bucket_entry_count); - Self::with_entries(&*inner, cur_ts, BucketEntryState::Unreliable, |k, v| { - if v.with(|e| { - e.has_node_info(routing_domain.into()) - && e.needs_ping(cur_ts, opt_relay_id == Some(k)) - }) { - node_refs.push(NodeRef::new( - self.clone(), - k, - v, - Some(NodeRefFilter::new().with_routing_domain(routing_domain)), - )); - } - Option::<()>::None - }); + Self::with_entries( + &*inner, + cur_ts, + BucketEntryState::Unreliable, + |rti, k, v| { + if v.with(rti, |_rti, e| { + e.has_node_info(routing_domain.into()) + && e.needs_ping(cur_ts, opt_relay_id == Some(k)) + }) { + node_refs.push(NodeRef::new( + self.clone(), + k, + v, + Some(NodeRefFilter::new().with_routing_domain(routing_domain)), + )); + } + Option::<()>::None + }, + ); node_refs } pub fn get_all_nodes(&self, cur_ts: u64) -> Vec { let inner = self.inner.read(); let mut node_refs = Vec::::with_capacity(inner.bucket_entry_count); - Self::with_entries(&*inner, cur_ts, BucketEntryState::Unreliable, |k, v| { - node_refs.push(NodeRef::new(self.clone(), k, v, None)); - Option::<()>::None - }); + Self::with_entries( + &*inner, + cur_ts, + BucketEntryState::Unreliable, + |_rti, k, v| { + node_refs.push(NodeRef::new(self.clone(), k, v, None)); + Option::<()>::None + }, + ); node_refs } @@ -627,7 +707,7 @@ impl RoutingTable { // in a locked fashion as to ensure the bucket entry state is always valid pub fn create_node_ref(&self, node_id: DHTKey, update_func: F) -> Option where - F: FnOnce(&mut BucketEntryInner), + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner), { // Ensure someone isn't trying register this node itself if node_id == self.node_id() { @@ -637,6 +717,7 @@ impl RoutingTable { // Lock this entire operation let mut inner = self.inner.write(); + let inner = &mut *inner; // Look up existing entry let idx = self.find_bucket_index(node_id); @@ -657,7 +738,7 @@ impl RoutingTable { // Update the entry let entry = bucket.entry(&node_id).unwrap(); - entry.with_mut(update_func); + entry.with_mut(inner, update_func); // Kick the bucket self.unlocked_inner.kick_queue.lock().insert(idx); @@ -669,9 +750,7 @@ impl RoutingTable { // Update the entry let bucket = &mut inner.buckets[idx]; let entry = bucket.entry(&node_id).unwrap(); - entry.with_mut(|e| { - update_func(e); - }); + entry.with_mut(inner, update_func); nr } @@ -731,7 +810,7 @@ impl RoutingTable { } } - self.create_node_ref(node_id, |e| { + self.create_node_ref(node_id, |_rti, e| { e.update_signed_node_info(routing_domain, signed_node_info); }) .map(|mut nr| { @@ -750,7 +829,7 @@ impl RoutingTable { descriptor: ConnectionDescriptor, timestamp: u64, ) -> Option { - let out = self.create_node_ref(node_id, |e| { + let out = self.create_node_ref(node_id, |_rti, e| { // this node is live because it literally just connected to us e.touch_last_seen(timestamp); }); @@ -783,9 +862,10 @@ impl RoutingTable { let mut health = RoutingTableHealth::default(); let cur_ts = intf::get_timestamp(); let inner = self.inner.read(); + let inner = &*inner; for bucket in &inner.buckets { for (_, v) in bucket.entries() { - match v.with(|e| e.state(cur_ts)) { + match v.with(inner, |_rti, e| e.state(cur_ts)) { BucketEntryState::Reliable => { health.reliable_entry_count += 1; } diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index d686e84d..4586189c 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -103,7 +103,7 @@ impl NodeRef { F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, { let inner = &*self.routing_table.inner.read(); - self.entry.with(|e| f(inner, e)) + self.entry.with(inner, f) } pub(super) fn operate_mut(&self, f: F) -> T @@ -111,7 +111,7 @@ impl NodeRef { F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, { let inner = &mut *self.routing_table.inner.write(); - self.entry.with_mut(|e| f(inner, e)) + self.entry.with_mut(inner, f) } // Filtering diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs new file mode 100644 index 00000000..480f6a7f --- /dev/null +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -0,0 +1,334 @@ +use super::*; +use crate::veilid_api::*; +use serde::*; + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct RouteSpecDetail { + /// The actual route spec + #[serde(with = "arc_serialize")] + route_spec: Arc, + /// Transfers up and down + transfer_stats_down_up: TransferStatsDownUp, + /// Latency stats + latency_stats: LatencyStats, + /// Accounting mechanism for this route's RPC latency + #[serde(skip)] + latency_stats_accounting: LatencyStatsAccounting, + /// Accounting mechanism for the bandwidth across this route + #[serde(skip)] + transfer_stats_accounting: TransferStatsAccounting, + /// Published private route, do not reuse for ephemeral routes + #[serde(skip)] + published: bool, + /// Timestamp of when the route was created + timestamp: u64, +} + +/// The core representation of the RouteSpecStore that can be serialized +#[derive(Debug, Serialize, Deserialize)] +pub struct RouteSpecStoreContent { + /// All of the routes we have allocated so far + details: HashMap, +} + +/// Ephemeral data used to help the RouteSpecStore operate efficiently +#[derive(Debug, Default)] +pub struct RouteSpecStoreCache { + /// The fastest routes by latency + fastest_routes: Vec, + /// The most reliable routes by node lifetime longevity + reliable_routes: Vec, + /// How many times nodes have been used + used_nodes: HashMap, + /// How many times nodes have been used at the terminal point of a route + used_end_nodes: HashMap, + /// Route spec hop cache, used to quickly disqualify routes + hop_cache: HashSet>, +} + +#[derive(Debug)] +pub struct RouteSpecStore { + /// Serialize RouteSpecStore content + content: RouteSpecStoreContent, + /// RouteSpecStore cache + cache: RouteSpecStoreCache, +} + +fn route_spec_to_hop_cache(spec: Arc) -> Vec { + let mut cache: Vec = Vec::with_capacity(spec.hops.len() * DHT_KEY_LENGTH); + for hop in spec.hops { + cache.extend_from_slice(&hop.dial_info.node_id.key.bytes); + } + cache +} + +fn node_sublist_to_hop_cache( + nodes: &[(DHTKey, Arc)], + start: usize, + len: usize, +) -> Vec { + let mut cache: Vec = Vec::with_capacity(len * DHT_KEY_LENGTH); + for node in &nodes[start..start + len] { + cache.extend_from_slice(&node.0.bytes) + } + cache +} + +impl RouteSpecStore { + pub fn new() -> Self { + Self { + content: RouteSpecStoreContent { + details: HashMap::new(), + }, + cache: Default::default(), + } + } + + pub fn from_cbor( + routing_table: RoutingTable, + cbor: &[u8], + ) -> Result { + let content: RouteSpecStoreContent = serde_cbor::from_slice(cbor) + .map_err(|e| VeilidAPIError::parse_error("invalid route spec store content", e))?; + let rss = RouteSpecStore { + content, + cache: Default::default(), + }; + rss.rebuild_cache(); + Ok(rss) + } + + pub fn to_cbor(&self) -> Vec { + serde_cbor::to_vec(&self.content).unwrap() + } + + fn rebuild_cache(&mut self) { + // + } + + fn detail_mut(&mut self, spec: Arc) -> &mut RouteSpecDetail { + self.content.details.get_mut(&spec.public_key).unwrap() + } + + /// Create a new route + /// Prefers nodes that are not currently in use by another route + /// The route is not yet tested for its reachability + /// Returns None if no route could be allocated at this time + pub fn allocate_route( + &mut self, + routing_table: RoutingTable, + reliable: bool, + hop_count: usize, + ) -> Option> { + use core::cmp::Ordering; + + let max_route_hop_count = { + let config = routing_table.network_manager().config(); + let c = config.get(); + let max_route_hop_count = c.network.rpc.max_route_hop_count; + max_route_hop_count.into() + }; + + if hop_count < 2 { + log_rtab!(error "Not allocating route less than two hops in length"); + return None; + } + + if hop_count > max_route_hop_count { + log_rtab!(error "Not allocating route longer than max route hop count"); + return None; + } + + // Get list of all nodes, and sort them for selection + let cur_ts = intf::get_timestamp(); + let dial_info_sort = if reliable { + Some(DialInfoDetail::reliable_sort) + } else { + None + }; + let filter = |rti, k: DHTKey, v: Option>| -> bool { + // Exclude our own node from routes + if v.is_none() { + return false; + } + let v = v.unwrap(); + + // Exclude nodes on our local network + let on_local_network = v.with(rti, |_rti, e| { + e.node_info(RoutingDomain::LocalNetwork).is_some() + }); + if on_local_network { + return false; + } + + // Exclude nodes with no publicinternet nodeinfo, or incompatible nodeinfo or node status won't route + v.with(rti, |_rti, e| { + let node_info_ok = if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) { + ni.has_any_dial_info() + } else { + false + }; + let node_status_ok = if let Some(ns) = e.node_status(RoutingDomain::PublicInternet) + { + ns.will_route() + } else { + false + }; + + node_info_ok && node_status_ok + }) + }; + let compare = |rti, + v1: &(DHTKey, Option>), + v2: &(DHTKey, Option>)| + -> Ordering { + // deprioritize nodes that we have already used as end points + let e1_used_end = self + .cache + .used_end_nodes + .get(&v1.0) + .cloned() + .unwrap_or_default(); + let e2_used_end = self + .cache + .used_end_nodes + .get(&v2.0) + .cloned() + .unwrap_or_default(); + let cmp_used_end = e1_used_end.cmp(&e2_used_end); + if !matches!(cmp_used_end, Ordering::Equal) { + return cmp_used_end; + } + + // deprioritize nodes we have used already anywhere + let e1_used = self + .cache + .used_nodes + .get(&v1.0) + .cloned() + .unwrap_or_default(); + let e2_used = self + .cache + .used_nodes + .get(&v2.0) + .cloned() + .unwrap_or_default(); + let cmp_used = e1_used.cmp(&e2_used); + if !matches!(cmp_used, Ordering::Equal) { + return cmp_used; + } + + // always prioritize reliable nodes, but sort by oldest or fastest + let cmpout = v1.1.unwrap().with(rti, |rti, e1| { + v2.1.unwrap().with(rti, |_rti, e2| { + if reliable { + BucketEntryInner::cmp_oldest_reliable(cur_ts, e1, e2) + } else { + BucketEntryInner::cmp_fastest_reliable(cur_ts, e1, e2) + } + }) + }); + cmpout + }; + let transform = |rti, k: DHTKey, v: Option>| -> (DHTKey, NodeInfo) { + // Return the key and the nodeinfo for that key + ( + k, + v.unwrap().with(rti, |_rti, e| { + e.node_info(RoutingDomain::PublicInternet.into()) + .unwrap() + .clone() + }), + ) + }; + + // Pull the whole routing table in sorted order + let node_count = routing_table.get_entry_count( + RoutingDomain::PublicInternet.into(), + BucketEntryState::Unreliable, + ); + let mut nodes = routing_table + .find_peers_with_sort_and_filter(node_count, cur_ts, filter, compare, transform); + + // If we couldn't find enough nodes, wait until we have more nodes in the routing table + if nodes.len() < hop_count { + log_rtab!(debug "Not enough nodes to construct route at this time. Try again later."); + return None; + } + + // Now go through nodes and try to build a route we haven't seen yet + let mut route_nodes = None; + for start in 0..(nodes.len() - hop_count) { + // Get the route cache key + let key = node_sublist_to_hop_cache(&nodes, start, hop_count); + + // try each route until we find a unique one + if !self.cache.hop_cache.contains(&key) { + route_nodes = Some(&nodes[start..start + hop_count]); + break; + } + } + if route_nodes.is_none() { + return None; + } + let route_node = route_nodes.unwrap(); + + // Got a unique route, lets build the detail, register it, and return it + let hops: Vec = route_node + .into_iter() + .map(|v| RouteHopSpec { + dial_info: NodeDialInfo { + node_id: NodeId::new(v.0), + dial_info: xxx, + }, + }) + .collect(); + + let (public_key, secret_key) = generate_secret(); + let route_spec = Arc::new(RouteSpec { + public_key, + secret_key, + hops, + }); + + let rsd = RouteSpecDetail { + route_spec, + transfer_stats_down_up: Default::default(), + latency_stats: Default::default(), + latency_stats_accounting: Default::default(), + transfer_stats_accounting: Default::default(), + published: false, + timestamp: cur_ts, + }; + + None + } + + pub fn release_route(&mut self, spec: Arc) {} + + pub fn best_route(&mut self, reliable: bool) -> Arc {} + + /// Mark route as published + /// When first deserialized, routes must be re-published in order to ensure they remain + /// in the RouteSpecStore. + pub fn publish_route(&mut self, spec: Arc) { + //compile private route here? + } + + pub fn record_latency( + &mut self, + spec: Arc, + latency: u64, + ) -> veilid_api::LatencyStats { + } + + pub fn add_down(&mut self, spec: Arc, bytes: u64) { + self.current_transfer.down += bytes; + } + + pub fn add_up(&mut self, spec: Arc, bytes: u64) {} + + pub fn roll_transfers(&mut self) { + // + } +} diff --git a/veilid-core/src/routing_table/routing_domain_editor.rs b/veilid-core/src/routing_table/routing_domain_editor.rs index fac64af7..c507b3fa 100644 --- a/veilid-core/src/routing_table/routing_domain_editor.rs +++ b/veilid-core/src/routing_table/routing_domain_editor.rs @@ -3,8 +3,24 @@ use super::*; enum RoutingDomainChange { ClearDialInfoDetails, ClearRelayNode, - SetRelayNode { relay_node: NodeRef }, - AddDialInfoDetail { dial_info_detail: DialInfoDetail }, + SetRelayNode { + relay_node: NodeRef, + }, + AddDialInfoDetail { + dial_info_detail: DialInfoDetail, + }, + SetupNode { + node_id: DHTKey, + node_id_secret: DHTKeySecret, + }, + SetupNetwork { + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + }, + SetNetworkClass { + network_class: Option, + }, } pub struct RoutingDomainEditor { @@ -67,9 +83,40 @@ impl RoutingDomainEditor { Ok(()) } + #[instrument(level = "debug", skip(self))] + pub fn setup_node(&mut self, node_id: DHTKey, node_id_secret: DHTKeySecret) { + self.changes.push(RoutingDomainChange::SetupNode { + node_id, + node_id_secret, + }) + } + #[instrument(level = "debug", skip(self))] + pub fn setup_network( + &mut self, + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + ) { + self.changes.push(RoutingDomainChange::SetupNetwork { + outbound_protocols, + inbound_protocols, + address_types, + }) + } + + #[instrument(level = "debug", skip(self))] + pub fn set_network_class(&mut self, network_class: Option) { + self.changes + .push(RoutingDomainChange::SetNetworkClass { network_class }) + } #[instrument(level = "debug", skip(self))] pub async fn commit(self) { + // No locking if we have nothing to do + if self.changes.is_empty() { + return; + } + let mut changed = false; { let node_id = self.routing_table.node_id(); @@ -81,17 +128,17 @@ impl RoutingDomainEditor { match change { RoutingDomainChange::ClearDialInfoDetails => { debug!("[{:?}] cleared dial info details", self.routing_domain); - detail.clear_dial_info_details(); + detail.common_mut().clear_dial_info_details(); changed = true; } RoutingDomainChange::ClearRelayNode => { debug!("[{:?}] cleared relay node", self.routing_domain); - detail.set_relay_node(None); + detail.common_mut().set_relay_node(None); changed = true; } RoutingDomainChange::SetRelayNode { relay_node } => { debug!("[{:?}] set relay node: {}", self.routing_domain, relay_node); - detail.set_relay_node(Some(relay_node)); + detail.common_mut().set_relay_node(Some(relay_node)); changed = true; } RoutingDomainChange::AddDialInfoDetail { dial_info_detail } => { @@ -99,7 +146,9 @@ impl RoutingDomainEditor { "[{:?}] add dial info detail: {:?}", self.routing_domain, dial_info_detail ); - detail.add_dial_info_detail(dial_info_detail.clone()); + detail + .common_mut() + .add_dial_info_detail(dial_info_detail.clone()); info!( "{:?} Dial Info: {}", @@ -112,8 +161,68 @@ impl RoutingDomainEditor { ); changed = true; } + RoutingDomainChange::SetupNode { + node_id, + node_id_secret, + } => { + debug!( + "[{:?}] setup node: {}", + self.routing_domain, + node_id.encode() + ); + detail.common_mut().setup_node(node_id, node_id_secret); + changed = true; + } + RoutingDomainChange::SetupNetwork { + outbound_protocols, + inbound_protocols, + address_types, + } => { + let old_outbound_protocols = detail.common().outbound_protocols(); + let old_inbound_protocols = detail.common().inbound_protocols(); + let old_address_types = detail.common().address_types(); + + let this_changed = old_outbound_protocols != outbound_protocols + || old_inbound_protocols != inbound_protocols + || old_address_types != address_types; + + debug!( + "[{:?}] setup network: {:?} {:?} {:?}", + self.routing_domain, + outbound_protocols, + inbound_protocols, + address_types + ); + + detail.common_mut().setup_network( + outbound_protocols, + inbound_protocols, + address_types, + ); + if this_changed { + changed = true; + } + } + RoutingDomainChange::SetNetworkClass { network_class } => { + let old_network_class = detail.common().network_class(); + + let this_changed = old_network_class != network_class; + + debug!( + "[{:?}] set network class: {:?}", + self.routing_domain, network_class, + ); + + detail.common_mut().set_network_class(network_class); + if this_changed { + changed = true; + } + } } } + if changed { + detail.common_mut().clear_cache() + } }); if changed { RoutingTable::reset_all_seen_our_node_info(inner, self.routing_domain); diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index ee29fa73..80ed405c 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -1,62 +1,204 @@ use super::*; +#[derive(Debug)] +pub struct RoutingDomainDetailCommon { + routing_domain: RoutingDomain, + node_id: DHTKey, + node_id_secret: DHTKeySecret, + network_class: Option, + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + relay_node: Option, + dial_info_details: Vec, + // caches + cached_peer_info: Mutex>, +} + +impl RoutingDomainDetailCommon { + pub fn new(routing_domain: RoutingDomain) -> Self { + Self { + routing_domain, + node_id: Default::default(), + node_id_secret: Default::default(), + network_class: Default::default(), + outbound_protocols: Default::default(), + inbound_protocols: Default::default(), + address_types: Default::default(), + relay_node: Default::default(), + dial_info_details: Default::default(), + cached_peer_info: Mutex::new(Default::default()), + } + } + + // Set from routing table + pub(super) fn setup_node(&mut self, node_id: DHTKey, node_id_secret: DHTKeySecret) { + self.node_id = node_id; + self.node_id_secret = node_id_secret; + self.clear_cache(); + } + // Set from network manager + pub(super) fn setup_network( + &mut self, + outbound_protocols: ProtocolTypeSet, + inbound_protocols: ProtocolTypeSet, + address_types: AddressTypeSet, + ) { + self.outbound_protocols = outbound_protocols; + self.inbound_protocols = inbound_protocols; + self.address_types = address_types; + } + + pub fn node_id(&self) -> DHTKey { + self.node_id + } + pub fn node_id_secret(&self) -> DHTKeySecret { + self.node_id_secret + } + pub(super) fn set_network_class(&mut self, network_class: Option) { + self.network_class = network_class; + } + pub fn network_class(&self) -> Option { + self.network_class + } + pub fn outbound_protocols(&self) -> ProtocolTypeSet { + self.outbound_protocols + } + pub fn inbound_protocols(&self) -> ProtocolTypeSet { + self.inbound_protocols + } + pub fn address_types(&self) -> AddressTypeSet { + self.address_types + } + pub fn relay_node(&self) -> Option { + self.relay_node.clone() + } + pub(super) fn set_relay_node(&mut self, opt_relay_node: Option) { + self.relay_node = opt_relay_node.map(|nr| { + nr.filtered_clone(NodeRefFilter::new().with_routing_domain(self.routing_domain)) + }) + } + pub fn dial_info_details(&self) -> &Vec { + &self.dial_info_details + } + pub(super) fn clear_dial_info_details(&mut self) { + self.dial_info_details.clear(); + } + pub(super) fn add_dial_info_detail(&mut self, did: DialInfoDetail) { + self.dial_info_details.push(did); + self.dial_info_details.sort(); + } + + pub fn has_valid_own_node_info(&self) -> bool { + self.network_class.unwrap_or(NetworkClass::Invalid) != NetworkClass::Invalid + } + + pub fn with_peer_info(&self, f: F) -> R + where + F: FnOnce(&PeerInfo) -> R, + { + let cpi = self.cached_peer_info.lock(); + if cpi.is_none() { + // Regenerate peer info + let pi = PeerInfo::new( + NodeId::new(self.node_id), + SignedNodeInfo::with_secret( + NodeInfo { + network_class: self.network_class.unwrap_or(NetworkClass::Invalid), + outbound_protocols: self.outbound_protocols, + address_types: self.address_types, + min_version: MIN_VERSION, + max_version: MAX_VERSION, + dial_info_detail_list: self.dial_info_details.clone(), + relay_peer_info: self + .relay_node + .and_then(|rn| rn.make_peer_info(self.routing_domain).map(Box::new)), + }, + NodeId::new(self.node_id), + &self.node_id_secret, + ) + .unwrap(), + ); + // Cache the peer info + *cpi = Some(pi); + } + f(cpi.as_ref().unwrap()) + } + + pub fn inbound_dial_info_filter(&self) -> DialInfoFilter { + DialInfoFilter::all() + .with_protocol_type_set(self.inbound_protocols) + .with_address_type_set(self.address_types) + } + pub fn outbound_dial_info_filter(&self) -> DialInfoFilter { + DialInfoFilter::all() + .with_protocol_type_set(self.outbound_protocols) + .with_address_type_set(self.address_types) + } + + pub(super) fn clear_cache(&self) { + *self.cached_peer_info.lock() = None; + } +} + /// General trait for all routing domains pub trait RoutingDomainDetail { + // Common accessors + fn common(&self) -> &RoutingDomainDetailCommon; + fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon; + + // Per-domain accessors fn can_contain_address(&self, address: Address) -> bool; - fn relay_node(&self) -> Option; - fn set_relay_node(&mut self, opt_relay_node: Option); - fn dial_info_details(&self) -> &Vec; - fn clear_dial_info_details(&mut self); - fn add_dial_info_detail(&mut self, did: DialInfoDetail); } +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// Public Internet routing domain internals -#[derive(Debug, Default)] +#[derive(Debug)] pub struct PublicInternetRoutingDomainDetail { - /// An optional node we relay through for this domain - relay_node: Option, - /// The dial infos on this domain we can be reached by - dial_info_details: Vec, + /// Common implementation for all routing domains + common: RoutingDomainDetailCommon, +} + +impl Default for PublicInternetRoutingDomainDetail { + fn default() -> Self { + Self { + common: RoutingDomainDetailCommon::new(RoutingDomain::PublicInternet), + } + } } impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { + fn common(&self) -> &RoutingDomainDetailCommon { + &self.common + } + fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon { + &mut self.common + } fn can_contain_address(&self, address: Address) -> bool { address.is_global() } - fn relay_node(&self) -> Option { - self.relay_node.clone() - } - fn set_relay_node(&mut self, opt_relay_node: Option) { - self.relay_node = opt_relay_node.map(|nr| { - nr.filtered_clone( - NodeRefFilter::new().with_routing_domain(RoutingDomain::PublicInternet), - ) - }) - } - fn dial_info_details(&self) -> &Vec { - &self.dial_info_details - } - fn clear_dial_info_details(&mut self) { - self.dial_info_details.clear(); - } - fn add_dial_info_detail(&mut self, did: DialInfoDetail) { - self.dial_info_details.push(did); - self.dial_info_details.sort(); - } } /// Local Network routing domain internals -#[derive(Debug, Default)] -pub struct LocalInternetRoutingDomainDetail { - /// An optional node we relay through for this domain - relay_node: Option, - /// The dial infos on this domain we can be reached by - dial_info_details: Vec, +#[derive(Debug)] +pub struct LocalNetworkRoutingDomainDetail { /// The local networks this domain will communicate with local_networks: Vec<(IpAddr, IpAddr)>, + /// Common implementation for all routing domains + common: RoutingDomainDetailCommon, } -impl LocalInternetRoutingDomainDetail { +impl Default for LocalNetworkRoutingDomainDetail { + fn default() -> Self { + Self { + local_networks: Default::default(), + common: RoutingDomainDetailCommon::new(RoutingDomain::LocalNetwork), + } + } +} + +impl LocalNetworkRoutingDomainDetail { pub fn set_local_networks(&mut self, mut local_networks: Vec<(IpAddr, IpAddr)>) -> bool { local_networks.sort(); if local_networks == self.local_networks { @@ -67,7 +209,13 @@ impl LocalInternetRoutingDomainDetail { } } -impl RoutingDomainDetail for LocalInternetRoutingDomainDetail { +impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { + fn common(&self) -> &RoutingDomainDetailCommon { + &self.common + } + fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon { + &mut self.common + } fn can_contain_address(&self, address: Address) -> bool { let ip = address.to_ip_addr(); for localnet in &self.local_networks { @@ -77,22 +225,4 @@ impl RoutingDomainDetail for LocalInternetRoutingDomainDetail { } false } - fn relay_node(&self) -> Option { - self.relay_node.clone() - } - fn set_relay_node(&mut self, opt_relay_node: Option) { - self.relay_node = opt_relay_node.map(|nr| { - nr.filtered_clone(NodeRefFilter::new().with_routing_domain(RoutingDomain::LocalNetwork)) - }); - } - fn dial_info_details(&self) -> &Vec { - &self.dial_info_details - } - fn clear_dial_info_details(&mut self) { - self.dial_info_details.clear(); - } - fn add_dial_info_detail(&mut self, did: DialInfoDetail) { - self.dial_info_details.push(did); - self.dial_info_details.sort(); - } } diff --git a/veilid-core/src/routing_table/tasks.rs b/veilid-core/src/routing_table/tasks.rs index 943f650b..d43ed5a4 100644 --- a/veilid-core/src/routing_table/tasks.rs +++ b/veilid-core/src/routing_table/tasks.rs @@ -22,8 +22,13 @@ impl RoutingTable { ); // Roll all bucket entry transfers - for b in &mut inner.buckets { - b.roll_transfers(last_ts, cur_ts); + let entries: Vec> = inner + .buckets + .iter() + .flat_map(|b| b.entries().map(|(_k, v)| v.clone())) + .collect(); + for v in entries { + v.with_mut(inner, |_rti, e| e.roll_transfers(last_ts, cur_ts)); } Ok(()) } diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index d98adf30..3510f826 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -7,8 +7,8 @@ pub enum Destination { Direct { /// The node to send to target: NodeRef, - /// An optional safety route specification to send from for sender privacy - safety_route_spec: Option>, + /// Require safety route or not + safety: bool, }, /// Send to node for relay purposes Relay { @@ -16,15 +16,17 @@ pub enum Destination { relay: NodeRef, /// The final destination the relay should send to target: DHTKey, - /// An optional safety route specification to send from for sender privacy - safety_route_spec: Option>, + /// Require safety route or not + safety: bool, }, /// Send to private route (privateroute) PrivateRoute { /// A private route to send to private_route: PrivateRoute, - /// An optional safety route specification to send from for sender privacy - safety_route_spec: Option>, + /// Require safety route or not + safety: bool, + /// Prefer reliability or not + reliable: bool, }, } @@ -32,115 +34,47 @@ impl Destination { pub fn direct(target: NodeRef) -> Self { Self::Direct { target, - safety_route_spec: None, + safety: false, } } pub fn relay(relay: NodeRef, target: DHTKey) -> Self { Self::Relay { relay, target, - safety_route_spec: None, + safety: false, } } - pub fn private_route(private_route: PrivateRoute) -> Self { + pub fn private_route(private_route: PrivateRoute, reliable: bool) -> Self { Self::PrivateRoute { private_route, - safety_route_spec: None, + safety: false, + reliable, } } - // pub fn target_id(&self) -> DHTKey { - // match self { - // Destination::Direct { - // target, - // safety_route_spec, - // } => target.node_id(), - // Destination::Relay { - // relay, - // target, - // safety_route_spec, - // } => *target, - // Destination::PrivateRoute { - // private_route, - // safety_route_spec, - // } => {} - // } - // } - // pub fn best_routing_domain(&self) -> RoutingDomain { - // match self { - // Destination::Direct { - // target, - // safety_route_spec, - // } => { - // if safety_route_spec.is_some() { - // RoutingDomain::PublicInternet - // } else { - // target - // .best_routing_domain() - // .unwrap_or(RoutingDomain::PublicInternet) - // } - // } - // Destination::Relay { - // relay, - // target, - // safety_route_spec, - // } => { - // if safety_route_spec.is_some() { - // RoutingDomain::PublicInternet - // } else { - // relay - // .best_routing_domain() - // .unwrap_or(RoutingDomain::PublicInternet) - // } - // } - // Destination::PrivateRoute { - // private_route: _, - // safety_route_spec: _, - // } => RoutingDomain::PublicInternet, - // } - // } - - pub fn safety_route_spec(&self) -> Option> { + pub fn with_safety(self) -> Self { match self { - Destination::Direct { - target: _, - safety_route_spec, - } => safety_route_spec.clone(), - Destination::Relay { - relay: _, - target: _, - safety_route_spec, - } => safety_route_spec.clone(), - Destination::PrivateRoute { - private_route: _, - safety_route_spec, - } => safety_route_spec.clone(), - } - } - pub fn with_safety_route_spec(self, safety_route_spec: Arc) -> Self { - match self { - Destination::Direct { + Destination::Direct { target, safety: _ } => Self::Direct { target, - safety_route_spec: _, - } => Self::Direct { - target, - safety_route_spec: Some(safety_route_spec), + safety: true, }, Destination::Relay { relay, target, - safety_route_spec: _, + safety: _, } => Self::Relay { relay, target, - safety_route_spec: Some(safety_route_spec), + safety: true, }, Destination::PrivateRoute { private_route, - safety_route_spec: _, + safety: _, + reliable, } => Self::PrivateRoute { private_route, - safety_route_spec: Some(safety_route_spec), + safety: true, + reliable, }, } } @@ -149,39 +83,29 @@ impl Destination { impl fmt::Display for Destination { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - Destination::Direct { - target, - safety_route_spec, - } => { - let sr = safety_route_spec - .as_ref() - .map(|_sr| "+SR".to_owned()) - .unwrap_or_default(); + Destination::Direct { target, safety } => { + let sr = if *safety { "+SR" } else { "" }; - write!(f, "{:?}{}", target, sr) + write!(f, "{}{}", target, sr) } Destination::Relay { relay, target, - safety_route_spec, + safety, } => { - let sr = safety_route_spec - .as_ref() - .map(|_sr| "+SR".to_owned()) - .unwrap_or_default(); + let sr = if *safety { "+SR" } else { "" }; - write!(f, "{:?}@{:?}{}", target.encode(), relay, sr) + write!(f, "{}@{}{}", target.encode(), relay, sr) } Destination::PrivateRoute { private_route, - safety_route_spec, + safety, + reliable, } => { - let sr = safety_route_spec - .as_ref() - .map(|_sr| "+SR".to_owned()) - .unwrap_or_default(); + let sr = if *safety { "+SR" } else { "" }; + let rl = if *reliable { "+RL" } else { "" }; - write!(f, "{}{}", private_route, sr) + write!(f, "{}{}{}", private_route, sr, rl) } } } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 9de635db..2867d719 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -544,7 +544,8 @@ impl RPCProcessor { } // Don't do this if our own signed node info isn't valid yet let routing_table = self.routing_table(); - if !routing_table.has_valid_own_node_info(RoutingDomain::PublicInternet) { + let network_manager = self.network_manager(); + if !RoutingTable::has_valid_own_node_info(network_manager, RoutingDomain::PublicInternet) { return None; } diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index 60eea998..2448b9b8 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -62,16 +62,34 @@ impl RPCProcessor { // add node information for the requesting node to our routing table let routing_table = self.routing_table(); - let rt2 = routing_table.clone(); - let rt3 = routing_table.clone(); + let network_manager = self.network_manager(); + let has_valid_own_node_info = + routing_table.has_valid_own_node_info(RoutingDomain::PublicInternet); + let own_peer_info = routing_table.get_own_peer_info(RoutingDomain::PublicInternet); // find N nodes closest to the target node in our routing table let closest_nodes = routing_table.find_closest_nodes( find_node_q.node_id, // filter - move |_k, v| rt2.filter_has_valid_signed_node_info(RoutingDomain::PublicInternet, v), + |rti, _k, v| { + RoutingTable::filter_has_valid_signed_node_info_inner( + rti, + RoutingDomain::PublicInternet, + has_valid_own_node_info, + v, + ) + }, // transform - move |k, v| rt3.transform_to_peer_info(RoutingDomain::PublicInternet, k, v), + |rti, k, v| { + let own_peer_info = own_peer_info.clone(); + RoutingTable::transform_to_peer_info_inner( + rti, + RoutingDomain::PublicInternet, + own_peer_info, + k, + v, + ) + }, ); // Make status answer diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index e1cc7788..9c73baf2 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -87,7 +87,7 @@ impl RPCProcessor { routing_domain, dial_info.clone(), ); - let will_validate_dial_info_filter = |e: &BucketEntryInner| { + let will_validate_dial_info_filter = |_rti, e: &BucketEntryInner| { if let Some(status) = &e.node_status(routing_domain) { status.will_validate_dial_info() } else { diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index f330ef44..1fea8f86 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -2002,25 +2002,6 @@ impl VeilidAPI { .map_err(|e| VeilidAPIError::internal(e)) } - //////////////////////////////////////////////////////////////// - // Safety / Private Route Handling - - #[instrument(level = "debug", err, skip(self))] - pub async fn new_safety_route_spec( - &self, - _hops: u8, - ) -> Result { - panic!("unimplemented"); - } - - #[instrument(level = "debug", err, skip(self))] - pub async fn new_private_route_spec( - &self, - _hops: u8, - ) -> Result { - panic!("unimplemented"); - } - //////////////////////////////////////////////////////////////// // Routing Context diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index 046785a9..6ddc3ef0 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -9,35 +9,17 @@ pub struct RouteHopSpec { } #[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct PrivateRouteSpec { +pub struct RouteSpec { // pub public_key: DHTKey, pub secret_key: DHTKeySecret, pub hops: Vec, } -impl PrivateRouteSpec { +impl RouteSpec { pub fn new() -> Self { let (pk, sk) = generate_secret(); - PrivateRouteSpec { - public_key: pk, - secret_key: sk, - hops: Vec::new(), - } - } -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct SafetyRouteSpec { - pub public_key: DHTKey, - pub secret_key: DHTKeySecret, - pub hops: Vec, -} - -impl SafetyRouteSpec { - pub fn new() -> Self { - let (pk, sk) = generate_secret(); - SafetyRouteSpec { + RouteSpec { public_key: pk, secret_key: sk, hops: Vec::new(), diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index b095fd3a..215a8b2c 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -10,10 +10,8 @@ pub enum Target { pub struct RoutingContextInner {} pub struct RoutingContextUnlockedInner { - /// Safety route specified here is for _this_ node's anonymity as a sender, used via the 'route' operation - safety_route_spec: Option>, - /// Private route specified here is for _this_ node's anonymity as a receiver, passed out via the 'respond_to' field for replies - private_route_spec: Option>, + /// Enforce use of private routing + privacy: bool, /// Choose reliable protocols over unreliable/faster protocols when available reliable: bool, } @@ -43,24 +41,18 @@ impl RoutingContext { api, inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - safety_route_spec: None, - private_route_spec: None, + privacy: false, reliable: false, }), } } - pub fn with_privacy( - self, - safety_route_spec: SafetyRouteSpec, - private_route_spec: PrivateRouteSpec, - ) -> Self { + pub fn with_privacy(self) -> Self { Self { api: self.api.clone(), inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - safety_route_spec: Some(Arc::new(safety_route_spec)), - private_route_spec: Some(Arc::new(private_route_spec)), + privacy: true, reliable: self.unlocked_inner.reliable, }), } @@ -71,8 +63,7 @@ impl RoutingContext { api: self.api.clone(), inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - safety_route_spec: self.unlocked_inner.safety_route_spec.clone(), - private_route_spec: self.unlocked_inner.private_route_spec.clone(), + privacy: self.unlocked_inner.privacy, reliable: true, }), } @@ -102,12 +93,13 @@ impl RoutingContext { } Ok(rpc_processor::Destination::Direct { target: nr, - safety_route_spec: self.unlocked_inner.safety_route_spec.clone(), + safety: self.unlocked_inner.privacy, }) } Target::PrivateRoute(pr) => Ok(rpc_processor::Destination::PrivateRoute { private_route: pr, - safety_route_spec: self.unlocked_inner.safety_route_spec.clone(), + safety: self.unlocked_inner.privacy, + reliable: self.unlocked_inner.reliable, }), } } diff --git a/veilid-core/src/veilid_api/serialize_helpers.rs b/veilid-core/src/veilid_api/serialize_helpers.rs index 58b51909..3f843e4b 100644 --- a/veilid-core/src/veilid_api/serialize_helpers.rs +++ b/veilid-core/src/veilid_api/serialize_helpers.rs @@ -113,3 +113,18 @@ pub mod opt_json_as_string { } } } + +pub mod arc_serialize { + use alloc::sync::Arc; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(v: &Arc, s: S) -> Result { + T::serialize(v.as_ref(), s) + } + + pub fn deserialize<'de, T: Deserialize<'de>, D: Deserializer<'de>>( + d: D, + ) -> Result, D::Error> { + Ok(Arc::new(T::deserialize(d)?)) + } +} From f7f166741bc8877a33073235cf8210ebd93431ff Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 9 Oct 2022 22:07:15 -0400 Subject: [PATCH 11/67] private route work --- veilid-core/proto/veilid.capnp | 7 +- veilid-core/src/attachment_manager.rs | 97 ++++--- veilid-core/src/core_context.rs | 2 +- veilid-core/src/network_manager/mod.rs | 109 +++++--- .../src/routing_table/route_spec_store.rs | 245 +++++++++++++----- veilid-core/src/veilid_api/mod.rs | 8 + veilid-core/src/veilid_api/privacy.rs | 55 ++-- 7 files changed, 365 insertions(+), 158 deletions(-) diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 558686b2..b972b191 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -132,8 +132,11 @@ struct RouteHopData { } struct RouteHop { - dialInfo @0 :NodeDialInfo; # dial info for this hop - nextHop @1 :RouteHopData; # Optional: next hop in encrypted blob + node :union { + nodeId @0 :NodeID; # node id only for established routes + peerInfo @1 :PeerInfo; # full peer info for this hop to establish the route + } + nextHop @2 :RouteHopData; # Optional: next hop in encrypted blob # Null means no next hop, at destination (only used in private route, safety routes must enclose a stub private route) } diff --git a/veilid-core/src/attachment_manager.rs b/veilid-core/src/attachment_manager.rs index 08176e37..3d283140 100644 --- a/veilid-core/src/attachment_manager.rs +++ b/veilid-core/src/attachment_manager.rs @@ -102,48 +102,77 @@ impl TryFrom for AttachmentState { } pub struct AttachmentManagerInner { - config: VeilidConfig, attachment_machine: CallbackStateMachine, - network_manager: NetworkManager, maintain_peers: bool, attach_timestamp: Option, update_callback: Option, attachment_maintainer_jh: Option>, } +pub struct AttachmentManagerUnlockedInner { + config: VeilidConfig, + network_manager: NetworkManager, +} + #[derive(Clone)] pub struct AttachmentManager { inner: Arc>, + unlocked_inner: Arc, } impl AttachmentManager { - fn new_inner( + fn new_unlocked_inner( config: VeilidConfig, + protected_store: ProtectedStore, table_store: TableStore, + block_store: BlockStore, crypto: Crypto, - ) -> AttachmentManagerInner { - AttachmentManagerInner { + ) -> AttachmentManagerUnlockedInner { + AttachmentManagerUnlockedInner { config: config.clone(), + network_manager: NetworkManager::new( + config, + protected_store, + table_store, + block_store, + crypto, + ), + } + } + fn new_inner() -> AttachmentManagerInner { + AttachmentManagerInner { attachment_machine: CallbackStateMachine::new(), - network_manager: NetworkManager::new(config, table_store, crypto), maintain_peers: false, attach_timestamp: None, update_callback: None, attachment_maintainer_jh: None, } } - pub fn new(config: VeilidConfig, table_store: TableStore, crypto: Crypto) -> Self { + pub fn new( + config: VeilidConfig, + protected_store: ProtectedStore, + table_store: TableStore, + block_store: BlockStore, + crypto: Crypto, + ) -> Self { Self { - inner: Arc::new(Mutex::new(Self::new_inner(config, table_store, crypto))), + inner: Arc::new(Mutex::new(Self::new_inner())), + unlocked_inner: Arc::new(Self::new_unlocked_inner( + config, + protected_store, + table_store, + block_store, + crypto, + )), } } pub fn config(&self) -> VeilidConfig { - self.inner.lock().config.clone() + self.unlocked_inner.config.clone() } pub fn network_manager(&self) -> NetworkManager { - self.inner.lock().network_manager.clone() + self.unlocked_inner.network_manager.clone() } pub fn is_attached(&self) -> bool { @@ -202,9 +231,10 @@ impl AttachmentManager { AttachmentManager::translate_attachment_state(&inner.attachment_machine.state()); // get reliable peer count from routing table - let routing_table = inner.network_manager.routing_table(); + let routing_table = self.network_manager().routing_table(); let health = routing_table.get_routing_table_health(); - let routing_table_config = &inner.config.get().network.routing_table; + let config = self.config(); + let routing_table_config = &config.get().network.routing_table; let new_peer_state_input = AttachmentManager::translate_routing_table_health(health, routing_table_config); @@ -223,11 +253,8 @@ impl AttachmentManager { #[instrument(level = "debug", skip(self))] async fn attachment_maintainer(self) { debug!("attachment starting"); - let netman = { - let mut inner = self.inner.lock(); - inner.attach_timestamp = Some(intf::get_timestamp()); - inner.network_manager.clone() - }; + self.inner.lock().attach_timestamp = Some(intf::get_timestamp()); + let netman = self.network_manager(); let mut restart; loop { @@ -286,7 +313,7 @@ impl AttachmentManager { #[instrument(level = "debug", skip_all, err)] pub async fn init(&self, update_callback: UpdateCallback) -> EyreResult<()> { trace!("init"); - let network_manager = { + { let mut inner = self.inner.lock(); inner.update_callback = Some(update_callback.clone()); let update_callback2 = update_callback.clone(); @@ -297,10 +324,9 @@ impl AttachmentManager { })) }, )); - inner.network_manager.clone() }; - network_manager.init(update_callback).await?; + self.network_manager().init(update_callback).await?; Ok(()) } @@ -309,30 +335,33 @@ impl AttachmentManager { pub async fn terminate(&self) { // Ensure we detached self.detach().await; - let network_manager = { - let inner = self.inner.lock(); - inner.network_manager.clone() - }; - network_manager.terminate().await; - let mut inner = self.inner.lock(); - inner.update_callback = None; + self.network_manager().terminate().await; + self.inner.lock().update_callback = None; } #[instrument(level = "trace", skip(self))] fn attach(&self) { // Create long-running connection maintenance routine - let this = self.clone(); - self.inner.lock().maintain_peers = true; - self.inner.lock().attachment_maintainer_jh = - Some(intf::spawn(this.attachment_maintainer())); + let inner = self.inner.lock(); + if inner.attachment_maintainer_jh.is_some() { + return; + } + inner.maintain_peers = true; + inner.attachment_maintainer_jh = Some(intf::spawn(self.clone().attachment_maintainer())); } #[instrument(level = "trace", skip(self))] async fn detach(&self) { - let attachment_maintainer_jh = self.inner.lock().attachment_maintainer_jh.take(); + let attachment_maintainer_jh = { + let mut inner = self.inner.lock(); + let attachment_maintainer_jh = inner.attachment_maintainer_jh.take(); + if attachment_maintainer_jh.is_some() { + // Terminate long-running connection maintenance routine + inner.maintain_peers = false; + } + attachment_maintainer_jh + }; if let Some(jh) = attachment_maintainer_jh { - // Terminate long-running connection maintenance routine - self.inner.lock().maintain_peers = false; jh.await; } } diff --git a/veilid-core/src/core_context.rs b/veilid-core/src/core_context.rs index 9897f68e..2640f455 100644 --- a/veilid-core/src/core_context.rs +++ b/veilid-core/src/core_context.rs @@ -103,7 +103,7 @@ impl ServicesContext { // Set up attachment manager trace!("init attachment manager"); let update_callback = self.update_callback.clone(); - let attachment_manager = AttachmentManager::new(self.config.clone(), table_store, crypto); + let attachment_manager = AttachmentManager::new(self.config.clone(), protected_store, table_store, block_store, crypto); if let Err(e) = attachment_manager.init(update_callback).await { self.shutdown().await; return Err(e); diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 455d35fd..d22716a8 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -152,6 +152,12 @@ struct NetworkManagerInner { } struct NetworkManagerUnlockedInner { + // Handles + config: VeilidConfig, + protected_store: ProtectedStore, + table_store: TableStore, + block_store: BlockStore, + crypto: Crypto, // Accessors routing_table: RwLock>, components: RwLock>, @@ -169,9 +175,6 @@ struct NetworkManagerUnlockedInner { #[derive(Clone)] pub struct NetworkManager { - config: VeilidConfig, - table_store: TableStore, - crypto: Crypto, inner: Arc>, unlocked_inner: Arc, } @@ -185,9 +188,20 @@ impl NetworkManager { public_address_inconsistencies_table: BTreeMap::new(), } } - fn new_unlocked_inner(config: VeilidConfig) -> NetworkManagerUnlockedInner { + fn new_unlocked_inner( + config: VeilidConfig, + protected_store: ProtectedStore, + table_store: TableStore, + block_store: BlockStore, + crypto: Crypto, + ) -> NetworkManagerUnlockedInner { let c = config.get(); NetworkManagerUnlockedInner { + config, + protected_store, + table_store, + block_store, + crypto, routing_table: RwLock::new(None), components: RwLock::new(None), update_callback: RwLock::new(None), @@ -202,13 +216,22 @@ impl NetworkManager { } } - pub fn new(config: VeilidConfig, table_store: TableStore, crypto: Crypto) -> Self { + pub fn new( + config: VeilidConfig, + protected_store: ProtectedStore, + table_store: TableStore, + block_store: BlockStore, + crypto: Crypto, + ) -> Self { let this = Self { - config: config.clone(), - table_store, - crypto, inner: Arc::new(Mutex::new(Self::new_inner())), - unlocked_inner: Arc::new(Self::new_unlocked_inner(config)), + unlocked_inner: Arc::new(Self::new_unlocked_inner( + config, + protected_store, + table_store, + block_store, + crypto, + )), }; // Set rolling transfers tick task { @@ -323,13 +346,25 @@ impl NetworkManager { this } pub fn config(&self) -> VeilidConfig { - self.config.clone() + self.unlocked_inner.config.clone() + } + pub fn with_config(&self, f: F) -> R + where + F: FnOnce(&VeilidConfigInner) -> R, + { + f(&*self.unlocked_inner.config.get()) + } + pub fn protected_store(&self) -> ProtectedStore { + self.unlocked_inner.protected_store.clone() } pub fn table_store(&self) -> TableStore { - self.table_store.clone() + self.unlocked_inner.table_store.clone() + } + pub fn block_store(&self) -> BlockStore { + self.unlocked_inner.block_store.clone() } pub fn crypto(&self) -> Crypto { - self.crypto.clone() + self.unlocked_inner.crypto.clone() } pub fn routing_table(&self) -> RoutingTable { self.unlocked_inner @@ -540,7 +575,7 @@ impl NetworkManager { } pub fn purge_client_whitelist(&self) { - let timeout_ms = self.config.get().network.client_whitelist_timeout_ms; + let timeout_ms = self.with_config(|c| c.network.client_whitelist_timeout_ms); let mut inner = self.inner.lock(); let cutoff_timestamp = intf::get_timestamp() - ((timeout_ms as u64) * 1000u64); // Remove clients from the whitelist that haven't been since since our whitelist timeout @@ -576,10 +611,7 @@ impl NetworkManager { RoutingDomain::PublicInternet.into(), BucketEntryState::Unreliable, ); - let min_peer_count = { - let c = self.config.get(); - c.network.dht.min_peer_count as usize - }; + let min_peer_count = self.with_config(|c| c.network.dht.min_peer_count as usize); // If none, then add the bootstrap nodes to it if live_public_internet_entry_count == 0 { @@ -857,7 +889,7 @@ impl NetworkManager { // Encode envelope let envelope = Envelope::new(version, ts, nonce, node_id, dest_node_id); envelope - .to_encrypted_data(self.crypto.clone(), body.as_ref(), &node_id_secret) + .to_encrypted_data(self.crypto(), body.as_ref(), &node_id_secret) .wrap_err("envelope failed to encode") } @@ -1095,6 +1127,11 @@ impl NetworkManager { } } + /// Get the contact method required for node A to reach node B + pub fn get_node_contact_method(node_a: &NodeInfo, node_b: &NodeInfo) -> ContactMethod { + unimplemented!(); + } + // Send a reverse connection signal and wait for the return receipt over it // Then send the data across the new connection // Only usable for PublicInternet routing domain @@ -1106,8 +1143,13 @@ impl NetworkManager { data: Vec, ) -> EyreResult> { // Build a return receipt for the signal - let receipt_timeout = - ms_to_us(self.config.get().network.reverse_connection_receipt_time_ms); + let receipt_timeout = ms_to_us( + self.unlocked_inner + .config + .get() + .network + .reverse_connection_receipt_time_ms, + ); let (receipt, eventual_value) = self.generate_single_shot_receipt(receipt_timeout, [])?; // Get our peer info @@ -1188,7 +1230,13 @@ impl NetworkManager { .unwrap_or_default()); // Build a return receipt for the signal - let receipt_timeout = ms_to_us(self.config.get().network.hole_punch_receipt_time_ms); + let receipt_timeout = ms_to_us( + self.unlocked_inner + .config + .get() + .network + .hole_punch_receipt_time_ms, + ); let (receipt, eventual_value) = self.generate_single_shot_receipt(receipt_timeout, [])?; // Get our peer info let peer_info = self @@ -1404,10 +1452,7 @@ impl NetworkManager { // Direct bootstrap request #[instrument(level = "trace", err, skip(self))] pub async fn boot_request(&self, dial_info: DialInfo) -> EyreResult> { - let timeout_ms = { - let c = self.config.get(); - c.network.rpc.timeout_ms - }; + let timeout_ms = self.with_config(|c| c.network.rpc.timeout_ms); // Send boot magic to requested peer address let data = BOOT_MAGIC.to_vec(); let out_data: Vec = network_result_value_or_log!(debug self @@ -1502,13 +1547,12 @@ impl NetworkManager { }; // Get timestamp range - let (tsbehind, tsahead) = { - let c = self.config.get(); + let (tsbehind, tsahead) = self.with_config(|c| { ( c.network.rpc.max_timestamp_behind_ms.map(ms_to_us), c.network.rpc.max_timestamp_ahead_ms.map(ms_to_us), ) - }; + }); // Validate timestamp isn't too old let ts = intf::get_timestamp(); @@ -1742,11 +1786,14 @@ impl NetworkManager { } let routing_table = self.routing_table(); - let c = self.config.get(); - let detect_address_changes = c.network.detect_address_changes; + let (detect_address_changes, ip6_prefix_size) = self.with_config(|c| { + ( + c.network.detect_address_changes, + c.network.max_connections_per_ip6_prefix_size as usize, + ) + }); // Get the ip(block) this report is coming from - let ip6_prefix_size = c.network.max_connections_per_ip6_prefix_size as usize; let ipblock = ip_to_ipblock( ip6_prefix_size, connection_descriptor.remote_address().to_ip_addr(), diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 480f6a7f..215b9056 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -4,9 +4,14 @@ use serde::*; #[derive(Clone, Debug, Serialize, Deserialize)] struct RouteSpecDetail { - /// The actual route spec - #[serde(with = "arc_serialize")] - route_spec: Arc, + /// Secret key + #[serde(skip)] + secret_key: DHTKeySecret, + /// Route hops + hops: Vec, + /// Route noderefs + #[serde(skip)] + hop_node_refs: Vec, /// Transfers up and down transfer_stats_down_up: TransferStatsDownUp, /// Latency stats @@ -18,10 +23,15 @@ struct RouteSpecDetail { #[serde(skip)] transfer_stats_accounting: TransferStatsAccounting, /// Published private route, do not reuse for ephemeral routes + /// Not serialized because all routes should be re-published when restarting #[serde(skip)] published: bool, /// Timestamp of when the route was created - timestamp: u64, + created_ts: u64, + /// Timestamp of when the route was last checked for validity + last_checked_ts: Option, + /// Directions this route is guaranteed to work in + directions: DirectionSet, } /// The core representation of the RouteSpecStore that can be serialized @@ -34,10 +44,6 @@ pub struct RouteSpecStoreContent { /// Ephemeral data used to help the RouteSpecStore operate efficiently #[derive(Debug, Default)] pub struct RouteSpecStoreCache { - /// The fastest routes by latency - fastest_routes: Vec, - /// The most reliable routes by node lifetime longevity - reliable_routes: Vec, /// How many times nodes have been used used_nodes: HashMap, /// How many times nodes have been used at the terminal point of a route @@ -62,14 +68,76 @@ fn route_spec_to_hop_cache(spec: Arc) -> Vec { cache } -fn node_sublist_to_hop_cache( - nodes: &[(DHTKey, Arc)], - start: usize, - len: usize, -) -> Vec { - let mut cache: Vec = Vec::with_capacity(len * DHT_KEY_LENGTH); - for node in &nodes[start..start + len] { - cache.extend_from_slice(&node.0.bytes) +/// number of route permutations is the number of unique orderings +/// for a set of nodes, given that the first node is fixed +fn get_route_permutation_count(hop_count: usize) -> usize { + if hop_count == 0 { + unreachable!(); + } + // a single node or two nodes is always fixed + if hop_count == 1 || hop_count == 2 { + return 1; + } + // more than two nodes has factorial permutation + // hop_count = 3 -> 2! -> 2 + // hop_count = 4 -> 3! -> 6 + (3..hop_count).into_iter().fold(2usize, |acc, x| acc * x) +} + +/// get the route permutation at particular 'perm' index, starting at the 'start' index +/// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum +/// number of permutations is given by get_route_permutation_count() +fn with_route_permutations(hop_count: usize, start: usize, f: F) -> bool +where + F: FnMut(&[usize]) -> bool, +{ + if hop_count == 0 { + unreachable!(); + } + // initial permutation + let mut permutation: Vec = Vec::with_capacity(hop_count); + for n in 0..hop_count { + permutation[n] = start + n; + } + // if we have one hop or two, then there's only one permutation + if hop_count == 1 || hop_count == 2 { + return f(&permutation); + } + + // heaps algorithm + fn heaps_permutation(permutation: &mut [usize], size: usize, f: F) -> bool + where + F: FnMut(&[usize]) -> bool, + { + if size == 1 { + if f(&permutation) { + return true; + } + return false; + } + + for i in 0..size { + if heaps_permutation(permutation, size - 1, f) { + return true; + } + if size % 2 == 1 { + permutation.swap(1, size); + } else { + permutation.swap(1 + i, size); + } + } + false + } + + // recurse + heaps_permutation(&mut permutation, hop_count - 1, f) +} + +/// get the hop cache key for a particular route permutation +fn route_permutation_to_hop_cache(nodes: &[(DHTKey, NodeInfo)], perm: &[usize]) -> Vec { + let mut cache: Vec = Vec::with_capacity(perm.len() * DHT_KEY_LENGTH); + for n in perm { + cache.extend_from_slice(&nodes[*n].0.bytes) } cache } @@ -84,30 +152,33 @@ impl RouteSpecStore { } } - pub fn from_cbor( - routing_table: RoutingTable, - cbor: &[u8], - ) -> Result { + pub fn load(routing_table: RoutingTable) -> Result { + // Get cbor blob from table store let content: RouteSpecStoreContent = serde_cbor::from_slice(cbor) .map_err(|e| VeilidAPIError::parse_error("invalid route spec store content", e))?; let rss = RouteSpecStore { content, cache: Default::default(), }; - rss.rebuild_cache(); + rss.rebuild_cache(routing_table); Ok(rss) } - pub fn to_cbor(&self) -> Vec { - serde_cbor::to_vec(&self.content).unwrap() + pub fn save(&self, routing_table: RoutingTable) -> Result<(), VeilidAPIError> { + // Save all the fields we care about to the cbor blob in table storage + let cbor = serde_cbor::to_vec(&self.content).unwrap(); + let table_store = routing_table.network_manager().table_store(); + table_store.open("") } - fn rebuild_cache(&mut self) { + fn rebuild_cache(&mut self, routing_table: RoutingTable) { // + // xxx also load secrets from pstore + let pstore = routing_table.network_manager().protected_store(); } - fn detail_mut(&mut self, spec: Arc) -> &mut RouteSpecDetail { - self.content.details.get_mut(&spec.public_key).unwrap() + fn detail_mut(&mut self, public_key: DHTKey) -> &mut RouteSpecDetail { + self.content.details.get_mut(&public_key).unwrap() } /// Create a new route @@ -119,7 +190,8 @@ impl RouteSpecStore { routing_table: RoutingTable, reliable: bool, hop_count: usize, - ) -> Option> { + directions: DirectionSet, + ) -> Option { use core::cmp::Ordering; let max_route_hop_count = { @@ -257,51 +329,101 @@ impl RouteSpecStore { } // Now go through nodes and try to build a route we haven't seen yet - let mut route_nodes = None; + let mut route_nodes: Vec = Vec::with_capacity(hop_count); for start in 0..(nodes.len() - hop_count) { - // Get the route cache key - let key = node_sublist_to_hop_cache(&nodes, start, hop_count); + // Try the permutations available starting with 'start' + let done = with_route_permutations(hop_count, start, |permutation: &[usize]| { + // Get the route cache key + let key = route_permutation_to_hop_cache(&nodes, permutation); - // try each route until we find a unique one - if !self.cache.hop_cache.contains(&key) { - route_nodes = Some(&nodes[start..start + hop_count]); + // Skip routes we have already seen + if self.cache.hop_cache.contains(&key) { + return false; + } + + // Ensure this route is viable by checking that each node can contact the next one + if directions.contains(Direction::Outbound) { + let our_node_info = + routing_table.get_own_node_info(RoutingDomain::PublicInternet); + let mut previous_node_info = &our_node_info; + let mut reachable = true; + for n in permutation { + let current_node_info = &nodes.get(*n).as_ref().unwrap().1; + let cm = NetworkManager::get_node_contact_method( + previous_node_info, + current_node_info, + ); + if matches!(cm, ContactMethod::Unreachable) { + reachable = false; + break; + } + previous_node_info = current_node_info; + } + if !reachable { + return false; + } + } + if directions.contains(Direction::Inbound) { + let our_node_info = + routing_table.get_own_node_info(RoutingDomain::PublicInternet); + let mut next_node_info = &our_node_info; + let mut reachable = true; + for n in permutation.iter().rev() { + let current_node_info = &nodes.get(*n).as_ref().unwrap().1; + let cm = NetworkManager::get_node_contact_method( + current_node_info, + next_node_info, + ); + if matches!(cm, ContactMethod::Unreachable) { + reachable = false; + break; + } + next_node_info = current_node_info; + } + if !reachable { + return false; + } + } + // Keep this route + route_nodes = permutation.to_vec(); + true + }); + if done { break; } } - if route_nodes.is_none() { + if route_nodes.is_empty() { return None; } - let route_node = route_nodes.unwrap(); // Got a unique route, lets build the detail, register it, and return it - let hops: Vec = route_node - .into_iter() - .map(|v| RouteHopSpec { - dial_info: NodeDialInfo { - node_id: NodeId::new(v.0), - dial_info: xxx, - }, - }) + let hops = route_nodes.iter().map(|v| nodes[*v].0).collect(); + let hop_node_refs = route_nodes + .iter() + .map(|v| routing_table.lookup_node_ref(nodes[*v].0).unwrap()) .collect(); let (public_key, secret_key) = generate_secret(); - let route_spec = Arc::new(RouteSpec { - public_key, - secret_key, - hops, - }); let rsd = RouteSpecDetail { - route_spec, + secret_key, + hops, + hop_node_refs, transfer_stats_down_up: Default::default(), latency_stats: Default::default(), latency_stats_accounting: Default::default(), transfer_stats_accounting: Default::default(), published: false, - timestamp: cur_ts, + created_ts: cur_ts, + last_checked_ts: None, + directions, }; - None + self.content.details.insert(public_key, rsd); + + // xxx insert into cache too + + Some(public_key) } pub fn release_route(&mut self, spec: Arc) {} @@ -311,15 +433,22 @@ impl RouteSpecStore { /// Mark route as published /// When first deserialized, routes must be re-published in order to ensure they remain /// in the RouteSpecStore. - pub fn publish_route(&mut self, spec: Arc) { - //compile private route here? + pub fn mark_route_published(&mut self, spec: Arc) { + self.detail_mut(spec).published = true; } - pub fn record_latency( - &mut self, - spec: Arc, - latency: u64, - ) -> veilid_api::LatencyStats { + /// Mark route as checked + pub fn touch_route_checked(&mut self, spec: Arc, cur_ts: u64) { + self.detail_mut(spec).last_checked_ts = cur_ts; + } + + pub fn record_latency(&mut self, spec: Arc, latency: u64) { + let lsa = self.detail_mut(spec).latency_stats_accounting; + self.detail_mut(spec).latency_stats = lsa.record_latency(latency); + } + + pub fn latency_stats(&self, spec: Arc) -> LatencyStats { + self.detail_mut(spec).latency_stats.clone() } pub fn add_down(&mut self, spec: Arc, bytes: u64) { diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 1fea8f86..b224e639 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -645,6 +645,14 @@ impl NodeInfo { } } +#[allow(clippy::derive_hash_xor_eq)] +#[derive(Debug, PartialOrd, Ord, Hash, Serialize, Deserialize, EnumSetType)] +pub enum Direction { + Inbound, + Outbound, +} +pub type DirectionSet = EnumSet; + #[allow(clippy::derive_hash_xor_eq)] #[derive(Debug, PartialOrd, Ord, Hash, Serialize, Deserialize, EnumSetType)] // Keep member order appropriate for sorting < preference diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index 6ddc3ef0..ea37ecfd 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -1,32 +1,5 @@ use super::*; -///////////////////////////////////////////////////////////////////////////////////////////////////// -// Privacy Specs - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct RouteHopSpec { - pub dial_info: NodeDialInfo, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct RouteSpec { - // - pub public_key: DHTKey, - pub secret_key: DHTKeySecret, - pub hops: Vec, -} - -impl RouteSpec { - pub fn new() -> Self { - let (pk, sk) = generate_secret(); - RouteSpec { - public_key: pk, - secret_key: sk, - hops: Vec::new(), - } - } -} - //////////////////////////////////////////////////////////////////////////////////////////////////// // Compiled Privacy Objects @@ -36,9 +9,27 @@ pub struct RouteHopData { pub blob: Vec, } +#[derive(Clone, Debug)] +pub enum RouteNode { + NodeId(DHTKey), + PeerInfo(PeerInfo), +} +impl fmt::Display for RouteNode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}", + match self { + RouteNode::NodeId(x) => x.encode(), + RouteNode::PeerInfo(pi) => pi.node_id.key.encode(), + } + ) + } +} + #[derive(Clone, Debug)] pub struct RouteHop { - pub dial_info: NodeDialInfo, + pub node: RouteNode, pub next_hop: Option, } @@ -46,7 +37,7 @@ pub struct RouteHop { pub struct PrivateRoute { pub public_key: DHTKey, pub hop_count: u8, - pub hops: Option, + pub first_hop: Option, } impl PrivateRoute { @@ -54,7 +45,7 @@ impl PrivateRoute { Self { public_key, hop_count: 0, - hops: None, + first_hop: None, } } } @@ -66,8 +57,8 @@ impl fmt::Display for PrivateRoute { "PR({:?}+{}{})", self.public_key, self.hop_count, - if let Some(hops) = &self.hops { - format!("->{}", hops.dial_info) + if let Some(first_hop) = &self.first_hop { + format!("->{}", first_hop.node) } else { "".to_owned() } From 9c59507ea000c8a1eed39f4fc92477c3f7b7fa22 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 11 Oct 2022 19:49:29 -0400 Subject: [PATCH 12/67] checkpoint --- .../src/routing_table/route_spec_store.rs | 116 ++++++++++++++---- 1 file changed, 95 insertions(+), 21 deletions(-) diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 215b9056..46368ef3 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -32,6 +32,8 @@ struct RouteSpecDetail { last_checked_ts: Option, /// Directions this route is guaranteed to work in directions: DirectionSet, + /// Reliability + reliable: bool, } /// The core representation of the RouteSpecStore that can be serialized @@ -60,10 +62,19 @@ pub struct RouteSpecStore { cache: RouteSpecStoreCache, } -fn route_spec_to_hop_cache(spec: Arc) -> Vec { - let mut cache: Vec = Vec::with_capacity(spec.hops.len() * DHT_KEY_LENGTH); - for hop in spec.hops { - cache.extend_from_slice(&hop.dial_info.node_id.key.bytes); +fn route_hops_to_hop_cache(hops: &[DHTKey]) -> Vec { + let mut cache: Vec = Vec::with_capacity(hops.len() * DHT_KEY_LENGTH); + for hop in hops { + cache.extend_from_slice(&hop.bytes); + } + cache +} + +/// get the hop cache key for a particular route permutation +fn route_permutation_to_hop_cache(nodes: &[(DHTKey, NodeInfo)], perm: &[usize]) -> Vec { + let mut cache: Vec = Vec::with_capacity(perm.len() * DHT_KEY_LENGTH); + for n in perm { + cache.extend_from_slice(&nodes[*n].0.bytes) } cache } @@ -104,7 +115,7 @@ where return f(&permutation); } - // heaps algorithm + // heaps algorithm, but skipping the first element fn heaps_permutation(permutation: &mut [usize], size: usize, f: F) -> bool where F: FnMut(&[usize]) -> bool, @@ -133,15 +144,6 @@ where heaps_permutation(&mut permutation, hop_count - 1, f) } -/// get the hop cache key for a particular route permutation -fn route_permutation_to_hop_cache(nodes: &[(DHTKey, NodeInfo)], perm: &[usize]) -> Vec { - let mut cache: Vec = Vec::with_capacity(perm.len() * DHT_KEY_LENGTH); - for n in perm { - cache.extend_from_slice(&nodes[*n].0.bytes) - } - cache -} - impl RouteSpecStore { pub fn new() -> Self { Self { @@ -329,15 +331,16 @@ impl RouteSpecStore { } // Now go through nodes and try to build a route we haven't seen yet - let mut route_nodes: Vec = Vec::with_capacity(hop_count); + let mut route_nodes: Vec = Vec::new(); + let mut cache_key: Vec = Vec::new(); for start in 0..(nodes.len() - hop_count) { // Try the permutations available starting with 'start' let done = with_route_permutations(hop_count, start, |permutation: &[usize]| { // Get the route cache key - let key = route_permutation_to_hop_cache(&nodes, permutation); + cache_key = route_permutation_to_hop_cache(&nodes, permutation); // Skip routes we have already seen - if self.cache.hop_cache.contains(&key) { + if self.cache.hop_cache.contains(&cache_key) { return false; } @@ -417,18 +420,89 @@ impl RouteSpecStore { created_ts: cur_ts, last_checked_ts: None, directions, + reliable, }; + // Keep route in spec store self.content.details.insert(public_key, rsd); - - // xxx insert into cache too + if !self.cache.hop_cache.insert(cache_key) { + panic!("route should never be inserted twice"); + } + for h in &hops { + self.cache + .used_nodes + .entry(*h) + .and_modify(|e| *e += 1) + .or_insert(1); + } + self.cache + .used_end_nodes + .entry(*hops.last().unwrap()) + .and_modify(|e| *e += 1) + .or_insert(1); Some(public_key) } - pub fn release_route(&mut self, spec: Arc) {} + pub fn release_route(&mut self, public_key: DHTKey) { + if let Some(detail) = self.content.details.remove(&public_key) { + // Remove from hop cache + let cache_key = route_hops_to_hop_cache(&detail.hops); + if !self.cache.hop_cache.remove(&cache_key) { + panic!("hop cache should have contained cache key"); + } + // Remove from used nodes cache + for h in &detail.hops { + match self.cache.used_nodes.entry(*h) { + std::collections::hash_map::Entry::Occupied(o) => { + *o.get_mut() -= 1; + if *o.get() == 0 { + o.remove(); + } + } + std::collections::hash_map::Entry::Vacant(_) => { + panic!("used_nodes cache should have contained hop"); + } + } + } + // Remove from end nodes cache + match self.cache.used_nodes.entry(*detail.hops.last().unwrap()) { + std::collections::hash_map::Entry::Occupied(o) => { + *o.get_mut() -= 1; + if *o.get() == 0 { + o.remove(); + } + } + std::collections::hash_map::Entry::Vacant(_) => { + panic!("used_nodes cache should have contained hop"); + } + } + } else { + panic!("can't release route that was never allocated"); + } + } - pub fn best_route(&mut self, reliable: bool) -> Arc {} + pub fn best_route( + &mut self, + reliable: bool, + min_hop_count: usize, + max_hop_count: usize, + directions: DirectionSet, + ) -> Option { + for detail in &self.content.details { + if detail.1.reliable == reliable + && detail.1.hops.len() >= min_hop_count + && detail.1.hops.len() <= max_hop_count + && detail.1.directions.is_subset(directions) + { + return Some(*detail.0); + } + } + None + } + + /// xxx add route compiler here + /// /// Mark route as published /// When first deserialized, routes must be re-published in order to ensure they remain From a06c2fb5a395c6f93528abca711b7ca997c23eff Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 12 Oct 2022 15:52:19 -0400 Subject: [PATCH 13/67] checkpoint --- veilid-core/src/network_manager/tasks.rs | 4 +- veilid-core/src/routing_table/find_nodes.rs | 28 +-- .../src/routing_table/route_spec_store.rs | 176 ++++++++++++------ 3 files changed, 137 insertions(+), 71 deletions(-) diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index e41c8c06..c34b9cb7 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -224,7 +224,7 @@ impl NetworkManager { #[instrument(level = "trace", skip(self), err)] pub(super) async fn bootstrap_task_routine(self, stop_token: StopToken) -> EyreResult<()> { let (bootstrap, bootstrap_nodes) = { - let c = self.config.get(); + let c = self.unlocked_inner.config.get(); ( c.network.bootstrap.clone(), c.network.bootstrap_nodes.clone(), @@ -487,7 +487,7 @@ impl NetworkManager { let routing_table = self.routing_table(); let mut ord = FuturesOrdered::new(); let min_peer_count = { - let c = self.config.get(); + let c = self.unlocked_inner.config.get(); c.network.dht.min_peer_count as usize }; diff --git a/veilid-core/src/routing_table/find_nodes.rs b/veilid-core/src/routing_table/find_nodes.rs index 3e3866d4..0e304d91 100644 --- a/veilid-core/src/routing_table/find_nodes.rs +++ b/veilid-core/src/routing_table/find_nodes.rs @@ -74,13 +74,13 @@ impl RoutingTable { } // Retrieve the fastest nodes in the routing table matching an entry filter - pub fn find_fast_public_nodes_filtered<'r, 'e, F>( + pub fn find_fast_public_nodes_filtered<'a, 'b, F>( &self, node_count: usize, mut entry_filter: F, ) -> Vec where - F: FnMut(&'r RoutingTableInner, &'e BucketEntryInner) -> bool, + F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, { self.find_fastest_nodes( // count @@ -201,7 +201,7 @@ impl RoutingTable { } } - pub fn find_peers_with_sort_and_filter( + pub fn find_peers_with_sort_and_filter<'a, 'b, F, C, T, O>( &self, node_count: usize, cur_ts: u64, @@ -210,13 +210,13 @@ impl RoutingTable { mut transform: T, ) -> Vec where - F: FnMut(&RoutingTableInner, DHTKey, Option>) -> bool, + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, C: FnMut( - &RoutingTableInner, - &(DHTKey, Option>), - &(DHTKey, Option>), + &'a RoutingTableInner, + &'b (DHTKey, Option>), + &'b (DHTKey, Option>), ) -> core::cmp::Ordering, - T: FnMut(&RoutingTableInner, DHTKey, Option>) -> O, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, { let inner = self.inner.read(); let inner = &*inner; @@ -259,15 +259,15 @@ impl RoutingTable { out } - pub fn find_fastest_nodes( + pub fn find_fastest_nodes<'a, T, F, O>( &self, node_count: usize, mut filter: F, transform: T, ) -> Vec where - F: FnMut(&RoutingTableInner, DHTKey, Option>) -> bool, - T: FnMut(&RoutingTableInner, DHTKey, Option>) -> O, + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, { let cur_ts = intf::get_timestamp(); let out = self.find_peers_with_sort_and_filter( @@ -341,15 +341,15 @@ impl RoutingTable { out } - pub fn find_closest_nodes( + pub fn find_closest_nodes<'a, F, T, O>( &self, node_id: DHTKey, filter: F, mut transform: T, ) -> Vec where - F: FnMut(&RoutingTableInner, DHTKey, Option>) -> bool, - T: FnMut(&RoutingTableInner, DHTKey, Option>) -> O, + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, { let cur_ts = intf::get_timestamp(); let node_count = { diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 46368ef3..bc75825c 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -37,7 +37,7 @@ struct RouteSpecDetail { } /// The core representation of the RouteSpecStore that can be serialized -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Default, Serialize, Deserialize)] pub struct RouteSpecStoreContent { /// All of the routes we have allocated so far details: HashMap, @@ -154,32 +154,95 @@ impl RouteSpecStore { } } - pub fn load(routing_table: RoutingTable) -> Result { + pub async fn load(routing_table: RoutingTable) -> EyreResult { // Get cbor blob from table store - let content: RouteSpecStoreContent = serde_cbor::from_slice(cbor) - .map_err(|e| VeilidAPIError::parse_error("invalid route spec store content", e))?; - let rss = RouteSpecStore { + let table_store = routing_table.network_manager().table_store(); + let rsstdb = table_store.open("RouteSpecStore", 1).await?; + let content = rsstdb.load_cbor(0, b"content").await?.unwrap_or_default(); + let mut rss = RouteSpecStore { content, cache: Default::default(), }; + + // Load secrets from pstore + let pstore = routing_table.network_manager().protected_store(); + let mut dead_keys = Vec::new(); + for (k, v) in &mut rss.content.details { + if let Some(secret_key) = pstore + .load_user_secret(&format!("RouteSpecStore_{}", k.encode())) + .await? + { + match secret_key.try_into() { + Ok(s) => { + v.secret_key = DHTKeySecret::new(s); + } + Err(_) => { + dead_keys.push(*k); + } + } + } else { + dead_keys.push(*k); + } + } + for k in dead_keys { + log_rtab!(debug "killing off private route: {}", k.encode()); + rss.content.details.remove(&k); + } + + // Rebuild the routespecstore cache rss.rebuild_cache(routing_table); Ok(rss) } - pub fn save(&self, routing_table: RoutingTable) -> Result<(), VeilidAPIError> { + pub async fn save(&self, routing_table: RoutingTable) -> EyreResult<()> { // Save all the fields we care about to the cbor blob in table storage - let cbor = serde_cbor::to_vec(&self.content).unwrap(); let table_store = routing_table.network_manager().table_store(); - table_store.open("") + let rsstdb = table_store.open("RouteSpecStore", 1).await?; + rsstdb.store_cbor(0, b"content", &self.content).await?; + + // Keep secrets in protected store as well + let pstore = routing_table.network_manager().protected_store(); + for (k, v) in &self.content.details { + if pstore + .save_user_secret( + &format!("RouteSpecStore_{}", k.encode()), + &v.secret_key.bytes, + ) + .await? + { + panic!("route spec should not already have secret key saved"); + } + } + + Ok(()) + } + + fn add_to_cache(&mut self, cache_key: Vec, rsd: &RouteSpecDetail) { + if !self.cache.hop_cache.insert(cache_key) { + panic!("route should never be inserted twice"); + } + for h in &rsd.hops { + self.cache + .used_nodes + .entry(*h) + .and_modify(|e| *e += 1) + .or_insert(1); + } + self.cache + .used_end_nodes + .entry(*rsd.hops.last().unwrap()) + .and_modify(|e| *e += 1) + .or_insert(1); } fn rebuild_cache(&mut self, routing_table: RoutingTable) { - // - // xxx also load secrets from pstore - let pstore = routing_table.network_manager().protected_store(); + for v in self.content.details.values() { + let cache_key = route_hops_to_hop_cache(&v.hops); + self.add_to_cache(cache_key, &v); + } } - fn detail_mut(&mut self, public_key: DHTKey) -> &mut RouteSpecDetail { + fn detail_mut(&mut self, public_key: &DHTKey) -> &mut RouteSpecDetail { self.content.details.get_mut(&public_key).unwrap() } @@ -187,13 +250,13 @@ impl RouteSpecStore { /// Prefers nodes that are not currently in use by another route /// The route is not yet tested for its reachability /// Returns None if no route could be allocated at this time - pub fn allocate_route( + pub async fn allocate_route( &mut self, routing_table: RoutingTable, reliable: bool, hop_count: usize, directions: DirectionSet, - ) -> Option { + ) -> EyreResult> { use core::cmp::Ordering; let max_route_hop_count = { @@ -204,13 +267,11 @@ impl RouteSpecStore { }; if hop_count < 2 { - log_rtab!(error "Not allocating route less than two hops in length"); - return None; + bail!("Not allocating route less than two hops in length"); } if hop_count > max_route_hop_count { - log_rtab!(error "Not allocating route longer than max route hop count"); - return None; + bail!("Not allocating route longer than max route hop count"); } // Get list of all nodes, and sort them for selection @@ -293,8 +354,8 @@ impl RouteSpecStore { } // always prioritize reliable nodes, but sort by oldest or fastest - let cmpout = v1.1.unwrap().with(rti, |rti, e1| { - v2.1.unwrap().with(rti, |_rti, e2| { + let cmpout = v1.1.as_ref().unwrap().with(rti, |rti, e1| { + v2.1.as_ref().unwrap().with(rti, |_rti, e2| { if reliable { BucketEntryInner::cmp_oldest_reliable(cur_ts, e1, e2) } else { @@ -321,13 +382,13 @@ impl RouteSpecStore { RoutingDomain::PublicInternet.into(), BucketEntryState::Unreliable, ); - let mut nodes = routing_table + let nodes = routing_table .find_peers_with_sort_and_filter(node_count, cur_ts, filter, compare, transform); // If we couldn't find enough nodes, wait until we have more nodes in the routing table if nodes.len() < hop_count { - log_rtab!(debug "Not enough nodes to construct route at this time. Try again later."); - return None; + log_rtab!(debug "not enough nodes to construct route at this time"); + return Ok(None); } // Now go through nodes and try to build a route we haven't seen yet @@ -396,7 +457,8 @@ impl RouteSpecStore { } } if route_nodes.is_empty() { - return None; + log_rtab!(debug "unable to find unique route at this time"); + return Ok(None); } // Got a unique route, lets build the detail, register it, and return it @@ -423,25 +485,13 @@ impl RouteSpecStore { reliable, }; + // Add to cache + self.add_to_cache(cache_key, &rsd); + // Keep route in spec store self.content.details.insert(public_key, rsd); - if !self.cache.hop_cache.insert(cache_key) { - panic!("route should never be inserted twice"); - } - for h in &hops { - self.cache - .used_nodes - .entry(*h) - .and_modify(|e| *e += 1) - .or_insert(1); - } - self.cache - .used_end_nodes - .entry(*hops.last().unwrap()) - .and_modify(|e| *e += 1) - .or_insert(1); - Some(public_key) + Ok(Some(public_key)) } pub fn release_route(&mut self, public_key: DHTKey) { @@ -482,7 +532,7 @@ impl RouteSpecStore { } } - pub fn best_route( + pub fn first_unpublished_route( &mut self, reliable: bool, min_hop_count: usize, @@ -494,6 +544,7 @@ impl RouteSpecStore { && detail.1.hops.len() >= min_hop_count && detail.1.hops.len() <= max_hop_count && detail.1.directions.is_subset(directions) + && !detail.1.published { return Some(*detail.0); } @@ -507,31 +558,46 @@ impl RouteSpecStore { /// Mark route as published /// When first deserialized, routes must be re-published in order to ensure they remain /// in the RouteSpecStore. - pub fn mark_route_published(&mut self, spec: Arc) { - self.detail_mut(spec).published = true; + pub fn mark_route_published(&mut self, key: &DHTKey) { + self.detail_mut(&key).published = true; } /// Mark route as checked - pub fn touch_route_checked(&mut self, spec: Arc, cur_ts: u64) { - self.detail_mut(spec).last_checked_ts = cur_ts; + pub fn touch_route_checked(&mut self, key: &DHTKey, cur_ts: u64) { + self.detail_mut(&key).last_checked_ts = Some(cur_ts); } - pub fn record_latency(&mut self, spec: Arc, latency: u64) { - let lsa = self.detail_mut(spec).latency_stats_accounting; - self.detail_mut(spec).latency_stats = lsa.record_latency(latency); + /// Record latency on the route + pub fn record_latency(&mut self, key: &DHTKey, latency: u64) { + let lsa = &mut self.detail_mut(&key).latency_stats_accounting; + self.detail_mut(&key).latency_stats = lsa.record_latency(latency); } - pub fn latency_stats(&self, spec: Arc) -> LatencyStats { - self.detail_mut(spec).latency_stats.clone() + /// Get the calculated latency stats + pub fn latency_stats(&self, key: &DHTKey) -> LatencyStats { + self.detail_mut(&key).latency_stats.clone() } - pub fn add_down(&mut self, spec: Arc, bytes: u64) { - self.current_transfer.down += bytes; + /// Add download transfers to route + pub fn add_down(&mut self, key: &DHTKey, bytes: u64) { + let tsa = &mut self.detail_mut(&key).transfer_stats_accounting; + tsa.add_down(bytes); } - pub fn add_up(&mut self, spec: Arc, bytes: u64) {} + /// Add upload transfers to route + pub fn add_up(&mut self, key: &DHTKey, bytes: u64) { + let tsa = &mut self.detail_mut(&key).transfer_stats_accounting; + tsa.add_up(bytes); + } - pub fn roll_transfers(&mut self) { - // + /// Process transfer statistics to get averages + pub fn roll_transfers(&mut self, last_ts: u64, cur_ts: u64) { + for rsd in self.content.details.values_mut() { + rsd.transfer_stats_accounting.roll_transfers( + last_ts, + cur_ts, + &mut rsd.transfer_stats_down_up, + ); + } } } From 2d526674a5e468ec47e01ba42ee4358b90f25776 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 12 Oct 2022 22:53:40 -0400 Subject: [PATCH 14/67] refactor get_contact_method --- veilid-core/src/network_manager/mod.rs | 222 +++--------------- veilid-core/src/routing_table/mod.rs | 152 ++++++++++-- veilid-core/src/routing_table/node_ref.rs | 5 +- .../src/routing_table/route_spec_store.rs | 40 +++- .../src/routing_table/routing_domains.rs | 218 ++++++++++++++++- .../src/rpc_processor/private_route.rs | 3 + veilid-core/src/veilid_api/debug.rs | 4 +- veilid-core/src/veilid_api/mod.rs | 2 +- 8 files changed, 424 insertions(+), 222 deletions(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index d22716a8..00d8ac63 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -114,23 +114,6 @@ struct ClientWhitelistEntry { last_seen_ts: u64, } -/// Mechanism required to contact another node -#[derive(Clone, Debug)] -pub(crate) enum ContactMethod { - /// Node is not reachable by any means - Unreachable, - /// Contact the node directly - Direct(DialInfo), - /// Request via signal the node connect back directly (relay_nr, target_node_ref) - SignalReverse(NodeRef, NodeRef), - /// Request via signal the node negotiate a hole punch (relay_nr, target_node_ref) - SignalHolePunch(NodeRef, NodeRef), - /// Must use an inbound relay to reach the node - InboundRelay(NodeRef), - /// Must use outbound relay to reach the node - OutboundRelay(NodeRef), -} - #[derive(Copy, Clone, Debug)] pub enum SendDataKind { Direct(ConnectionDescriptor), @@ -138,6 +121,25 @@ pub enum SendDataKind { Existing(ConnectionDescriptor), } +/// Mechanism required to contact another node +#[derive(Clone, Debug)] +pub(crate) enum NodeContactMethod { + /// Node is not reachable by any means + Unreachable, + /// Connection should have already existed + Existing, + /// Contact the node directly + Direct(DialInfo), + /// Request via signal the node connect back directly (relay, target) + SignalReverse(NodeRef, NodeRef), + /// Request via signal the node negotiate a hole punch (relay, target_node) + SignalHolePunch(NodeRef, NodeRef), + /// Must use an inbound relay to reach the node + InboundRelay(NodeRef), + /// Must use outbound relay to reach the node + OutboundRelay(NodeRef), +} + #[derive(Copy, Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Hash)] struct PublicAddressCheckCacheKey(ProtocolType, AddressType); @@ -965,173 +967,6 @@ impl NetworkManager { Ok(()) } - #[instrument(level = "trace", skip(self), ret)] - fn get_contact_method_public(&self, target_node_ref: NodeRef) -> ContactMethod { - // Scope noderef down to protocols we can do outbound - let routing_table = self.routing_table(); - - let public_outbound_nrf = - routing_table.get_outbound_node_ref_filter(RoutingDomain::PublicInternet); - let target_node_ref = target_node_ref.filtered_clone(public_outbound_nrf.clone()); - - // Get the best match internet dial info if we have it - let opt_target_public_did = target_node_ref.first_filtered_dial_info_detail(); - if let Some(target_public_did) = opt_target_public_did { - // Do we need to signal before going inbound? - if !target_public_did.class.requires_signal() { - // Go direct without signaling - return ContactMethod::Direct(target_public_did.dial_info); - } - - // Get the target's inbound relay, it must have one or it is not reachable - // Note that .relay() never returns our own node. We can't relay to ourselves. - if let Some(inbound_relay_nr) = target_node_ref.relay(RoutingDomain::PublicInternet) { - // Scope down to protocols we can do outbound - let inbound_relay_nr = inbound_relay_nr.filtered_clone(public_outbound_nrf.clone()); - // Can we reach the inbound relay? - if inbound_relay_nr.first_filtered_dial_info_detail().is_some() { - // Can we receive anything inbound ever? - let our_network_class = routing_table - .get_network_class(RoutingDomain::PublicInternet) - .unwrap_or(NetworkClass::Invalid); - if matches!(our_network_class, NetworkClass::InboundCapable) { - ///////// Reverse connection - - // Get the best match dial info for an reverse inbound connection - let reverse_dif = routing_table - .get_inbound_dial_info_filter(RoutingDomain::PublicInternet) - .filtered( - &target_node_ref - .node_info_outbound_filter(RoutingDomain::PublicInternet), - ); - if let Some(reverse_did) = routing_table.first_filtered_dial_info_detail( - RoutingDomain::PublicInternet.into(), - &reverse_dif, - ) { - // Ensure we aren't on the same public IP address (no hairpin nat) - if reverse_did.dial_info.to_ip_addr() - != target_public_did.dial_info.to_ip_addr() - { - // Can we receive a direct reverse connection? - if !reverse_did.class.requires_signal() { - return ContactMethod::SignalReverse( - inbound_relay_nr, - target_node_ref, - ); - } - } - } - - ///////// UDP hole-punch - - // Does the target have a direct udp dialinfo we can reach? - let udp_target_nr = target_node_ref.filtered_clone( - NodeRefFilter::new().with_protocol_type(ProtocolType::UDP), - ); - if let Some(target_udp_dialinfo_detail) = - udp_target_nr.first_filtered_dial_info_detail() - { - // Does the self node have a direct udp dialinfo the target can reach? - let inbound_udp_dif = routing_table - .get_inbound_dial_info_filter(RoutingDomain::PublicInternet) - .filtered( - &target_node_ref - .node_info_outbound_filter(RoutingDomain::PublicInternet), - ) - .filtered( - &DialInfoFilter::all().with_protocol_type(ProtocolType::UDP), - ); - if let Some(self_udp_dialinfo_detail) = routing_table - .first_filtered_dial_info_detail( - RoutingDomain::PublicInternet.into(), - &inbound_udp_dif, - ) - { - // Ensure we aren't on the same public IP address (no hairpin nat) - if target_udp_dialinfo_detail.dial_info.to_ip_addr() - != self_udp_dialinfo_detail.dial_info.to_ip_addr() - { - // The target and ourselves have a udp dialinfo that they can reach - return ContactMethod::SignalHolePunch( - inbound_relay_nr, - udp_target_nr, - ); - } - } - } - // Otherwise we have to inbound relay - } - - return ContactMethod::InboundRelay(inbound_relay_nr); - } - } - } - // If the other node is not inbound capable at all, it needs to have an inbound relay - else if let Some(target_inbound_relay_nr) = - target_node_ref.relay(RoutingDomain::PublicInternet) - { - // Can we reach the full relay? - if target_inbound_relay_nr - .first_filtered_dial_info_detail() - .is_some() - { - return ContactMethod::InboundRelay(target_inbound_relay_nr); - } - } - - // If we can't reach the node by other means, try our outbound relay if we have one - if let Some(relay_node) = self - .routing_table() - .relay_node(RoutingDomain::PublicInternet) - { - return ContactMethod::OutboundRelay(relay_node); - } - - ContactMethod::Unreachable - } - - #[instrument(level = "trace", skip(self), ret)] - fn get_contact_method_local(&self, target_node_ref: NodeRef) -> ContactMethod { - // Scope noderef down to protocols we can do outbound - let routing_table = self.routing_table(); - - let local_outbound_nrf = - routing_table.get_outbound_node_ref_filter(RoutingDomain::LocalNetwork); - let target_node_ref = target_node_ref.filtered_clone(local_outbound_nrf); - - // Get the best matching local direct dial info if we have it - if target_node_ref.is_filter_dead() { - return ContactMethod::Unreachable; - } - let opt_target_local_did = target_node_ref.first_filtered_dial_info_detail(); - if let Some(target_local_did) = opt_target_local_did { - return ContactMethod::Direct(target_local_did.dial_info); - } - return ContactMethod::Unreachable; - } - - // Figure out how to reach a node - #[instrument(level = "trace", skip(self), ret)] - pub(crate) fn get_contact_method(&self, target_node_ref: NodeRef) -> ContactMethod { - let routing_domain = match target_node_ref.best_routing_domain() { - Some(rd) => rd, - None => { - log_net!("no routing domain for node {:?}", target_node_ref); - return ContactMethod::Unreachable; - } - }; - - match routing_domain { - RoutingDomain::LocalNetwork => self.get_contact_method_local(target_node_ref), - RoutingDomain::PublicInternet => self.get_contact_method_public(target_node_ref), - } - } - - /// Get the contact method required for node A to reach node B - pub fn get_node_contact_method(node_a: &NodeInfo, node_b: &NodeInfo) -> ContactMethod { - unimplemented!(); - } - // Send a reverse connection signal and wait for the return receipt over it // Then send the data across the new connection // Only usable for PublicInternet routing domain @@ -1366,19 +1201,21 @@ impl NetworkManager { // info!("{}", "no existing connection".red()); // If we don't have last_connection, try to reach out to the peer via its dial info - let contact_method = this.get_contact_method(node_ref.clone()); + let contact_method = this + .routing_table() + .get_node_contact_method(node_ref.clone())?; log_net!( "send_data via {:?} to dialinfo {:?}", contact_method, node_ref ); match contact_method { - ContactMethod::OutboundRelay(relay_nr) - | ContactMethod::InboundRelay(relay_nr) => { + NodeContactMethod::OutboundRelay(relay_nr) + | NodeContactMethod::InboundRelay(relay_nr) => { network_result_try!(this.send_data(relay_nr, data).await?); Ok(NetworkResult::value(SendDataKind::Indirect)) } - ContactMethod::Direct(dial_info) => { + NodeContactMethod::Direct(dial_info) => { let connection_descriptor = network_result_try!( this.net().send_data_to_dial_info(dial_info, data).await? ); @@ -1389,7 +1226,7 @@ impl NetworkManager { connection_descriptor, ))) } - ContactMethod::SignalReverse(relay_nr, target_node_ref) => { + NodeContactMethod::SignalReverse(relay_nr, target_node_ref) => { let connection_descriptor = network_result_try!( this.do_reverse_connect(relay_nr, target_node_ref, data) .await? @@ -1398,7 +1235,7 @@ impl NetworkManager { connection_descriptor, ))) } - ContactMethod::SignalHolePunch(relay_nr, target_node_ref) => { + NodeContactMethod::SignalHolePunch(relay_nr, target_node_ref) => { let connection_descriptor = network_result_try!( this.do_hole_punch(relay_nr, target_node_ref, data).await? ); @@ -1406,7 +1243,10 @@ impl NetworkManager { connection_descriptor, ))) } - ContactMethod::Unreachable => Ok(NetworkResult::no_connection_other( + NodeContactMethod::Existing => Ok(NetworkResult::no_connection_other( + "should have found an existing connection", + )), + NodeContactMethod::Unreachable => Ok(NetworkResult::no_connection_other( "Can't send to this node", )), } diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 1e86c2d6..0ed4ad5b 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -323,6 +323,110 @@ impl RoutingTable { true } + /// Look up the best way for two nodes to reach each other over a specific routing domain + #[instrument(level = "trace", skip(self), ret)] + pub fn get_contact_method( + &self, + routing_domain: RoutingDomain, + node_a_id: &DHTKey, + node_a: &NodeInfo, + node_b_id: &DHTKey, + node_b: &NodeInfo, + dial_info_filter: DialInfoFilter, + reliable: bool, + ) -> ContactMethod { + let inner = &*self.inner.read(); + Self::with_routing_domain(inner, routing_domain, |rdd| { + rdd.get_contact_method( + inner, + node_a_id, + node_a, + node_b_id, + node_b, + dial_info_filter, + reliable, + ) + }) + } + + // Figure out how to reach a node from our own node over the best routing domain and reference the nodes we want to access + #[instrument(level = "trace", skip(self), ret)] + pub(crate) fn get_node_contact_method( + &self, + target_node_ref: NodeRef, + ) -> EyreResult { + // Lock the routing table for read to ensure the table doesn't change + let inner = &*self.inner.read(); + + // Figure out the best routing domain to get the contact method over + let routing_domain = match target_node_ref.best_routing_domain() { + Some(rd) => rd, + None => { + log_net!("no routing domain for node {:?}", target_node_ref); + return Ok(NodeContactMethod::Unreachable); + } + }; + + // Node A is our own node + let node_a = self.get_own_node_info(routing_domain); + let node_a_id = self.node_id(); + + // Node B is the target node + let node_b = target_node_ref.operate(|_rti, e| e.node_info(routing_domain).unwrap()); + let node_b_id = target_node_ref.node_id(); + + // Dial info filter comes from the target node ref + let dial_info_filter = target_node_ref.dial_info_filter(); + let reliable = target_node_ref.reliable(); + + let cm = self.get_contact_method( + routing_domain, + &node_a_id, + &node_a, + &node_b_id, + node_b, + dial_info_filter, + reliable, + ); + + // Translate the raw contact method to a referenced contact method + Ok(match cm { + ContactMethod::Unreachable => NodeContactMethod::Unreachable, + ContactMethod::Existing => NodeContactMethod::Existing, + ContactMethod::Direct(di) => NodeContactMethod::Direct(di), + ContactMethod::SignalReverse(relay_key, target_key) => { + let relay_nr = self + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .ok_or_else(|| eyre!("couldn't look up relay"))?; + if target_node_ref.node_id() != target_key { + bail!("target noderef didn't match target key"); + } + NodeContactMethod::SignalReverse(relay_nr, target_node_ref) + } + ContactMethod::SignalHolePunch(relay_key, target_key) => { + let relay_nr = self + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .ok_or_else(|| eyre!("couldn't look up relay"))?; + if target_node_ref.node_id() != target_key { + bail!("target noderef didn't match target key"); + } + NodeContactMethod::SignalHolePunch(relay_nr, target_node_ref) + } + ContactMethod::InboundRelay(relay_key) => { + let relay_nr = self + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .ok_or_else(|| eyre!("couldn't look up relay"))?; + NodeContactMethod::InboundRelay(relay_nr) + } + ContactMethod::OutboundRelay(relay_key) => { + let relay_nr = self + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .ok_or_else(|| eyre!("couldn't look up relay"))?; + NodeContactMethod::OutboundRelay(relay_nr) + } + }) + } + #[instrument(level = "debug", skip(self))] pub fn edit_routing_domain(&self, domain: RoutingDomain) -> RoutingDomainEditor { RoutingDomainEditor::new(self.clone(), domain) @@ -487,8 +591,8 @@ impl RoutingTable { } } - // Attempt to empty the routing table - // should only be performed when there are no node_refs (detached) + /// Attempt to empty the routing table + /// should only be performed when there are no node_refs (detached) pub fn purge_buckets(&self) { let mut inner = self.inner.write(); let inner = &mut *inner; @@ -505,7 +609,7 @@ impl RoutingTable { ); } - // Attempt to remove last_connections from entries + /// Attempt to remove last_connections from entries pub fn purge_last_connections(&self) { let mut inner = self.inner.write(); let inner = &mut *inner; @@ -526,8 +630,8 @@ impl RoutingTable { ); } - // Attempt to settle buckets and remove entries down to the desired number - // which may not be possible due extant NodeRefs + /// Attempt to settle buckets and remove entries down to the desired number + /// which may not be possible due extant NodeRefs fn kick_bucket(inner: &mut RoutingTableInner, idx: usize) { let bucket = &mut inner.buckets[idx]; let bucket_depth = Self::bucket_depth(idx); @@ -702,9 +806,9 @@ impl RoutingTable { self.unlocked_inner.kick_queue.lock().insert(idx); } - // Create a node reference, possibly creating a bucket entry - // the 'update_func' closure is called on the node, and, if created, - // in a locked fashion as to ensure the bucket entry state is always valid + /// Create a node reference, possibly creating a bucket entry + /// the 'update_func' closure is called on the node, and, if created, + /// in a locked fashion as to ensure the bucket entry state is always valid pub fn create_node_ref(&self, node_id: DHTKey, update_func: F) -> Option where F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner), @@ -759,6 +863,7 @@ impl RoutingTable { Some(noderef) } + /// Resolve an existing routing table entry and return a reference to it pub fn lookup_node_ref(&self, node_id: DHTKey) -> Option { if node_id == self.unlocked_inner.node_id { log_rtab!(debug "can't look up own node id in routing table"); @@ -772,9 +877,26 @@ impl RoutingTable { .map(|e| NodeRef::new(self.clone(), node_id, e, None)) } - // Shortcut function to add a node to our routing table if it doesn't exist - // and add the dial info we have for it. Returns a noderef filtered to - // the routing domain in which this node was registered for convenience. + /// Resolve an existing routing table entry and return a filtered reference to it + pub fn lookup_and_filter_noderef( + &self, + node_id: DHTKey, + routing_domain_set: RoutingDomainSet, + dial_info_filter: DialInfoFilter, + ) -> Option { + let nr = self.lookup_node_ref(node_id)?; + Some( + nr.filtered_clone( + NodeRefFilter::new() + .with_dial_info_filter(dial_info_filter) + .with_routing_domain_set(routing_domain_set), + ), + ) + } + + /// Shortcut function to add a node to our routing table if it doesn't exist + /// and add the dial info we have for it. Returns a noderef filtered to + /// the routing domain in which this node was registered for convenience. pub fn register_node_with_signed_node_info( &self, routing_domain: RoutingDomain, @@ -821,8 +943,8 @@ impl RoutingTable { }) } - // Shortcut function to add a node to our routing table if it doesn't exist - // and add the last peer address we have for it, since that's pretty common + /// Shortcut function to add a node to our routing table if it doesn't exist + /// and add the last peer address we have for it, since that's pretty common pub fn register_node_with_existing_connection( &self, node_id: DHTKey, @@ -840,8 +962,8 @@ impl RoutingTable { out } - // Ticks about once per second - // to run tick tasks which may run at slower tick rates as configured + /// Ticks about once per second + /// to run tick tasks which may run at slower tick rates as configured pub async fn tick(&self) -> EyreResult<()> { // Do rolling transfers every ROLLING_TRANSFERS_INTERVAL_SECS secs self.unlocked_inner.rolling_transfers_task.tick().await?; diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 4586189c..d3aa3ac5 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -6,7 +6,7 @@ use alloc::fmt; // We should ping them with some frequency and 30 seconds is typical timeout const CONNECTIONLESS_TIMEOUT_SECS: u32 = 29; -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct NodeRefFilter { pub routing_domain_set: RoutingDomainSet, pub dial_info_filter: DialInfoFilter, @@ -131,6 +131,9 @@ impl NodeRef { pub fn set_reliable(&mut self) { self.reliable = true; } + pub fn reliable(&self) -> bool { + self.reliable + } pub fn merge_filter(&mut self, filter: NodeRefFilter) { if let Some(self_filter) = self.filter.take() { diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index bc75825c..6ee67a36 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -274,6 +274,10 @@ impl RouteSpecStore { bail!("Not allocating route longer than max route hop count"); } + // Lock routing table for reading, make sure things don't change + // because we want to iterate the table without changes being made to it + let rti = routing_table.inner.read(); + // Get list of all nodes, and sort them for selection let cur_ts = intf::get_timestamp(); let dial_info_sort = if reliable { @@ -409,19 +413,25 @@ impl RouteSpecStore { if directions.contains(Direction::Outbound) { let our_node_info = routing_table.get_own_node_info(RoutingDomain::PublicInternet); - let mut previous_node_info = &our_node_info; + let our_node_id = routing_table.node_id(); + let mut previous_node = &(our_node_id, our_node_info); let mut reachable = true; for n in permutation { - let current_node_info = &nodes.get(*n).as_ref().unwrap().1; - let cm = NetworkManager::get_node_contact_method( - previous_node_info, - current_node_info, + let current_node = nodes.get(*n).unwrap(); + let cm = routing_table.get_contact_method( + RoutingDomain::PublicInternet, + &previous_node.0, + &previous_node.1, + ¤t_node.0, + ¤t_node.1, + DialInfoFilter::all(), + reliable, ); if matches!(cm, ContactMethod::Unreachable) { reachable = false; break; } - previous_node_info = current_node_info; + previous_node = current_node; } if !reachable { return false; @@ -430,19 +440,25 @@ impl RouteSpecStore { if directions.contains(Direction::Inbound) { let our_node_info = routing_table.get_own_node_info(RoutingDomain::PublicInternet); - let mut next_node_info = &our_node_info; + let our_node_id = routing_table.node_id(); + let mut next_node = &(our_node_id, our_node_info); let mut reachable = true; for n in permutation.iter().rev() { - let current_node_info = &nodes.get(*n).as_ref().unwrap().1; - let cm = NetworkManager::get_node_contact_method( - current_node_info, - next_node_info, + let current_node = nodes.get(*n).unwrap(); + let cm = routing_table.get_contact_method( + RoutingDomain::PublicInternet, + &next_node.0, + &next_node.1, + ¤t_node.0, + ¤t_node.1, + DialInfoFilter::all(), + reliable, ); if matches!(cm, ContactMethod::Unreachable) { reachable = false; break; } - next_node_info = current_node_info; + next_node = current_node; } if !reachable { return false; diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 80ed405c..6475cd01 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -1,5 +1,24 @@ use super::*; +/// Mechanism required to contact another node +#[derive(Clone, Debug)] +pub(crate) enum ContactMethod { + /// Node is not reachable by any means + Unreachable, + /// Connection should have already existed + Existing, + /// Contact the node directly + Direct(DialInfo), + /// Request via signal the node connect back directly (relay, target) + SignalReverse(DHTKey, DHTKey), + /// Request via signal the node negotiate a hole punch (relay, target_node) + SignalHolePunch(DHTKey, DHTKey), + /// Must use an inbound relay to reach the node + InboundRelay(DHTKey), + /// Must use outbound relay to reach the node + OutboundRelay(DHTKey), +} + #[derive(Debug)] pub struct RoutingDomainDetailCommon { routing_domain: RoutingDomain, @@ -147,8 +166,21 @@ pub trait RoutingDomainDetail { fn common(&self) -> &RoutingDomainDetailCommon; fn common_mut(&mut self) -> &mut RoutingDomainDetailCommon; - // Per-domain accessors + /// Can this routing domain contain a particular address fn can_contain_address(&self, address: Address) -> bool; + + /// Get the contact method required for node A to reach node B in this routing domain + /// Routing table must be locked for reading to use this function + fn get_contact_method( + &self, + rti: &RoutingTableInner, + node_a_id: &DHTKey, + node_a: &NodeInfo, + node_b_id: &DHTKey, + node_b: &NodeInfo, + dial_info_filter: DialInfoFilter, + reliable: bool, + ) -> ContactMethod; } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -168,6 +200,30 @@ impl Default for PublicInternetRoutingDomainDetail { } } +fn first_filtered_dial_info_detail( + from_node: &NodeInfo, + to_node: &NodeInfo, + dial_info_filter: &DialInfoFilter, + reliable: bool, +) -> Option { + let direct_dial_info_filter = dial_info_filter.clone().filtered( + &DialInfoFilter::all() + .with_address_type_set(from_node.address_types) + .with_protocol_type_set(from_node.outbound_protocols), + ); + + // Get first filtered dialinfo + let sort = if reliable { + Some(DialInfoDetail::reliable_sort) + } else { + None + }; + let direct_filter = |did: &DialInfoDetail| did.matches_filter(&direct_dial_info_filter); + + // Get the best match dial info for node B if we have it + to_node.first_filtered_dial_info_detail(sort, direct_filter) +} + impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { fn common(&self) -> &RoutingDomainDetailCommon { &self.common @@ -178,6 +234,128 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { fn can_contain_address(&self, address: Address) -> bool { address.is_global() } + fn get_contact_method( + &self, + _rti: &RoutingTableInner, + node_a_id: &DHTKey, + node_a: &NodeInfo, + node_b_id: &DHTKey, + node_b: &NodeInfo, + dial_info_filter: DialInfoFilter, + reliable: bool, + ) -> ContactMethod { + // Get the best match dial info for node B if we have it + if let Some(target_did) = + first_filtered_dial_info_detail(node_a, node_b, &dial_info_filter, reliable) + { + // Do we need to signal before going inbound? + if !target_did.class.requires_signal() { + // Go direct without signaling + return ContactMethod::Direct(target_did.dial_info); + } + + // Get the target's inbound relay, it must have one or it is not reachable + if let Some(inbound_relay) = node_b.relay_peer_info { + // Note that relay_peer_info could be node_a, in which case a connection already exists + // and we shouldn't have even gotten here + if inbound_relay.node_id.key == *node_a_id { + return ContactMethod::Existing; + } + + // Can node A reach the inbound relay directly? + if first_filtered_dial_info_detail( + node_a, + &inbound_relay.signed_node_info.node_info, + &dial_info_filter, + reliable, + ) + .is_some() + { + // Can node A receive anything inbound ever? + if matches!(node_a.network_class, NetworkClass::InboundCapable) { + ///////// Reverse connection + + // Get the best match dial info for an reverse inbound connection from node B to node A + if let Some(reverse_did) = first_filtered_dial_info_detail( + node_b, + node_a, + &dial_info_filter, + reliable, + ) { + // Ensure we aren't on the same public IP address (no hairpin nat) + if reverse_did.dial_info.to_ip_addr() + != target_did.dial_info.to_ip_addr() + { + // Can we receive a direct reverse connection? + if !reverse_did.class.requires_signal() { + return ContactMethod::SignalReverse( + inbound_relay.node_id.key, + *node_b_id, + ); + } + } + } + + ///////// UDP hole-punch + + // Does node B have a direct udp dialinfo node A can reach? + let udp_dial_info_filter = dial_info_filter + .clone() + .filtered(&DialInfoFilter::all().with_protocol_type(ProtocolType::UDP)); + if let Some(target_udp_did) = first_filtered_dial_info_detail( + node_a, + node_b, + &udp_dial_info_filter, + reliable, + ) { + // Does node A have a direct udp dialinfo that node B can reach? + if let Some(reverse_udp_did) = first_filtered_dial_info_detail( + node_b, + node_a, + &udp_dial_info_filter, + reliable, + ) { + // Ensure we aren't on the same public IP address (no hairpin nat) + if reverse_udp_did.dial_info.to_ip_addr() + != target_udp_did.dial_info.to_ip_addr() + { + // The target and ourselves have a udp dialinfo that they can reach + return ContactMethod::SignalHolePunch( + inbound_relay.node_id.key, + *node_b_id, + ); + } + } + } + // Otherwise we have to inbound relay + } + + return ContactMethod::InboundRelay(inbound_relay.node_id.key); + } + } + } + // If the node B has no direct dial info, it needs to have an inbound relay + else if let Some(inbound_relay) = node_b.relay_peer_info { + // Can we reach the full relay? + if first_filtered_dial_info_detail( + node_a, + &inbound_relay.signed_node_info.node_info, + &dial_info_filter, + reliable, + ) + .is_some() + { + return ContactMethod::InboundRelay(inbound_relay.node_id.key); + } + } + + // If node A can't reach the node by other means, it may need to use its own relay + if let Some(outbound_relay) = node_a.relay_peer_info { + return ContactMethod::OutboundRelay(outbound_relay.node_id.key); + } + + ContactMethod::Unreachable + } } /// Local Network routing domain internals @@ -225,4 +403,42 @@ impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { } false } + + fn get_contact_method( + &self, + _rti: &RoutingTableInner, + _node_a_id: &DHTKey, + node_a: &NodeInfo, + _node_b_id: &DHTKey, + node_b: &NodeInfo, + dial_info_filter: DialInfoFilter, + reliable: bool, + ) -> ContactMethod { + // Scope the filter down to protocols node A can do outbound + let dial_info_filter = dial_info_filter.filtered( + &DialInfoFilter::all() + .with_address_type_set(node_a.address_types) + .with_protocol_type_set(node_a.outbound_protocols), + ); + + // If the filter is dead then we won't be able to connect + if dial_info_filter.is_dead() { + return ContactMethod::Unreachable; + } + + // Get first filtered dialinfo + let sort = if reliable { + Some(DialInfoDetail::reliable_sort) + } else { + None + }; + let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); + + let opt_target_did = node_b.first_filtered_dial_info_detail(sort, filter); + if let Some(target_did) = opt_target_did { + return ContactMethod::Direct(target_did.dial_info); + } + + ContactMethod::Unreachable + } } diff --git a/veilid-core/src/rpc_processor/private_route.rs b/veilid-core/src/rpc_processor/private_route.rs index b552e768..dbf11c9d 100644 --- a/veilid-core/src/rpc_processor/private_route.rs +++ b/veilid-core/src/rpc_processor/private_route.rs @@ -1,6 +1,9 @@ use super::*; impl RPCProcessor { + +xxx move this into route spec store + ////////////////////////////////////////////////////////////////////// fn compile_safety_route( &self, diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 16b7b6b7..96f46ae1 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -341,7 +341,9 @@ impl VeilidAPI { nr.merge_filter(NodeRefFilter::new().with_routing_domain(routing_domain)) } - let cm = network_manager.get_contact_method(nr); + let cm = routing_table + .get_node_contact_method(nr) + .map_err(VeilidAPIError::internal)?; Ok(format!("{:#?}", cm)) } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index b224e639..92f6fede 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -926,7 +926,7 @@ impl FromStr for SocketAddress { ////////////////////////////////////////////////////////////////// -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct DialInfoFilter { pub protocol_type_set: ProtocolTypeSet, pub address_type_set: AddressTypeSet, From e85d72f21afeed3ebc6128f1a33e6c55f3360b7a Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 13 Oct 2022 22:05:43 -0400 Subject: [PATCH 15/67] more private route work --- doc/config/sample.config | 2 + doc/config/veilid-server-config.md | 1 + veilid-core/src/routing_table/mod.rs | 24 ++- .../src/routing_table/route_spec_store.rs | 178 ++++++++++++++++-- veilid-core/src/rpc_processor/destination.rs | 1 + veilid-core/src/rpc_processor/mod.rs | 165 ++++++++-------- .../src/rpc_processor/private_route.rs | 154 --------------- .../src/tests/common/test_veilid_config.rs | 2 + veilid-core/src/veilid_config.rs | 33 +++- veilid-flutter/example/lib/config.dart | 1 + veilid-flutter/lib/veilid.dart | 8 +- veilid-server/src/settings.rs | 9 +- veilid-wasm/tests/web.rs | 1 + 13 files changed, 323 insertions(+), 256 deletions(-) delete mode 100644 veilid-core/src/rpc_processor/private_route.rs diff --git a/doc/config/sample.config b/doc/config/sample.config index 3c7a4950..1b169390 100644 --- a/doc/config/sample.config +++ b/doc/config/sample.config @@ -64,6 +64,8 @@ core: max_timestamp_ahead_ms: 10000 timeout_ms: 10000 max_route_hop_count: 7 + default_route_hop_count: 2 + dht: resolve_node_timeout: resolve_node_count: 20 diff --git a/doc/config/veilid-server-config.md b/doc/config/veilid-server-config.md index be37c7ed..210b52b4 100644 --- a/doc/config/veilid-server-config.md +++ b/doc/config/veilid-server-config.md @@ -229,6 +229,7 @@ rpc: max_timestamp_ahead_ms: 10000 timeout_ms: 10000 max_route_hop_count: 7 + default_route_hop_count: 2 ``` #### core:network:dht diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 0ed4ad5b..ab6fcc09 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -90,7 +90,7 @@ pub struct RoutingTable { } impl RoutingTable { - fn new_inner() -> RoutingTableInner { + fn new_inner(config: VeilidConfig) -> RoutingTableInner { RoutingTableInner { buckets: Vec::new(), public_internet_routing_domain: PublicInternetRoutingDomainDetail::default(), @@ -100,7 +100,7 @@ impl RoutingTable { self_transfer_stats_accounting: TransferStatsAccounting::new(), self_transfer_stats: TransferStatsDownUp::default(), recent_peers: LruCache::new(RECENT_PEERS_TABLE_SIZE), - route_spec_store: RouteSpecStore::new(), + route_spec_store: RouteSpecStore::new(config), } } fn new_unlocked_inner( @@ -121,7 +121,7 @@ impl RoutingTable { pub fn new(network_manager: NetworkManager) -> Self { let config = network_manager.config(); let this = Self { - inner: Arc::new(RwLock::new(Self::new_inner())), + inner: Arc::new(RwLock::new(Self::new_inner(config.clone()))), unlocked_inner: Arc::new(Self::new_unlocked_inner(config, network_manager)), }; // Set rolling transfers tick task @@ -217,6 +217,22 @@ impl RoutingTable { } } + pub fn with_route_spec_store_mut(&self, f: F) -> R + where + F: FnOnce(&mut RouteSpecStore) -> R, + { + let inner = self.inner.write(); + f(&mut inner.route_spec_store) + } + + pub fn with_route_spec_store(&self, f: F) -> R + where + F: FnOnce(&RouteSpecStore) -> R, + { + let inner = self.inner.read(); + f(&inner.route_spec_store) + } + pub fn relay_node(&self, domain: RoutingDomain) -> Option { let inner = self.inner.read(); Self::with_routing_domain(&*inner, domain, |rd| rd.common().relay_node()) @@ -564,7 +580,7 @@ impl RoutingTable { error!("kick_buckets_task not stopped: {}", e); } - *self.inner.write() = Self::new_inner(); + *self.inner.write() = Self::new_inner(self.unlocked_inner.config.clone()); debug!("finished routing table terminate"); } diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 6ee67a36..a7b0d80e 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -2,6 +2,24 @@ use super::*; use crate::veilid_api::*; use serde::*; +/// Options for safety routes (sender privacy) +#[derive(Copy, Clone, Debug, Serialize, Deserialize)] +pub struct SafetySpec { + /// 0 = no safety route, just use node's node id, more hops is safer but slower + pub hop_count: usize, + /// prefer more reliable protocols and relays over faster ones + pub reliable: bool, +} + +/// Compiled route (safety route + private route) +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CompiledRoute { + /// The safety route attached to the private route + safety_route: SafetyRoute, + /// The secret used to encrypt the message payload + secret: DHTKeySecret, +} + #[derive(Clone, Debug, Serialize, Deserialize)] struct RouteSpecDetail { /// Secret key @@ -30,6 +48,8 @@ struct RouteSpecDetail { created_ts: u64, /// Timestamp of when the route was last checked for validity last_checked_ts: Option, + /// Timestamp of when the route was last used for anything + last_used_ts: Option, /// Directions this route is guaranteed to work in directions: DirectionSet, /// Reliability @@ -56,10 +76,14 @@ pub struct RouteSpecStoreCache { #[derive(Debug)] pub struct RouteSpecStore { + /// Maximum number of hops in a route + max_route_hop_count: usize, + /// Default number of hops in a route + default_route_hop_count: usize, /// Serialize RouteSpecStore content content: RouteSpecStoreContent, /// RouteSpecStore cache - cache: RouteSpecStoreCache, + cache: Mutex, } fn route_hops_to_hop_cache(hops: &[DHTKey]) -> Vec { @@ -145,8 +169,17 @@ where } impl RouteSpecStore { - pub fn new() -> Self { + pub fn new(config: VeilidConfig) -> Self { + let (max_route_hop_count, default_route_hop_count) = { + let c = config.get(); + let max_route_hop_count = c.network.rpc.max_route_hop_count; + let default_route_hop_count = c.network.rpc.max_route_hop_count; + (max_route_hop_count.into(), default_route_hop_count.into()) + }; + Self { + max_route_hop_count, + default_route_hop_count, content: RouteSpecStoreContent { details: HashMap::new(), }, @@ -155,11 +188,20 @@ impl RouteSpecStore { } pub async fn load(routing_table: RoutingTable) -> EyreResult { + let (max_route_hop_count, default_route_hop_count) = { + let c = routing_table.unlocked_inner.config.get(); + let max_route_hop_count = c.network.rpc.max_route_hop_count; + let default_route_hop_count = c.network.rpc.max_route_hop_count; + (max_route_hop_count.into(), default_route_hop_count.into()) + }; + // Get cbor blob from table store let table_store = routing_table.network_manager().table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; let content = rsstdb.load_cbor(0, b"content").await?.unwrap_or_default(); let mut rss = RouteSpecStore { + max_route_hop_count, + default_route_hop_count, content, cache: Default::default(), }; @@ -259,18 +301,11 @@ impl RouteSpecStore { ) -> EyreResult> { use core::cmp::Ordering; - let max_route_hop_count = { - let config = routing_table.network_manager().config(); - let c = config.get(); - let max_route_hop_count = c.network.rpc.max_route_hop_count; - max_route_hop_count.into() - }; - - if hop_count < 2 { - bail!("Not allocating route less than two hops in length"); + if hop_count < 1 { + bail!("Not allocating route less than one hop in length"); } - if hop_count > max_route_hop_count { + if hop_count > self.max_route_hop_count { bail!("Not allocating route longer than max route hop count"); } @@ -497,6 +532,7 @@ impl RouteSpecStore { published: false, created_ts: cur_ts, last_checked_ts: None, + last_used_ts: None, directions, reliable, }; @@ -568,8 +604,117 @@ impl RouteSpecStore { None } - /// xxx add route compiler here - /// + ////////////////////////////////////////////////////////////////////// + + /// Compiles a safety route to the private route, with caching + pub fn compile_safety_route( + &self, + safety_spec: SafetySpec, + private_route: PrivateRoute, + ) -> Result { + // xxx implement caching first! + // xxx implement, ensure we handle hops == 0 for our safetyspec + + // Ensure the total hop count isn't too long for our config + let pr_hopcount = private_route.hop_count as usize; + let sr_hopcount = safety_route_spec.hops.len(); + let hopcount = 1 + sr_hopcount + pr_hopcount; + if hopcount > self.max_route_hop_count { + return Err(RPCError::internal("hop count too long for route")); + } + + // Create hops + let hops = if sr_hopcount == 0 { + SafetyRouteHops::Private(private_route) + } else { + // start last blob-to-encrypt data off as private route + let mut blob_data = { + let mut pr_message = ::capnp::message::Builder::new_default(); + let mut pr_builder = pr_message.init_root::(); + encode_private_route(&private_route, &mut pr_builder)?; + let mut blob_data = builder_to_vec(pr_message)?; + + // append the private route tag so we know how to decode it later + blob_data.push(1u8); + blob_data + }; + + // Encode each hop from inside to outside + // skips the outermost hop since that's entering the + // safety route and does not include the dialInfo + // (outer hop is a RouteHopData, not a RouteHop). + // Each loop mutates 'nonce', and 'blob_data' + let mut nonce = Crypto::get_random_nonce(); + for h in (1..sr_hopcount).rev() { + // Get blob to encrypt for next hop + blob_data = { + // Encrypt the previous blob ENC(nonce, DH(PKhop,SKsr)) + let dh_secret = self + .crypto + .cached_dh( + &safety_route_spec.hops[h].dial_info.node_id.key, + &safety_route_spec.secret_key, + ) + .map_err(RPCError::map_internal("dh failed"))?; + let enc_msg_data = + Crypto::encrypt_aead(blob_data.as_slice(), &nonce, &dh_secret, None) + .map_err(RPCError::map_internal("encryption failed"))?; + + // Make route hop data + let route_hop_data = RouteHopData { + nonce, + blob: enc_msg_data, + }; + + // Make route hop + let route_hop = RouteHop { + dial_info: safety_route_spec.hops[h].dial_info.clone(), + next_hop: Some(route_hop_data), + }; + + // Make next blob from route hop + let mut rh_message = ::capnp::message::Builder::new_default(); + let mut rh_builder = rh_message.init_root::(); + encode_route_hop(&route_hop, &mut rh_builder)?; + let mut blob_data = builder_to_vec(rh_message)?; + + // Append the route hop tag so we know how to decode it later + blob_data.push(0u8); + blob_data + }; + + // Make another nonce for the next hop + nonce = Crypto::get_random_nonce(); + } + + // Encode first RouteHopData + let dh_secret = self + .crypto + .cached_dh( + &safety_route_spec.hops[0].dial_info.node_id.key, + &safety_route_spec.secret_key, + ) + .map_err(RPCError::map_internal("dh failed"))?; + let enc_msg_data = Crypto::encrypt_aead(blob_data.as_slice(), &nonce, &dh_secret, None) + .map_err(RPCError::map_internal("encryption failed"))?; + + let route_hop_data = RouteHopData { + nonce, + blob: enc_msg_data, + }; + + SafetyRouteHops::Data(route_hop_data) + }; + + // Build safety route + let safety_route = SafetyRoute { + public_key: safety_route_spec.public_key, + hop_count: safety_route_spec.hops.len() as u8, + hops, + }; + + Ok(safety_route) + } /// Mark route as published /// When first deserialized, routes must be re-published in order to ensure they remain @@ -583,6 +728,11 @@ impl RouteSpecStore { self.detail_mut(&key).last_checked_ts = Some(cur_ts); } + /// Mark route as used + pub fn touch_route_used(&mut self, key: &DHTKey, cur_ts: u64) { + self.detail_mut(&key).last_used_ts = Some(cur_ts); + } + /// Record latency on the route pub fn record_latency(&mut self, key: &DHTKey, latency: u64) { let lsa = &mut self.detail_mut(&key).latency_stats_accounting; diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index 3510f826..872bebd2 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -8,6 +8,7 @@ pub enum Destination { /// The node to send to target: NodeRef, /// Require safety route or not + xxx convert back to safety spec, bubble up to api safety: bool, }, /// Send to node for relay purposes diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 2867d719..e8f8a221 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -133,10 +133,10 @@ impl Answer { } struct RenderedOperation { - message: Vec, // The rendered operation bytes - node_id: DHTKey, // Node id we're sending to - node_ref: Option, // Node to send envelope to (may not be destination node id in case of relay) - hop_count: usize, // Total safety + private route hop count + message: Vec, // The rendered operation bytes + node_id: DHTKey, // Destination node id we're sending to + node_ref: NodeRef, // Node to send envelope to (may not be destination node id in case of relay) + hop_count: usize, // Total safety + private route hop count } ///////////////////////////////////////////////////////////////////// @@ -151,6 +151,7 @@ pub struct RPCProcessorUnlockedInner { queue_size: u32, concurrency: u32, max_route_hop_count: usize, + default_route_hop_count: usize, validate_dial_info_receipt_time_ms: u32, update_callback: UpdateCallback, waiting_rpc_table: OperationWaiter, @@ -187,21 +188,13 @@ impl RPCProcessor { let mut queue_size = c.network.rpc.queue_size; let mut timeout = ms_to_us(c.network.rpc.timeout_ms); let mut max_route_hop_count = c.network.rpc.max_route_hop_count as usize; + let mut default_route_hop_count = c.network.rpc.default_route_hop_count as usize; if concurrency == 0 { concurrency = intf::get_concurrency() / 2; if concurrency == 0 { concurrency = 1; } } - if queue_size == 0 { - queue_size = 1024; - } - if timeout == 0 { - timeout = 10000000; - } - if max_route_hop_count == 0 { - max_route_hop_count = 7usize; - } let validate_dial_info_receipt_time_ms = c.network.dht.validate_dial_info_receipt_time_ms; RPCProcessorUnlockedInner { @@ -209,6 +202,7 @@ impl RPCProcessor { queue_size, concurrency, max_route_hop_count, + default_route_hop_count, validate_dial_info_receipt_time_ms, update_callback, waiting_rpc_table: OperationWaiter::new(), @@ -403,6 +397,65 @@ impl RPCProcessor { out } + // Wrap an operation with a private route inside a safety route + pub(super) fn wrap_with_route( + &self, + safety_spec: SafetySpec, + private_route: PrivateRoute, + message_data: Vec, + ) -> Result { + let compiled_route: CompiledRoute = self.routing_table().with_route_spec_store(|rss| { + // Compile the safety route with the private route + rss.compile_safety_route(safety_spec, private_route) + })?; + + // Verify hop count isn't larger than out maximum routed hop count + if compiled_route.safety_route.hop_count as usize > self.unlocked_inner.max_route_hop_count + { + return Err(RPCError::internal("hop count too long for route")) + .map_err(logthru_rpc!(warn)); + } + + // Encrypt routed operation + // Xmsg + ENC(Xmsg, DH(PKapr, SKbsr)) + let nonce = Crypto::get_random_nonce(); + let dh_secret = self + .crypto + .cached_dh(&private_route.public_key, &compiled_route.secret) + .map_err(RPCError::map_internal("dh failed"))?; + let enc_msg_data = Crypto::encrypt_aead(&message_data, &nonce, &dh_secret, None) + .map_err(RPCError::map_internal("encryption failed"))?; + + // Make the routed operation + let operation = RoutedOperation::new(nonce, enc_msg_data); + + // Prepare route operation + let route = RPCOperationRoute { + safety_route, + operation, + }; + let operation = + RPCOperation::new_statement(RPCStatement::new(RPCStatementDetail::Route(route)), None); + + // Convert message to bytes and return it + let mut route_msg = ::capnp::message::Builder::new_default(); + let mut route_operation = route_msg.init_root::(); + operation.encode(&mut route_operation)?; + let out = builder_to_vec(route_msg)?; + + // out_node_id = sr + // .hops + // .first() + // .ok_or_else(RPCError::else_internal("no hop in safety route"))? + // .dial_info + // .node_id + // .key; + + //out_hop_count = 1 + sr.hops.len(); + + Ok(out) + } + /// Produce a byte buffer that represents the wire encoding of the entire /// unencrypted envelope body for a RPC message. This incorporates /// wrapping a private and/or safety route if they are specified. @@ -412,14 +465,11 @@ impl RPCProcessor { dest: Destination, operation: &RPCOperation, ) -> Result { - let out_node_id; // Envelope Node Id - let mut out_node_ref: Option = None; // Node to send envelope to - let out_hop_count: usize; // Total safety + private route hop count - let out_message; // Envelope data + let out: RenderedOperation; // Encode message to a builder and make a message reader for it // Then produce the message as an unencrypted byte buffer - let message_vec = { + let message = { let mut msg_builder = ::capnp::message::Builder::new_default(); let mut op_builder = msg_builder.init_root::(); operation.encode(&mut op_builder)?; @@ -430,12 +480,12 @@ impl RPCProcessor { match dest { Destination::Direct { target: ref node_ref, - ref safety_route_spec, + safety, } | Destination::Relay { relay: ref node_ref, target: _, - ref safety_route_spec, + safety, } => { // Send to a node without a private route // -------------------------------------- @@ -444,7 +494,7 @@ impl RPCProcessor { let (node_ref, node_id) = if let Destination::Relay { relay: _, target: ref dht_key, - safety_route_spec: _, + safety: _, } = dest { (node_ref.clone(), dht_key.clone()) @@ -454,83 +504,40 @@ impl RPCProcessor { }; // Handle the existence of safety route - match safety_route_spec.as_ref() { - None => { + match safety { + false => { // If no safety route is being used, and we're not sending to a private // route, we can use a direct envelope instead of routing - out_message = message_vec; - - // Message goes directly to the node - out_node_id = node_id; - out_node_ref = Some(node_ref); - out_hop_count = 1; + out = RenderedOperation { + message, + node_id, + node_ref, + hop_count: 1, + }; } - Some(sr) => { + true => { // No private route was specified for the request // but we are using a safety route, so we must create an empty private route let private_route = PrivateRoute::new_stub(node_id); - // first - out_node_id = sr - .hops - .first() - .ok_or_else(RPCError::else_internal("no hop in safety route"))? - .dial_info - .node_id - .key; - out_message = - self.wrap_with_route(Some(sr.clone()), private_route, message_vec)?; - out_hop_count = 1 + sr.hops.len(); + // Wrap with safety route + out = self.wrap_with_route(true, private_route, message)?; } }; } Destination::PrivateRoute { private_route, - safety_route_spec, + safety, + reliable, } => { // Send to private route // --------------------- // Reply with 'route' operation - out_node_id = match safety_route_spec { - None => { - // If no safety route, the first node is the first hop of the private route - out_hop_count = private_route.hop_count as usize; - let out_node_id = match &private_route.hops { - Some(rh) => rh.dial_info.node_id.key, - _ => return Err(RPCError::internal("private route has no hops")), - }; - out_message = self.wrap_with_route(None, private_route, message_vec)?; - out_node_id - } - Some(sr) => { - // If safety route is in use, first node is the first hop of the safety route - out_hop_count = 1 + sr.hops.len() + (private_route.hop_count as usize); - let out_node_id = sr - .hops - .first() - .ok_or_else(RPCError::else_internal("no hop in safety route"))? - .dial_info - .node_id - .key; - out_message = self.wrap_with_route(Some(sr), private_route, message_vec)?; - out_node_id - } - } + out = self.wrap_with_route(safety, private_route, message)?; } } - // Verify hop count isn't larger than out maximum routed hop count - if out_hop_count > self.unlocked_inner.max_route_hop_count { - return Err(RPCError::internal("hop count too long for route")) - .map_err(logthru_rpc!(warn)); - } - - Ok(RenderedOperation { - message: out_message, - node_id: out_node_id, - node_ref: out_node_ref, - hop_count: out_hop_count, - }) + Ok(out) } // Get signed node info to package with RPC messages to improve diff --git a/veilid-core/src/rpc_processor/private_route.rs b/veilid-core/src/rpc_processor/private_route.rs deleted file mode 100644 index dbf11c9d..00000000 --- a/veilid-core/src/rpc_processor/private_route.rs +++ /dev/null @@ -1,154 +0,0 @@ -use super::*; - -impl RPCProcessor { - -xxx move this into route spec store - - ////////////////////////////////////////////////////////////////////// - fn compile_safety_route( - &self, - safety_route_spec: Arc, - private_route: PrivateRoute, - ) -> Result { - // Ensure the total hop count isn't too long for our config - let pr_hopcount = private_route.hop_count as usize; - let sr_hopcount = safety_route_spec.hops.len(); - let hopcount = 1 + sr_hopcount + pr_hopcount; - if hopcount > self.unlocked_inner.max_route_hop_count { - return Err(RPCError::internal("hop count too long for route")); - } - - // Create hops - let hops = if sr_hopcount == 0 { - SafetyRouteHops::Private(private_route) - } else { - // start last blob-to-encrypt data off as private route - let mut blob_data = { - let mut pr_message = ::capnp::message::Builder::new_default(); - let mut pr_builder = pr_message.init_root::(); - encode_private_route(&private_route, &mut pr_builder)?; - let mut blob_data = builder_to_vec(pr_message)?; - - // append the private route tag so we know how to decode it later - blob_data.push(1u8); - blob_data - }; - - // Encode each hop from inside to outside - // skips the outermost hop since that's entering the - // safety route and does not include the dialInfo - // (outer hop is a RouteHopData, not a RouteHop). - // Each loop mutates 'nonce', and 'blob_data' - let mut nonce = Crypto::get_random_nonce(); - for h in (1..sr_hopcount).rev() { - // Get blob to encrypt for next hop - blob_data = { - // Encrypt the previous blob ENC(nonce, DH(PKhop,SKsr)) - let dh_secret = self - .crypto - .cached_dh( - &safety_route_spec.hops[h].dial_info.node_id.key, - &safety_route_spec.secret_key, - ) - .map_err(RPCError::map_internal("dh failed"))?; - let enc_msg_data = - Crypto::encrypt_aead(blob_data.as_slice(), &nonce, &dh_secret, None) - .map_err(RPCError::map_internal("encryption failed"))?; - - // Make route hop data - let route_hop_data = RouteHopData { - nonce, - blob: enc_msg_data, - }; - - // Make route hop - let route_hop = RouteHop { - dial_info: safety_route_spec.hops[h].dial_info.clone(), - next_hop: Some(route_hop_data), - }; - - // Make next blob from route hop - let mut rh_message = ::capnp::message::Builder::new_default(); - let mut rh_builder = rh_message.init_root::(); - encode_route_hop(&route_hop, &mut rh_builder)?; - let mut blob_data = builder_to_vec(rh_message)?; - - // Append the route hop tag so we know how to decode it later - blob_data.push(0u8); - blob_data - }; - - // Make another nonce for the next hop - nonce = Crypto::get_random_nonce(); - } - - // Encode first RouteHopData - let dh_secret = self - .crypto - .cached_dh( - &safety_route_spec.hops[0].dial_info.node_id.key, - &safety_route_spec.secret_key, - ) - .map_err(RPCError::map_internal("dh failed"))?; - let enc_msg_data = Crypto::encrypt_aead(blob_data.as_slice(), &nonce, &dh_secret, None) - .map_err(RPCError::map_internal("encryption failed"))?; - - let route_hop_data = RouteHopData { - nonce, - blob: enc_msg_data, - }; - - SafetyRouteHops::Data(route_hop_data) - }; - - // Build safety route - let safety_route = SafetyRoute { - public_key: safety_route_spec.public_key, - hop_count: safety_route_spec.hops.len() as u8, - hops, - }; - - Ok(safety_route) - } - - // Wrap an operation inside a route - pub(super) fn wrap_with_route( - &self, - safety_route_spec: Option>, - private_route: PrivateRoute, - message_data: Vec, - ) -> Result, RPCError> { - // Encrypt routed operation - // Xmsg + ENC(Xmsg, DH(PKapr, SKbsr)) - let nonce = Crypto::get_random_nonce(); - let safety_route_spec = - safety_route_spec.unwrap_or_else(|| Arc::new(SafetyRouteSpec::new())); - let dh_secret = self - .crypto - .cached_dh(&private_route.public_key, &safety_route_spec.secret_key) - .map_err(RPCError::map_internal("dh failed"))?; - let enc_msg_data = Crypto::encrypt_aead(&message_data, &nonce, &dh_secret, None) - .map_err(RPCError::map_internal("encryption failed"))?; - - // Compile the safety route with the private route - let safety_route = self.compile_safety_route(safety_route_spec, private_route)?; - - // Make the routed operation - let operation = RoutedOperation::new(nonce, enc_msg_data); - - // Prepare route operation - let route = RPCOperationRoute { - safety_route, - operation, - }; - let operation = - RPCOperation::new_statement(RPCStatement::new(RPCStatementDetail::Route(route)), None); - - // Convert message to bytes and return it - let mut route_msg = ::capnp::message::Builder::new_default(); - let mut route_operation = route_msg.init_root::(); - operation.encode(&mut route_operation)?; - let out = builder_to_vec(route_msg)?; - Ok(out) - } -} diff --git a/veilid-core/src/tests/common/test_veilid_config.rs b/veilid-core/src/tests/common/test_veilid_config.rs index a84d7d11..c9f52197 100644 --- a/veilid-core/src/tests/common/test_veilid_config.rs +++ b/veilid-core/src/tests/common/test_veilid_config.rs @@ -208,6 +208,7 @@ fn config_callback(key: String) -> ConfigCallbackReturn { "network.rpc.max_timestamp_ahead_ms" => Ok(Box::new(Some(10_000u32))), "network.rpc.timeout_ms" => Ok(Box::new(10_000u32)), "network.rpc.max_route_hop_count" => Ok(Box::new(7u8)), + "network.rpc.default_route_hop_count" => Ok(Box::new(2u8)), "network.dht.resolve_node_timeout_ms" => Ok(Box::new(Option::::None)), "network.dht.resolve_node_count" => Ok(Box::new(20u32)), "network.dht.resolve_node_fanout" => Ok(Box::new(3u32)), @@ -325,6 +326,7 @@ pub async fn test_config() { assert_eq!(inner.network.rpc.queue_size, 128u32); assert_eq!(inner.network.rpc.timeout_ms, 10_000u32); assert_eq!(inner.network.rpc.max_route_hop_count, 7u8); + assert_eq!(inner.network.rpc.default_route_hop_count, 2u8); assert_eq!(inner.network.routing_table.limit_over_attached, 64u32); assert_eq!(inner.network.routing_table.limit_fully_attached, 32u32); assert_eq!(inner.network.routing_table.limit_attached_strong, 16u32); diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index 21e1eb6e..ca576b5f 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -117,7 +117,7 @@ pub struct VeilidConfigWS { /// ```yaml /// wss: /// connect: true -/// listen: false +/// listen: false /// max_connections: 16 /// listen_address: ':5150' /// path: 'ws' @@ -192,6 +192,7 @@ pub struct VeilidConfigRPC { pub max_timestamp_ahead_ms: Option, pub timeout_ms: u32, pub max_route_hop_count: u8, + pub default_route_hop_count: u8, } /// Configure the network routing table @@ -334,7 +335,7 @@ pub struct VeilidConfigInner { /// The Veilid Configuration /// -/// Veilid is configured +/// Veilid is configured #[derive(Clone)] pub struct VeilidConfig { inner: Arc>, @@ -444,6 +445,7 @@ impl VeilidConfig { get_config!(inner.network.rpc.max_timestamp_ahead_ms); get_config!(inner.network.rpc.timeout_ms); get_config!(inner.network.rpc.max_route_hop_count); + get_config!(inner.network.rpc.default_route_hop_count); get_config!(inner.network.upnp); get_config!(inner.network.natpmp); get_config!(inner.network.detect_address_changes); @@ -634,6 +636,33 @@ impl VeilidConfig { ); } } + if inner.network.rpc.max_route_hop_count == 0 { + apibail_generic!( + "max route hop count must be >= 1 in 'network.rpc.max_route_hop_count'" + ); + } + if inner.network.rpc.max_route_hop_count > 7 { + apibail_generic!( + "max route hop count must be <= 7 in 'network.rpc.max_route_hop_count'" + ); + } + if inner.network.rpc.default_route_hop_count == 0 { + apibail_generic!( + "default route hop count must be >= 1 in 'network.rpc.default_route_hop_count'" + ); + } + if inner.network.rpc.default_route_hop_count > inner.network.rpc.max_route_hop_count { + apibail_generic!( + "default route hop count must be <= max route hop count in 'network.rpc.default_route_hop_count <= network.rpc.max_route_hop_count'" + ); + } + if inner.network.rpc.queue_size < 256 { + apibail_generic!("rpc queue size must be >= 256 in 'network.rpc.queue_size'"); + } + if inner.network.rpc.timeout_ms < 1000 { + apibail_generic!("rpc timeout must be >= 1000 in 'network.rpc.timeout_ms'"); + } + Ok(()) } diff --git a/veilid-flutter/example/lib/config.dart b/veilid-flutter/example/lib/config.dart index 5f8b4e6d..3f224264 100644 --- a/veilid-flutter/example/lib/config.dart +++ b/veilid-flutter/example/lib/config.dart @@ -66,6 +66,7 @@ Future getDefaultVeilidConfig() async { maxTimestampAheadMs: 10000, timeoutMs: 10000, maxRouteHopCount: 7, + defaultRouteHopCount: 2, ), dht: VeilidConfigDHT( resolveNodeTimeoutMs: null, diff --git a/veilid-flutter/lib/veilid.dart b/veilid-flutter/lib/veilid.dart index 843e2938..70f41a68 100644 --- a/veilid-flutter/lib/veilid.dart +++ b/veilid-flutter/lib/veilid.dart @@ -657,6 +657,7 @@ class VeilidConfigRPC { int? maxTimestampAheadMs; int timeoutMs; int maxRouteHopCount; + int defaultRouteHopCount; VeilidConfigRPC( {required this.concurrency, @@ -664,7 +665,8 @@ class VeilidConfigRPC { this.maxTimestampBehindMs, this.maxTimestampAheadMs, required this.timeoutMs, - required this.maxRouteHopCount}); + required this.maxRouteHopCount, + required this.defaultRouteHopCount}); Map get json { return { @@ -674,6 +676,7 @@ class VeilidConfigRPC { 'max_timestamp_ahead_ms': maxTimestampAheadMs, 'timeout_ms': timeoutMs, 'max_route_hop_count': maxRouteHopCount, + 'default_route_hop_count': defaultRouteHopCount, }; } @@ -683,7 +686,8 @@ class VeilidConfigRPC { maxTimestampBehindMs = json['max_timestamp_behind_ms'], maxTimestampAheadMs = json['max_timestamp_ahead_ms'], timeoutMs = json['timeout_ms'], - maxRouteHopCount = json['max_route_hop_count']; + maxRouteHopCount = json['max_route_hop_count'], + defaultRouteHopCount = json['default_route_hop_count']; } //////////// diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index 582ca4fd..f40dd2ea 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -82,6 +82,7 @@ core: max_timestamp_ahead_ms: 10000 timeout_ms: 10000 max_route_hop_count: 7 + default_route_hop_count: 2 dht: resolve_node_timeout: resolve_node_count: 20 @@ -539,6 +540,7 @@ pub struct Rpc { pub max_timestamp_ahead_ms: Option, pub timeout_ms: u32, pub max_route_hop_count: u8, + pub default_route_hop_count: u8, } #[derive(Debug, Deserialize, Serialize)] @@ -756,7 +758,7 @@ impl Settings { /// in /etc/veilid-server. If a config is not found in this location, it will /// follow the XDG user directory spec, and look in `~/.config/veilid-server/`. /// - /// For Windows, a user-local config may be created at + /// For Windows, a user-local config may be created at /// `C:\Users\\AppData\Roaming\Veilid\Veilid`, and for macOS, at /// `/Users//Library/Application Support/org.Veilid.Veilid` /// @@ -965,6 +967,7 @@ impl Settings { set_config_value!(inner.core.network.rpc.max_timestamp_ahead_ms, value); set_config_value!(inner.core.network.rpc.timeout_ms, value); set_config_value!(inner.core.network.rpc.max_route_hop_count, value); + set_config_value!(inner.core.network.rpc.default_route_hop_count, value); set_config_value!(inner.core.network.dht.resolve_node_timeout_ms, value); set_config_value!(inner.core.network.dht.resolve_node_count, value); set_config_value!(inner.core.network.dht.resolve_node_fanout, value); @@ -1142,6 +1145,9 @@ impl Settings { "network.rpc.max_route_hop_count" => { Ok(Box::new(inner.core.network.rpc.max_route_hop_count)) } + "network.rpc.default_route_hop_count" => { + Ok(Box::new(inner.core.network.rpc.default_route_hop_count)) + } "network.dht.resolve_node_timeout_ms" => { Ok(Box::new(inner.core.network.dht.resolve_node_timeout_ms)) } @@ -1486,6 +1492,7 @@ mod tests { assert_eq!(s.core.network.rpc.max_timestamp_ahead_ms, Some(10_000u32)); assert_eq!(s.core.network.rpc.timeout_ms, 10_000u32); assert_eq!(s.core.network.rpc.max_route_hop_count, 7); + assert_eq!(s.core.network.rpc.default_route_hop_count, 2); // assert_eq!(s.core.network.dht.resolve_node_timeout_ms, None); assert_eq!(s.core.network.dht.resolve_node_count, 20u32); diff --git a/veilid-wasm/tests/web.rs b/veilid-wasm/tests/web.rs index e98612a8..299f9acf 100644 --- a/veilid-wasm/tests/web.rs +++ b/veilid-wasm/tests/web.rs @@ -46,6 +46,7 @@ fn init_callbacks() { case "network.rpc.max_timestamp_ahead": return 10000000; case "network.rpc.timeout": return 10000000; case "network.rpc.max_route_hop_count": return 7; + case "network.rpc.default_route_hop_count": return 2; case "network.dht.resolve_node_timeout": return null; case "network.dht.resolve_node_count": return 20; case "network.dht.resolve_node_fanout": return 3; From 63768580c654f8df161b30aea7523f7f324ad3de Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 16 Oct 2022 19:59:59 -0400 Subject: [PATCH 16/67] checkpoint --- doc/config/sample.config | 2 +- doc/config/veilid-server-config.md | 2 +- veilid-core/src/routing_table/find_nodes.rs | 29 ++- veilid-core/src/routing_table/mod.rs | 92 ++++++--- .../src/routing_table/route_spec_store.rs | 174 ++++++++++++------ veilid-core/src/rpc_processor/destination.rs | 27 ++- veilid-core/src/rpc_processor/mod.rs | 40 ---- veilid-core/src/veilid_config.rs | 4 +- veilid-flutter/example/lib/config.dart | 2 +- veilid-wasm/tests/web.rs | 2 +- 10 files changed, 225 insertions(+), 149 deletions(-) diff --git a/doc/config/sample.config b/doc/config/sample.config index 1b169390..f4693374 100644 --- a/doc/config/sample.config +++ b/doc/config/sample.config @@ -63,7 +63,7 @@ core: max_timestamp_behind_ms: 10000 max_timestamp_ahead_ms: 10000 timeout_ms: 10000 - max_route_hop_count: 7 + max_route_hop_count: 5 default_route_hop_count: 2 dht: diff --git a/doc/config/veilid-server-config.md b/doc/config/veilid-server-config.md index 210b52b4..b7968693 100644 --- a/doc/config/veilid-server-config.md +++ b/doc/config/veilid-server-config.md @@ -228,7 +228,7 @@ rpc: max_timestamp_behind_ms: 10000 max_timestamp_ahead_ms: 10000 timeout_ms: 10000 - max_route_hop_count: 7 + max_route_hop_count: 4 default_route_hop_count: 2 ``` diff --git a/veilid-core/src/routing_table/find_nodes.rs b/veilid-core/src/routing_table/find_nodes.rs index 0e304d91..952c4b14 100644 --- a/veilid-core/src/routing_table/find_nodes.rs +++ b/veilid-core/src/routing_table/find_nodes.rs @@ -218,17 +218,36 @@ impl RoutingTable { ) -> core::cmp::Ordering, T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, { - let inner = self.inner.read(); - let inner = &*inner; - let self_node_id = self.unlocked_inner.node_id; + let inner = &*self.inner.read(); + Self::find_peers_with_sort_and_filter_inner( + inner, node_count, cur_ts, filter, compare, transform, + ) + } + pub fn find_peers_with_sort_and_filter_inner<'a, 'b, F, C, T, O>( + inner: &RoutingTableInner, + node_count: usize, + cur_ts: u64, + mut filter: F, + compare: C, + mut transform: T, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + C: FnMut( + &'a RoutingTableInner, + &'b (DHTKey, Option>), + &'b (DHTKey, Option>), + ) -> core::cmp::Ordering, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + { // collect all the nodes for sorting let mut nodes = Vec::<(DHTKey, Option>)>::with_capacity(inner.bucket_entry_count + 1); // add our own node (only one of there with the None entry) - if filter(inner, self_node_id, None) { - nodes.push((self_node_id, None)); + if filter(inner, inner.node_id, None) { + nodes.push((inner.node_id, None)); } // add all nodes from buckets diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index ab6fcc09..2bccb4fb 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -219,18 +219,18 @@ impl RoutingTable { pub fn with_route_spec_store_mut(&self, f: F) -> R where - F: FnOnce(&mut RouteSpecStore) -> R, + F: FnOnce(&mut RouteSpecStore, &mut RoutingTableInner) -> R, { - let inner = self.inner.write(); - f(&mut inner.route_spec_store) + let inner = &mut *self.inner.write(); + f(&mut inner.route_spec_store, inner) } pub fn with_route_spec_store(&self, f: F) -> R where - F: FnOnce(&RouteSpecStore) -> R, + F: FnOnce(&RouteSpecStore, &RoutingTableInner) -> R, { - let inner = self.inner.read(); - f(&inner.route_spec_store) + let inner = &*self.inner.read(); + f(&inner.route_spec_store, inner) } pub fn relay_node(&self, domain: RoutingDomain) -> Option { @@ -339,6 +339,30 @@ impl RoutingTable { true } + #[instrument(level = "trace", skip(inner), ret)] + fn get_contact_method_inner( + inner: &RoutingTableInner, + routing_domain: RoutingDomain, + node_a_id: &DHTKey, + node_a: &NodeInfo, + node_b_id: &DHTKey, + node_b: &NodeInfo, + dial_info_filter: DialInfoFilter, + reliable: bool, + ) -> ContactMethod { + Self::with_routing_domain(inner, routing_domain, |rdd| { + rdd.get_contact_method( + inner, + node_a_id, + node_a, + node_b_id, + node_b, + dial_info_filter, + reliable, + ) + }) + } + /// Look up the best way for two nodes to reach each other over a specific routing domain #[instrument(level = "trace", skip(self), ret)] pub fn get_contact_method( @@ -352,17 +376,16 @@ impl RoutingTable { reliable: bool, ) -> ContactMethod { let inner = &*self.inner.read(); - Self::with_routing_domain(inner, routing_domain, |rdd| { - rdd.get_contact_method( - inner, - node_a_id, - node_a, - node_b_id, - node_b, - dial_info_filter, - reliable, - ) - }) + Self::get_contact_method_inner( + inner, + routing_domain, + node_a_id, + node_a, + node_b_id, + node_b, + dial_info_filter, + reliable, + ) } // Figure out how to reach a node from our own node over the best routing domain and reference the nodes we want to access @@ -384,11 +407,11 @@ impl RoutingTable { }; // Node A is our own node - let node_a = self.get_own_node_info(routing_domain); + let node_a = get_own_node_info_inner(inner, routing_domain); let node_a_id = self.node_id(); // Node B is the target node - let node_b = target_node_ref.operate(|_rti, e| e.node_info(routing_domain).unwrap()); + let node_b = target_node_ref.xxx operate(|_rti, e| e.node_info(routing_domain).unwrap()); let node_b_id = target_node_ref.node_id(); // Dial info filter comes from the target node ref @@ -411,8 +434,7 @@ impl RoutingTable { ContactMethod::Existing => NodeContactMethod::Existing, ContactMethod::Direct(di) => NodeContactMethod::Direct(di), ContactMethod::SignalReverse(relay_key, target_key) => { - let relay_nr = self - .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + let relay_nr = Self::lookup_and_filter_noderef_inner(inner, self.clone(), relay_key, routing_domain.into(), dial_info_filter) .ok_or_else(|| eyre!("couldn't look up relay"))?; if target_node_ref.node_id() != target_key { bail!("target noderef didn't match target key"); @@ -420,8 +442,7 @@ impl RoutingTable { NodeContactMethod::SignalReverse(relay_nr, target_node_ref) } ContactMethod::SignalHolePunch(relay_key, target_key) => { - let relay_nr = self - .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + let relay_nr = Self::lookup_and_filter_noderef_inner(inner, self.clone(), relay_key, routing_domain.into(), dial_info_filter) .ok_or_else(|| eyre!("couldn't look up relay"))?; if target_node_ref.node_id() != target_key { bail!("target noderef didn't match target key"); @@ -429,14 +450,12 @@ impl RoutingTable { NodeContactMethod::SignalHolePunch(relay_nr, target_node_ref) } ContactMethod::InboundRelay(relay_key) => { - let relay_nr = self - .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + let relay_nr = Self::lookup_and_filter_noderef_nner(inner, self.clone(), relay_key, routing_domain.into(), dial_info_filter) .ok_or_else(|| eyre!("couldn't look up relay"))?; NodeContactMethod::InboundRelay(relay_nr) } ContactMethod::OutboundRelay(relay_key) => { - let relay_nr = self - .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + let relay_nr = Self::lookup_and_filter_noderef(inner, self.clone(), relay_key, routing_domain.into(), dial_info_filter) .ok_or_else(|| eyre!("couldn't look up relay"))?; NodeContactMethod::OutboundRelay(relay_nr) } @@ -486,13 +505,19 @@ impl RoutingTable { } /// Return a copy of our node's nodeinfo - pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo { - let inner = &*self.inner.read(); + fn get_own_node_info_inner( + inner: &RoutingTableInner, + routing_domain: RoutingDomain, + ) -> NodeInfo { Self::with_routing_domain(inner, routing_domain, |rdd| { rdd.common() .with_peer_info(|pi| pi.signed_node_info.node_info.clone()) }) } + pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo { + let inner = &*self.inner.read(); + Self::get_own_node_info_inner(inner, routing_domain) + } /// Return our currently registered network class pub fn has_valid_own_node_info(&self, routing_domain: RoutingDomain) -> bool { @@ -880,6 +905,15 @@ impl RoutingTable { } /// Resolve an existing routing table entry and return a reference to it + fn lookup_node_ref_inner(inner: &RoutingTableInner, routing_table: RoutingTable, node_id: DHTKey) -> Option { + { + let idx = routing_table.find_bucket_index(node_id); + let bucket = &inner.buckets[idx]; + bucket + .entry(&node_id) + .map(|e| NodeRef::new(routing_table, node_id, e, None)) + } + pub fn lookup_node_ref(&self, node_id: DHTKey) -> Option { if node_id == self.unlocked_inner.node_id { log_rtab!(debug "can't look up own node id in routing table"); diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index a7b0d80e..8228acb9 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -5,6 +5,8 @@ use serde::*; /// Options for safety routes (sender privacy) #[derive(Copy, Clone, Debug, Serialize, Deserialize)] pub struct SafetySpec { + /// preferred safety route if it still exists + pub preferred_route: Option, /// 0 = no safety route, just use node's node id, more hops is safer but slower pub hop_count: usize, /// prefer more reliable protocols and relays over faster ones @@ -76,6 +78,10 @@ pub struct RouteSpecStoreCache { #[derive(Debug)] pub struct RouteSpecStore { + /// Our node id + node_id: DHTKey, + /// Our node id secret + node_id_secret: DHTKeySecret, /// Maximum number of hops in a route max_route_hop_count: usize, /// Default number of hops in a route @@ -83,7 +89,7 @@ pub struct RouteSpecStore { /// Serialize RouteSpecStore content content: RouteSpecStoreContent, /// RouteSpecStore cache - cache: Mutex, + cache: RouteSpecStoreCache, } fn route_hops_to_hop_cache(hops: &[DHTKey]) -> Vec { @@ -170,16 +176,13 @@ where impl RouteSpecStore { pub fn new(config: VeilidConfig) -> Self { - let (max_route_hop_count, default_route_hop_count) = { - let c = config.get(); - let max_route_hop_count = c.network.rpc.max_route_hop_count; - let default_route_hop_count = c.network.rpc.max_route_hop_count; - (max_route_hop_count.into(), default_route_hop_count.into()) - }; + let c = config.get(); Self { - max_route_hop_count, - default_route_hop_count, + node_id: c.network.node_id, + node_id_secret: c.network.node_id_secret, + max_route_hop_count: c.network.rpc.max_route_hop_count.into(), + default_route_hop_count: c.network.rpc.default_route_hop_count.into(), content: RouteSpecStoreContent { details: HashMap::new(), }, @@ -188,20 +191,17 @@ impl RouteSpecStore { } pub async fn load(routing_table: RoutingTable) -> EyreResult { - let (max_route_hop_count, default_route_hop_count) = { - let c = routing_table.unlocked_inner.config.get(); - let max_route_hop_count = c.network.rpc.max_route_hop_count; - let default_route_hop_count = c.network.rpc.max_route_hop_count; - (max_route_hop_count.into(), default_route_hop_count.into()) - }; - + let config = routing_table.network_manager().config(); + let c = config.get(); // Get cbor blob from table store let table_store = routing_table.network_manager().table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; let content = rsstdb.load_cbor(0, b"content").await?.unwrap_or_default(); let mut rss = RouteSpecStore { - max_route_hop_count, - default_route_hop_count, + node_id: c.network.node_id, + node_id_secret: c.network.node_id_secret, + max_route_hop_count: c.network.rpc.max_route_hop_count.into(), + default_route_hop_count: c.network.rpc.default_route_hop_count.into(), content, cache: Default::default(), }; @@ -284,17 +284,17 @@ impl RouteSpecStore { } } - fn detail_mut(&mut self, public_key: &DHTKey) -> &mut RouteSpecDetail { - self.content.details.get_mut(&public_key).unwrap() + fn detail_mut(&mut self, public_key: &DHTKey) -> Option<&mut RouteSpecDetail> { + self.content.details.get_mut(&public_key) } /// Create a new route /// Prefers nodes that are not currently in use by another route /// The route is not yet tested for its reachability /// Returns None if no route could be allocated at this time - pub async fn allocate_route( + pub fn allocate_route( &mut self, - routing_table: RoutingTable, + rti: &RoutingTableInner, reliable: bool, hop_count: usize, directions: DirectionSet, @@ -309,10 +309,6 @@ impl RouteSpecStore { bail!("Not allocating route longer than max route hop count"); } - // Lock routing table for reading, make sure things don't change - // because we want to iterate the table without changes being made to it - let rti = routing_table.inner.read(); - // Get list of all nodes, and sort them for selection let cur_ts = intf::get_timestamp(); let dial_info_sort = if reliable { @@ -417,12 +413,20 @@ impl RouteSpecStore { }; // Pull the whole routing table in sorted order - let node_count = routing_table.get_entry_count( + let node_count = RoutingTable::get_entry_count_inner( + rti, RoutingDomain::PublicInternet.into(), BucketEntryState::Unreliable, ); - let nodes = routing_table - .find_peers_with_sort_and_filter(node_count, cur_ts, filter, compare, transform); + let nodes = RoutingTable::find_peers_with_sort_and_filter_inner( + rti, + self.node_id, + node_count, + cur_ts, + filter, + compare, + transform, + ); // If we couldn't find enough nodes, wait until we have more nodes in the routing table if nodes.len() < hop_count { @@ -447,13 +451,14 @@ impl RouteSpecStore { // Ensure this route is viable by checking that each node can contact the next one if directions.contains(Direction::Outbound) { let our_node_info = - routing_table.get_own_node_info(RoutingDomain::PublicInternet); - let our_node_id = routing_table.node_id(); + RoutingTable::get_own_node_info_inner(rti, RoutingDomain::PublicInternet); + let our_node_id = self.node_id; let mut previous_node = &(our_node_id, our_node_info); let mut reachable = true; for n in permutation { let current_node = nodes.get(*n).unwrap(); - let cm = routing_table.get_contact_method( + let cm = RoutingTable::get_contact_method_inner( + rti, RoutingDomain::PublicInternet, &previous_node.0, &previous_node.1, @@ -474,13 +479,14 @@ impl RouteSpecStore { } if directions.contains(Direction::Inbound) { let our_node_info = - routing_table.get_own_node_info(RoutingDomain::PublicInternet); - let our_node_id = routing_table.node_id(); + RoutingTable::get_own_node_info_inner(rti, RoutingDomain::PublicInternet); + let our_node_id = self.node_id; let mut next_node = &(our_node_id, our_node_info); let mut reachable = true; for n in permutation.iter().rev() { let current_node = nodes.get(*n).unwrap(); - let cm = routing_table.get_contact_method( + let cm = RoutingTable::get_contact_method_inner( + rti, RoutingDomain::PublicInternet, &next_node.0, &next_node.1, @@ -608,20 +614,53 @@ impl RouteSpecStore { /// Compiles a safety route to the private route, with caching pub fn compile_safety_route( - &self, + &mut self, + rti: &RoutingTableInner, safety_spec: SafetySpec, private_route: PrivateRoute, ) -> Result { + let pr_hopcount = private_route.hop_count as usize; + if pr_hopcount > self.max_route_hop_count { + return Err(RPCError::internal("private route hop count too long")); + } + + // See if the preferred route is here + let opt_safety_rsd: Option<&mut RouteSpecDetail> = + if let Some(preferred_route) = safety_spec.preferred_route { + self.detail_mut(&preferred_route) + } else { + // Preferred safety route was not requested + None + }; + let safety_rsd: &mut RouteSpecDetail = if let Some(safety_rsd) = opt_safety_rsd { + // Safety route exists + safety_rsd + } else { + // Select a safety route from the pool or make one if we don't have one that matches + if let Some(sr_pubkey) = self.first_unpublished_route( + safety_spec.reliable, + safety_spec.hop_count, + safety_spec.hop_count, + Direction::Outbound.into(), + ) { + // Found a route to use + self.detail_mut(&sr_pubkey).unwrap() + } else { + // No route found, gotta allocate one + self.allocate_route(rti) + } + }; + // xxx implement caching first! + // xxx implement, ensure we handle hops == 0 for our safetyspec // Ensure the total hop count isn't too long for our config - let pr_hopcount = private_route.hop_count as usize; - let sr_hopcount = safety_route_spec.hops.len(); - let hopcount = 1 + sr_hopcount + pr_hopcount; - if hopcount > self.max_route_hop_count { - return Err(RPCError::internal("hop count too long for route")); + let sr_hopcount = safety_spec.hop_count; + if sr_hopcount > self.max_route_hop_count { + return Err(RPCError::internal("private route hop count too long")); } + let total_hopcount = sr_hopcount + pr_hopcount; // Create hops let hops = if sr_hopcount == 0 { @@ -719,41 +758,66 @@ impl RouteSpecStore { /// Mark route as published /// When first deserialized, routes must be re-published in order to ensure they remain /// in the RouteSpecStore. - pub fn mark_route_published(&mut self, key: &DHTKey) { - self.detail_mut(&key).published = true; + pub fn mark_route_published(&mut self, key: &DHTKey) -> EyreResult<()> { + self.detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .published = true; + Ok(()) } /// Mark route as checked - pub fn touch_route_checked(&mut self, key: &DHTKey, cur_ts: u64) { - self.detail_mut(&key).last_checked_ts = Some(cur_ts); + pub fn touch_route_checked(&mut self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { + self.detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .last_checked_ts = Some(cur_ts); + Ok(()) } /// Mark route as used - pub fn touch_route_used(&mut self, key: &DHTKey, cur_ts: u64) { - self.detail_mut(&key).last_used_ts = Some(cur_ts); + pub fn touch_route_used(&mut self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { + self.detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .last_used_ts = Some(cur_ts); + Ok(()) } /// Record latency on the route - pub fn record_latency(&mut self, key: &DHTKey, latency: u64) { - let lsa = &mut self.detail_mut(&key).latency_stats_accounting; + pub fn record_latency(&mut self, key: &DHTKey, latency: u64) -> EyreResult<()> { + let lsa = &mut self + .detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .latency_stats_accounting; self.detail_mut(&key).latency_stats = lsa.record_latency(latency); + Ok(()) } /// Get the calculated latency stats - pub fn latency_stats(&self, key: &DHTKey) -> LatencyStats { - self.detail_mut(&key).latency_stats.clone() + pub fn latency_stats(&mut self, key: &DHTKey) -> EyreResult { + Ok(self + .detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .latency_stats + .clone()) } /// Add download transfers to route - pub fn add_down(&mut self, key: &DHTKey, bytes: u64) { - let tsa = &mut self.detail_mut(&key).transfer_stats_accounting; + pub fn add_down(&mut self, key: &DHTKey, bytes: u64) -> EyreResult<()> { + let tsa = &mut self + .detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .transfer_stats_accounting; tsa.add_down(bytes); + Ok(()) } /// Add upload transfers to route - pub fn add_up(&mut self, key: &DHTKey, bytes: u64) { - let tsa = &mut self.detail_mut(&key).transfer_stats_accounting; + pub fn add_up(&mut self, key: &DHTKey, bytes: u64) -> EyreResult<()> { + let tsa = &mut self + .detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .transfer_stats_accounting; tsa.add_up(bytes); + Ok(()) } /// Process transfer statistics to get averages diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index 872bebd2..76d86990 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -8,8 +8,7 @@ pub enum Destination { /// The node to send to target: NodeRef, /// Require safety route or not - xxx convert back to safety spec, bubble up to api - safety: bool, + safety: Option, }, /// Send to node for relay purposes Relay { @@ -18,14 +17,14 @@ pub enum Destination { /// The final destination the relay should send to target: DHTKey, /// Require safety route or not - safety: bool, + safety: Option, }, /// Send to private route (privateroute) PrivateRoute { /// A private route to send to private_route: PrivateRoute, /// Require safety route or not - safety: bool, + safety: Option, /// Prefer reliability or not reliable: bool, }, @@ -35,29 +34,29 @@ impl Destination { pub fn direct(target: NodeRef) -> Self { Self::Direct { target, - safety: false, + safety: None, } } pub fn relay(relay: NodeRef, target: DHTKey) -> Self { Self::Relay { relay, target, - safety: false, + safety: None, } } pub fn private_route(private_route: PrivateRoute, reliable: bool) -> Self { Self::PrivateRoute { private_route, - safety: false, + safety: None, reliable, } } - pub fn with_safety(self) -> Self { + pub fn with_safety(self, spec: SafetySpec) -> Self { match self { Destination::Direct { target, safety: _ } => Self::Direct { target, - safety: true, + safety: Some(spec), }, Destination::Relay { relay, @@ -66,7 +65,7 @@ impl Destination { } => Self::Relay { relay, target, - safety: true, + safety: Some(spec), }, Destination::PrivateRoute { private_route, @@ -74,7 +73,7 @@ impl Destination { reliable, } => Self::PrivateRoute { private_route, - safety: true, + safety: Some(spec), reliable, }, } @@ -85,7 +84,7 @@ impl fmt::Display for Destination { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Destination::Direct { target, safety } => { - let sr = if *safety { "+SR" } else { "" }; + let sr = if safety.is_some() { "+SR" } else { "" }; write!(f, "{}{}", target, sr) } @@ -94,7 +93,7 @@ impl fmt::Display for Destination { target, safety, } => { - let sr = if *safety { "+SR" } else { "" }; + let sr = if safety.is_some() { "+SR" } else { "" }; write!(f, "{}@{}{}", target.encode(), relay, sr) } @@ -103,7 +102,7 @@ impl fmt::Display for Destination { safety, reliable, } => { - let sr = if *safety { "+SR" } else { "" }; + let sr = if safety.is_some() { "+SR" } else { "" }; let rl = if *reliable { "+RL" } else { "" }; write!(f, "{}{}{}", private_route, sr, rl) diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index e8f8a221..dab096a6 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -409,13 +409,6 @@ impl RPCProcessor { rss.compile_safety_route(safety_spec, private_route) })?; - // Verify hop count isn't larger than out maximum routed hop count - if compiled_route.safety_route.hop_count as usize > self.unlocked_inner.max_route_hop_count - { - return Err(RPCError::internal("hop count too long for route")) - .map_err(logthru_rpc!(warn)); - } - // Encrypt routed operation // Xmsg + ENC(Xmsg, DH(PKapr, SKbsr)) let nonce = Crypto::get_random_nonce(); @@ -613,17 +606,6 @@ impl RPCProcessor { hop_count, } = self.render_operation(dest, &operation)?; - // If we need to resolve the first hop, do it - let node_ref = match node_ref { - None => match self.resolve_node(node_id).await? { - None => { - return Ok(NetworkResult::no_connection_other(node_id)); - } - Some(nr) => nr, - }, - Some(nr) => nr, - }; - // Calculate answer timeout // Timeout is number of hops times the timeout per hop let timeout = self.unlocked_inner.timeout * (hop_count as u64); @@ -687,17 +669,6 @@ impl RPCProcessor { hop_count: _, } = self.render_operation(dest, &operation)?; - // If we need to resolve the first hop, do it - let node_ref = match node_ref { - None => match self.resolve_node(node_id).await? { - None => { - return Ok(NetworkResult::no_connection_other(node_id)); - } - Some(nr) => nr, - }, - Some(nr) => nr, - }; - // Send statement let bytes = message.len() as u64; let send_ts = intf::get_timestamp(); @@ -782,17 +753,6 @@ impl RPCProcessor { hop_count: _, } = self.render_operation(dest, &operation)?; - // If we need to resolve the first hop, do it - let node_ref = match node_ref { - None => match self.resolve_node(node_id).await? { - None => { - return Ok(NetworkResult::no_connection_other(node_id)); - } - Some(nr) => nr, - }, - Some(nr) => nr, - }; - // Send the reply let bytes = message.len() as u64; let send_ts = intf::get_timestamp(); diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index ca576b5f..9fdb88a8 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -641,9 +641,9 @@ impl VeilidConfig { "max route hop count must be >= 1 in 'network.rpc.max_route_hop_count'" ); } - if inner.network.rpc.max_route_hop_count > 7 { + if inner.network.rpc.max_route_hop_count > 5 { apibail_generic!( - "max route hop count must be <= 7 in 'network.rpc.max_route_hop_count'" + "max route hop count must be <= 5 in 'network.rpc.max_route_hop_count'" ); } if inner.network.rpc.default_route_hop_count == 0 { diff --git a/veilid-flutter/example/lib/config.dart b/veilid-flutter/example/lib/config.dart index 3f224264..9eb209c3 100644 --- a/veilid-flutter/example/lib/config.dart +++ b/veilid-flutter/example/lib/config.dart @@ -65,7 +65,7 @@ Future getDefaultVeilidConfig() async { maxTimestampBehindMs: 10000, maxTimestampAheadMs: 10000, timeoutMs: 10000, - maxRouteHopCount: 7, + maxRouteHopCount: 4, defaultRouteHopCount: 2, ), dht: VeilidConfigDHT( diff --git a/veilid-wasm/tests/web.rs b/veilid-wasm/tests/web.rs index 299f9acf..230f0609 100644 --- a/veilid-wasm/tests/web.rs +++ b/veilid-wasm/tests/web.rs @@ -45,7 +45,7 @@ fn init_callbacks() { case "network.rpc.max_timestamp_behind": return 10000000; case "network.rpc.max_timestamp_ahead": return 10000000; case "network.rpc.timeout": return 10000000; - case "network.rpc.max_route_hop_count": return 7; + case "network.rpc.max_route_hop_count": return 4; case "network.rpc.default_route_hop_count": return 2; case "network.dht.resolve_node_timeout": return null; case "network.dht.resolve_node_count": return 20; From 6d5df71ac1e195f172a621acf32bc8f7c05d4ff0 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 18 Oct 2022 21:53:45 -0400 Subject: [PATCH 17/67] routing table refactor --- veilid-core/src/network_manager/mod.rs | 141 +- veilid-core/src/routing_table/bucket_entry.rs | 17 +- veilid-core/src/routing_table/find_nodes.rs | 647 --------- veilid-core/src/routing_table/mod.rs | 1287 +++++++---------- veilid-core/src/routing_table/node_ref.rs | 14 +- .../src/routing_table/route_spec_store.rs | 67 +- .../routing_table/routing_domain_editor.rs | 7 +- .../src/routing_table/routing_table_inner.rs | 1033 +++++++++++++ veilid-core/src/routing_table/tasks.rs | 2 +- .../coders/private_safety_route.rs | 158 +- veilid-core/src/rpc_processor/destination.rs | 44 +- veilid-core/src/rpc_processor/mod.rs | 6 +- .../src/rpc_processor/rpc_find_node.rs | 9 +- veilid-core/src/veilid_api/debug.rs | 2 +- veilid-core/src/veilid_api/privacy.rs | 4 +- veilid-core/src/veilid_api/routing_context.rs | 48 +- veilid-server/src/settings.rs | 4 +- 17 files changed, 1904 insertions(+), 1586 deletions(-) delete mode 100644 veilid-core/src/routing_table/find_nodes.rs create mode 100644 veilid-core/src/routing_table/routing_table_inner.rs diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 00d8ac63..6bca250c 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -644,7 +644,7 @@ impl NetworkManager { Ok(()) } - // Get our node's capabilities + /// Get our node's capabilities in the PublicInternet routing domain fn generate_public_internet_node_status(&self) -> PublicInternetNodeStatus { let node_info = self .routing_table() @@ -664,6 +664,7 @@ impl NetworkManager { will_validate_dial_info, } } + /// Get our node's capabilities in the LocalNetwork routing domain fn generate_local_network_node_status(&self) -> LocalNetworkNodeStatus { let node_info = self .routing_table() @@ -689,7 +690,7 @@ impl NetworkManager { } } - // Generates a multi-shot/normal receipt + /// Generates a multi-shot/normal receipt #[instrument(level = "trace", skip(self, extra_data, callback), err)] pub fn generate_receipt>( &self, @@ -715,7 +716,7 @@ impl NetworkManager { Ok(out) } - // Generates a single-shot/normal receipt + /// Generates a single-shot/normal receipt #[instrument(level = "trace", skip(self, extra_data), err)] pub fn generate_single_shot_receipt>( &self, @@ -741,7 +742,7 @@ impl NetworkManager { Ok((out, instance)) } - // Process a received out-of-band receipt + /// Process a received out-of-band receipt #[instrument(level = "trace", skip(self, receipt_data), ret)] pub async fn handle_out_of_band_receipt>( &self, @@ -759,7 +760,7 @@ impl NetworkManager { receipt_manager.handle_receipt(receipt, None).await } - // Process a received in-band receipt + /// Process a received in-band receipt #[instrument(level = "trace", skip(self, receipt_data), ret)] pub async fn handle_in_band_receipt>( &self, @@ -871,7 +872,7 @@ impl NetworkManager { } } - // Builds an envelope for sending over the network + /// Builds an envelope for sending over the network #[instrument(level = "trace", skip(self, body), err)] fn build_envelope>( &self, @@ -895,10 +896,10 @@ impl NetworkManager { .wrap_err("envelope failed to encode") } - // Called by the RPC handler when we want to issue an RPC request or response - // node_ref is the direct destination to which the envelope will be sent - // If 'node_id' is specified, it can be different than node_ref.node_id() - // which will cause the envelope to be relayed + /// Called by the RPC handler when we want to issue an RPC request or response + /// node_ref is the direct destination to which the envelope will be sent + /// If 'node_id' is specified, it can be different than node_ref.node_id() + /// which will cause the envelope to be relayed #[instrument(level = "trace", skip(self, body), ret, err)] pub async fn send_envelope>( &self, @@ -942,7 +943,7 @@ impl NetworkManager { self.send_data(node_ref.clone(), out).await } - // Called by the RPC handler when we want to issue an direct receipt + /// Called by the RPC handler when we want to issue an direct receipt #[instrument(level = "trace", skip(self, rcpt_data), err)] pub async fn send_out_of_band_receipt( &self, @@ -967,9 +968,9 @@ impl NetworkManager { Ok(()) } - // Send a reverse connection signal and wait for the return receipt over it - // Then send the data across the new connection - // Only usable for PublicInternet routing domain + /// Send a reverse connection signal and wait for the return receipt over it + /// Then send the data across the new connection + /// Only usable for PublicInternet routing domain #[instrument(level = "trace", skip(self, data), err)] pub async fn do_reverse_connect( &self, @@ -1041,9 +1042,9 @@ impl NetworkManager { } } - // Send a hole punch signal and do a negotiating ping and wait for the return receipt - // Then send the data across the new connection - // Only usable for PublicInternet routing domain + /// Send a hole punch signal and do a negotiating ping and wait for the return receipt + /// Then send the data across the new connection + /// Only usable for PublicInternet routing domain #[instrument(level = "trace", skip(self, data), err)] pub async fn do_hole_punch( &self, @@ -1146,15 +1147,99 @@ impl NetworkManager { } } - // Send raw data to a node - // - // We may not have dial info for a node, but have an existing connection for it - // because an inbound connection happened first, and no FindNodeQ has happened to that - // node yet to discover its dial info. The existing connection should be tried first - // in this case. - // - // Sending to a node requires determining a NetworkClass compatible mechanism - // + /// Figure out how to reach a node from our own node over the best routing domain and reference the nodes we want to access + /// Uses NodeRefs to ensure nodes are referenced, this is not a part of 'RoutingTable' because RoutingTable is not + /// allowed to use NodeRefs due to recursive locking + #[instrument(level = "trace", skip(self), ret)] + pub(crate) fn get_node_contact_method( + &self, + target_node_ref: NodeRef, + ) -> EyreResult { + let routing_table = self.routing_table(); + + // Figure out the best routing domain to get the contact method over + let routing_domain = match target_node_ref.best_routing_domain() { + Some(rd) => rd, + None => { + log_net!("no routing domain for node {:?}", target_node_ref); + return Ok(NodeContactMethod::Unreachable); + } + }; + + // Node A is our own node + let node_a = routing_table.get_own_node_info(routing_domain); + let node_a_id = routing_table.node_id(); + + // Node B is the target node + let node_b = match target_node_ref.node_info(routing_domain) { + Some(ni) => ni, + None => { + log_net!("no node info for node {:?}", target_node_ref); + return Ok(NodeContactMethod::Unreachable); + } + }; + let node_b_id = target_node_ref.node_id(); + + // Dial info filter comes from the target node ref + let dial_info_filter = target_node_ref.dial_info_filter(); + let reliable = target_node_ref.reliable(); + + let cm = routing_table.get_contact_method( + routing_domain, + &node_a_id, + &node_a, + &node_b_id, + &node_b, + dial_info_filter, + reliable, + ); + + // Translate the raw contact method to a referenced contact method + Ok(match cm { + ContactMethod::Unreachable => NodeContactMethod::Unreachable, + ContactMethod::Existing => NodeContactMethod::Existing, + ContactMethod::Direct(di) => NodeContactMethod::Direct(di), + ContactMethod::SignalReverse(relay_key, target_key) => { + let relay_nr = routing_table + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .ok_or_else(|| eyre!("couldn't look up relay"))?; + if target_node_ref.node_id() != target_key { + bail!("target noderef didn't match target key"); + } + NodeContactMethod::SignalReverse(relay_nr, target_node_ref) + } + ContactMethod::SignalHolePunch(relay_key, target_key) => { + let relay_nr = routing_table + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .ok_or_else(|| eyre!("couldn't look up relay"))?; + if target_node_ref.node_id() != target_key { + bail!("target noderef didn't match target key"); + } + NodeContactMethod::SignalHolePunch(relay_nr, target_node_ref) + } + ContactMethod::InboundRelay(relay_key) => { + let relay_nr = routing_table + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .ok_or_else(|| eyre!("couldn't look up relay"))?; + NodeContactMethod::InboundRelay(relay_nr) + } + ContactMethod::OutboundRelay(relay_key) => { + let relay_nr = routing_table + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .ok_or_else(|| eyre!("couldn't look up relay"))?; + NodeContactMethod::OutboundRelay(relay_nr) + } + }) + } + + /// Send raw data to a node + /// + /// We may not have dial info for a node, but have an existing connection for it + /// because an inbound connection happened first, and no FindNodeQ has happened to that + /// node yet to discover its dial info. The existing connection should be tried first + /// in this case. + /// + /// Sending to a node requires determining a NetworkClass compatible mechanism pub fn send_data( &self, node_ref: NodeRef, @@ -1201,9 +1286,7 @@ impl NetworkManager { // info!("{}", "no existing connection".red()); // If we don't have last_connection, try to reach out to the peer via its dial info - let contact_method = this - .routing_table() - .get_node_contact_method(node_ref.clone())?; + let contact_method = this.get_node_contact_method(node_ref.clone())?; log_net!( "send_data via {:?} to dialinfo {:?}", contact_method, diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index cb60d8ec..fd563f7c 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -286,20 +286,10 @@ impl BucketEntryInner { self.last_connections.clear(); } - // Gets the 'last connection' that matches a specific connection key - // pub(super) fn last_connection( - // &self, - // protocol_type: ProtocolType, - // address_type: AddressType, - // ) -> Option<(ConnectionDescriptor, u64)> { - // let key = LastConnectionKey(protocol_type, address_type); - // self.last_connections.get(&key).cloned() - // } - // Gets all the 'last connections' that match a particular filter pub(super) fn last_connections( &self, - routing_table_inner: &RoutingTableInner, + rti: &RoutingTableInner, filter: Option, ) -> Vec<(ConnectionDescriptor, u64)> { let mut out: Vec<(ConnectionDescriptor, u64)> = self @@ -308,10 +298,7 @@ impl BucketEntryInner { .filter_map(|(k, v)| { let include = if let Some(filter) = &filter { let remote_address = v.0.remote_address().address(); - if let Some(routing_domain) = RoutingTable::routing_domain_for_address_inner( - routing_table_inner, - remote_address, - ) { + if let Some(routing_domain) = rti.routing_domain_for_address(remote_address) { if filter.routing_domain_set.contains(routing_domain) && filter.dial_info_filter.protocol_type_set.contains(k.0) && filter.dial_info_filter.address_type_set.contains(k.1) diff --git a/veilid-core/src/routing_table/find_nodes.rs b/veilid-core/src/routing_table/find_nodes.rs deleted file mode 100644 index 952c4b14..00000000 --- a/veilid-core/src/routing_table/find_nodes.rs +++ /dev/null @@ -1,647 +0,0 @@ -use super::*; - -use crate::dht::*; -use crate::xx::*; -use crate::*; - -pub type LowLevelProtocolPorts = BTreeSet<(LowLevelProtocolType, AddressType, u16)>; -pub type ProtocolToPortMapping = BTreeMap<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>; -#[derive(Clone, Debug)] -pub struct LowLevelPortInfo { - pub low_level_protocol_ports: LowLevelProtocolPorts, - pub protocol_to_port: ProtocolToPortMapping, -} - -impl RoutingTable { - // Makes a filter that finds nodes with a matching inbound dialinfo - pub fn make_inbound_dial_info_entry_filter( - routing_domain: RoutingDomain, - dial_info_filter: DialInfoFilter, - ) -> impl FnMut(&RoutingTableInner, &BucketEntryInner) -> bool { - // does it have matching public dial info? - move |_rti, e| { - if let Some(ni) = e.node_info(routing_domain) { - if ni - .first_filtered_dial_info_detail(DialInfoDetail::NO_SORT, |did| { - did.matches_filter(&dial_info_filter) - }) - .is_some() - { - return true; - } - } - false - } - } - - // Makes a filter that finds nodes capable of dialing a particular outbound dialinfo - pub fn make_outbound_dial_info_entry_filter<'s>( - routing_domain: RoutingDomain, - dial_info: DialInfo, - ) -> impl FnMut(&RoutingTableInner, &'s BucketEntryInner) -> bool { - // does the node's outbound capabilities match the dialinfo? - move |_rti, e| { - if let Some(ni) = e.node_info(routing_domain) { - let dif = DialInfoFilter::all() - .with_protocol_type_set(ni.outbound_protocols) - .with_address_type_set(ni.address_types); - if dial_info.matches_filter(&dif) { - return true; - } - } - false - } - } - - // Make a filter that wraps another filter - pub fn combine_entry_filters<'a, 'b, F, G>( - mut f1: F, - mut f2: G, - ) -> impl FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool - where - F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, - G: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, - { - move |rti, e| { - if !f1(rti, e) { - return false; - } - if !f2(rti, e) { - return false; - } - true - } - } - - // Retrieve the fastest nodes in the routing table matching an entry filter - pub fn find_fast_public_nodes_filtered<'a, 'b, F>( - &self, - node_count: usize, - mut entry_filter: F, - ) -> Vec - where - F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, - { - self.find_fastest_nodes( - // count - node_count, - // filter - |rti, _k: DHTKey, v: Option>| { - let entry = v.unwrap(); - entry.with(rti, |rti, e| { - // skip nodes on local network - if e.node_info(RoutingDomain::LocalNetwork).is_some() { - return false; - } - // skip nodes not on public internet - if e.node_info(RoutingDomain::PublicInternet).is_none() { - return false; - } - // skip nodes that dont match entry filter - entry_filter(rti, e) - }) - }, - // transform - |_rti, k: DHTKey, v: Option>| { - NodeRef::new(self.clone(), k, v.unwrap().clone(), None) - }, - ) - } - - // Retrieve up to N of each type of protocol capable nodes - pub fn find_bootstrap_nodes_filtered(&self, max_per_type: usize) -> Vec { - let protocol_types = vec![ - ProtocolType::UDP, - ProtocolType::TCP, - ProtocolType::WS, - ProtocolType::WSS, - ]; - let mut nodes_proto_v4 = vec![0usize, 0usize, 0usize, 0usize]; - let mut nodes_proto_v6 = vec![0usize, 0usize, 0usize, 0usize]; - - self.find_fastest_nodes( - // count - protocol_types.len() * 2 * max_per_type, - // filter - move |rti, _k: DHTKey, v: Option>| { - let entry = v.unwrap(); - entry.with(rti, |_rti, e| { - // skip nodes on our local network here - if e.has_node_info(RoutingDomain::LocalNetwork.into()) { - return false; - } - - // does it have some dial info we need? - let filter = |n: &NodeInfo| { - let mut keep = false; - for did in &n.dial_info_detail_list { - if matches!(did.dial_info.address_type(), AddressType::IPV4) { - for (n, protocol_type) in protocol_types.iter().enumerate() { - if nodes_proto_v4[n] < max_per_type - && did.dial_info.protocol_type() == *protocol_type - { - nodes_proto_v4[n] += 1; - keep = true; - } - } - } else if matches!(did.dial_info.address_type(), AddressType::IPV6) { - for (n, protocol_type) in protocol_types.iter().enumerate() { - if nodes_proto_v6[n] < max_per_type - && did.dial_info.protocol_type() == *protocol_type - { - nodes_proto_v6[n] += 1; - keep = true; - } - } - } - } - keep - }; - - e.node_info(RoutingDomain::PublicInternet) - .map(filter) - .unwrap_or(false) - }) - }, - // transform - |_rti, k: DHTKey, v: Option>| { - NodeRef::new(self.clone(), k, v.unwrap().clone(), None) - }, - ) - } - - pub fn filter_has_valid_signed_node_info_inner( - inner: &RoutingTableInner, - routing_domain: RoutingDomain, - has_valid_own_node_info: bool, - v: Option>, - ) -> bool { - match v { - None => has_valid_own_node_info, - Some(entry) => entry.with(inner, |_rti, e| { - e.signed_node_info(routing_domain.into()) - .map(|sni| sni.has_valid_signature()) - .unwrap_or(false) - }), - } - } - - pub fn transform_to_peer_info_inner( - inner: &RoutingTableInner, - routing_domain: RoutingDomain, - own_peer_info: PeerInfo, - k: DHTKey, - v: Option>, - ) -> PeerInfo { - match v { - None => own_peer_info, - Some(entry) => entry.with(inner, |_rti, e| { - e.make_peer_info(k, routing_domain).unwrap() - }), - } - } - - pub fn find_peers_with_sort_and_filter<'a, 'b, F, C, T, O>( - &self, - node_count: usize, - cur_ts: u64, - mut filter: F, - compare: C, - mut transform: T, - ) -> Vec - where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - C: FnMut( - &'a RoutingTableInner, - &'b (DHTKey, Option>), - &'b (DHTKey, Option>), - ) -> core::cmp::Ordering, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, - { - let inner = &*self.inner.read(); - Self::find_peers_with_sort_and_filter_inner( - inner, node_count, cur_ts, filter, compare, transform, - ) - } - - pub fn find_peers_with_sort_and_filter_inner<'a, 'b, F, C, T, O>( - inner: &RoutingTableInner, - node_count: usize, - cur_ts: u64, - mut filter: F, - compare: C, - mut transform: T, - ) -> Vec - where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - C: FnMut( - &'a RoutingTableInner, - &'b (DHTKey, Option>), - &'b (DHTKey, Option>), - ) -> core::cmp::Ordering, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, - { - // collect all the nodes for sorting - let mut nodes = - Vec::<(DHTKey, Option>)>::with_capacity(inner.bucket_entry_count + 1); - - // add our own node (only one of there with the None entry) - if filter(inner, inner.node_id, None) { - nodes.push((inner.node_id, None)); - } - - // add all nodes from buckets - Self::with_entries( - &*inner, - cur_ts, - BucketEntryState::Unreliable, - |rti, k, v| { - // Apply filter - if filter(rti, k, Some(v.clone())) { - nodes.push((k, Some(v.clone()))); - } - Option::<()>::None - }, - ); - - // sort by preference for returning nodes - nodes.sort_by(|a, b| compare(inner, a, b)); - - // return transformed vector for filtered+sorted nodes - let cnt = usize::min(node_count, nodes.len()); - let mut out = Vec::::with_capacity(cnt); - for node in nodes { - let val = transform(inner, node.0, node.1); - out.push(val); - } - - out - } - - pub fn find_fastest_nodes<'a, T, F, O>( - &self, - node_count: usize, - mut filter: F, - transform: T, - ) -> Vec - where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, - { - let cur_ts = intf::get_timestamp(); - let out = self.find_peers_with_sort_and_filter( - node_count, - cur_ts, - // filter - |rti, k, v| { - if let Some(entry) = &v { - // always filter out dead nodes - if entry.with(rti, |_rti, e| e.state(cur_ts) == BucketEntryState::Dead) { - false - } else { - filter(rti, k, v) - } - } else { - // always filter out self peer, as it is irrelevant to the 'fastest nodes' search - false - } - }, - // sort - |rti, (a_key, a_entry), (b_key, b_entry)| { - // same nodes are always the same - if a_key == b_key { - return core::cmp::Ordering::Equal; - } - // our own node always comes last (should not happen, here for completeness) - if a_entry.is_none() { - return core::cmp::Ordering::Greater; - } - if b_entry.is_none() { - return core::cmp::Ordering::Less; - } - // reliable nodes come first - let ae = a_entry.as_ref().unwrap(); - let be = b_entry.as_ref().unwrap(); - ae.with(rti, |rti, ae| { - be.with(rti, |_rti, be| { - let ra = ae.check_reliable(cur_ts); - let rb = be.check_reliable(cur_ts); - if ra != rb { - if ra { - return core::cmp::Ordering::Less; - } else { - return core::cmp::Ordering::Greater; - } - } - - // latency is the next metric, closer nodes first - let a_latency = match ae.peer_stats().latency.as_ref() { - None => { - // treat unknown latency as slow - return core::cmp::Ordering::Greater; - } - Some(l) => l, - }; - let b_latency = match be.peer_stats().latency.as_ref() { - None => { - // treat unknown latency as slow - return core::cmp::Ordering::Less; - } - Some(l) => l, - }; - // Sort by average latency - a_latency.average.cmp(&b_latency.average) - }) - }) - }, - // transform, - transform, - ); - out - } - - pub fn find_closest_nodes<'a, F, T, O>( - &self, - node_id: DHTKey, - filter: F, - mut transform: T, - ) -> Vec - where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, - { - let cur_ts = intf::get_timestamp(); - let node_count = { - let c = self.unlocked_inner.config.get(); - c.network.dht.max_find_node_count as usize - }; - let out = self.find_peers_with_sort_and_filter( - node_count, - cur_ts, - // filter - filter, - // sort - |rti, (a_key, a_entry), (b_key, b_entry)| { - // same nodes are always the same - if a_key == b_key { - return core::cmp::Ordering::Equal; - } - - // reliable nodes come first, pessimistically treating our own node as unreliable - let ra = a_entry - .as_ref() - .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); - let rb = b_entry - .as_ref() - .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); - if ra != rb { - if ra { - return core::cmp::Ordering::Less; - } else { - return core::cmp::Ordering::Greater; - } - } - - // distance is the next metric, closer nodes first - let da = distance(a_key, &node_id); - let db = distance(b_key, &node_id); - da.cmp(&db) - }, - // transform, - &mut transform, - ); - log_rtab!(">> find_closest_nodes: node count = {}", out.len()); - out - } - - // Build a map of protocols to low level ports - // This way we can get the set of protocols required to keep our NAT mapping alive for keepalive pings - // Only one protocol per low level protocol/port combination is required - // For example, if WS/WSS and TCP protocols are on the same low-level TCP port, only TCP keepalives will be required - // and we do not need to do WS/WSS keepalive as well. If they are on different ports, then we will need WS/WSS keepalives too. - pub fn get_low_level_port_info(&self) -> LowLevelPortInfo { - let mut low_level_protocol_ports = - BTreeSet::<(LowLevelProtocolType, AddressType, u16)>::new(); - let mut protocol_to_port = - BTreeMap::<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>::new(); - let our_dids = self.all_filtered_dial_info_details( - RoutingDomain::PublicInternet.into(), - &DialInfoFilter::all(), - ); - for did in our_dids { - low_level_protocol_ports.insert(( - did.dial_info.protocol_type().low_level_protocol_type(), - did.dial_info.address_type(), - did.dial_info.socket_address().port(), - )); - protocol_to_port.insert( - (did.dial_info.protocol_type(), did.dial_info.address_type()), - ( - did.dial_info.protocol_type().low_level_protocol_type(), - did.dial_info.socket_address().port(), - ), - ); - } - LowLevelPortInfo { - low_level_protocol_ports, - protocol_to_port, - } - } - - fn make_public_internet_relay_node_filter(&self) -> impl Fn(&BucketEntryInner) -> bool { - // Get all our outbound protocol/address types - let outbound_dif = self.get_outbound_dial_info_filter(RoutingDomain::PublicInternet); - let mapped_port_info = self.get_low_level_port_info(); - - move |e: &BucketEntryInner| { - // Ensure this node is not on the local network - if e.has_node_info(RoutingDomain::LocalNetwork.into()) { - return false; - } - - // Disqualify nodes that don't cover all our inbound ports for tcp and udp - // as we need to be able to use the relay for keepalives for all nat mappings - let mut low_level_protocol_ports = mapped_port_info.low_level_protocol_ports.clone(); - - let can_serve_as_relay = e - .node_info(RoutingDomain::PublicInternet) - .map(|n| { - let dids = n.all_filtered_dial_info_details( - Some(DialInfoDetail::reliable_sort), // By default, choose reliable protocol for relay - |did| did.matches_filter(&outbound_dif), - ); - for did in &dids { - let pt = did.dial_info.protocol_type(); - let at = did.dial_info.address_type(); - if let Some((llpt, port)) = mapped_port_info.protocol_to_port.get(&(pt, at)) - { - low_level_protocol_ports.remove(&(*llpt, at, *port)); - } - } - low_level_protocol_ports.is_empty() - }) - .unwrap_or(false); - if !can_serve_as_relay { - return false; - } - - true - } - } - - #[instrument(level = "trace", skip(self), ret)] - pub fn find_inbound_relay( - &self, - routing_domain: RoutingDomain, - cur_ts: u64, - ) -> Option { - // Get relay filter function - let relay_node_filter = match routing_domain { - RoutingDomain::PublicInternet => self.make_public_internet_relay_node_filter(), - RoutingDomain::LocalNetwork => { - unimplemented!(); - } - }; - - // Go through all entries and find fastest entry that matches filter function - let inner = self.inner.read(); - let inner = &*inner; - let mut best_inbound_relay: Option<(DHTKey, Arc)> = None; - - // Iterate all known nodes for candidates - Self::with_entries(inner, cur_ts, BucketEntryState::Unreliable, |rti, k, v| { - let v2 = v.clone(); - v.with(rti, |rti, e| { - // Ensure we have the node's status - if let Some(node_status) = e.node_status(routing_domain) { - // Ensure the node will relay - if node_status.will_relay() { - // Compare against previous candidate - if let Some(best_inbound_relay) = best_inbound_relay.as_mut() { - // Less is faster - let better = best_inbound_relay.1.with(rti, |_rti, best| { - BucketEntryInner::cmp_fastest_reliable(cur_ts, e, best) - == std::cmp::Ordering::Less - }); - // Now apply filter function and see if this node should be included - if better && relay_node_filter(e) { - *best_inbound_relay = (k, v2); - } - } else if relay_node_filter(e) { - // Always store the first candidate - best_inbound_relay = Some((k, v2)); - } - } - } - }); - // Don't end early, iterate through all entries - Option::<()>::None - }); - // Return the best inbound relay noderef - best_inbound_relay.map(|(k, e)| NodeRef::new(self.clone(), k, e, None)) - } - - #[instrument(level = "trace", skip(self), ret)] - pub fn register_find_node_answer(&self, peers: Vec) -> Vec { - let node_id = self.node_id(); - - // register nodes we'd found - let mut out = Vec::::with_capacity(peers.len()); - for p in peers { - // if our own node if is in the list then ignore it, as we don't add ourselves to our own routing table - if p.node_id.key == node_id { - continue; - } - - // node can not be its own relay - if let Some(rpi) = &p.signed_node_info.node_info.relay_peer_info { - if rpi.node_id == p.node_id { - continue; - } - } - - // register the node if it's new - if let Some(nr) = self.register_node_with_signed_node_info( - RoutingDomain::PublicInternet, - p.node_id.key, - p.signed_node_info.clone(), - false, - ) { - out.push(nr); - } - } - out - } - - #[instrument(level = "trace", skip(self), ret, err)] - pub async fn find_node( - &self, - node_ref: NodeRef, - node_id: DHTKey, - ) -> EyreResult>> { - let rpc_processor = self.rpc_processor(); - - let res = network_result_try!( - rpc_processor - .clone() - .rpc_call_find_node(Destination::direct(node_ref), node_id) - .await? - ); - - // register nodes we'd found - Ok(NetworkResult::value( - self.register_find_node_answer(res.answer), - )) - } - - #[instrument(level = "trace", skip(self), ret, err)] - pub async fn find_self(&self, node_ref: NodeRef) -> EyreResult>> { - let node_id = self.node_id(); - self.find_node(node_ref, node_id).await - } - - #[instrument(level = "trace", skip(self), ret, err)] - pub async fn find_target(&self, node_ref: NodeRef) -> EyreResult>> { - let node_id = node_ref.node_id(); - self.find_node(node_ref, node_id).await - } - - #[instrument(level = "trace", skip(self))] - pub async fn reverse_find_node(&self, node_ref: NodeRef, wide: bool) { - // Ask bootstrap node to 'find' our own node so we can get some more nodes near ourselves - // and then contact those nodes to inform -them- that we exist - - // Ask bootstrap server for nodes closest to our own node - let closest_nodes = network_result_value_or_log!(debug match self.find_self(node_ref.clone()).await { - Err(e) => { - log_rtab!(error - "find_self failed for {:?}: {:?}", - &node_ref, e - ); - return; - } - Ok(v) => v, - } => { - return; - }); - - // Ask each node near us to find us as well - if wide { - for closest_nr in closest_nodes { - network_result_value_or_log!(debug match self.find_self(closest_nr.clone()).await { - Err(e) => { - log_rtab!(error - "find_self failed for {:?}: {:?}", - &closest_nr, e - ); - continue; - } - Ok(v) => v, - } => { - // Do nothing with non-values - continue; - }); - } - } - } -} diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 2bccb4fb..efbdd077 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -1,11 +1,11 @@ mod bucket; mod bucket_entry; mod debug; -mod find_nodes; mod node_ref; mod route_spec_store; mod routing_domain_editor; mod routing_domains; +mod routing_table_inner; mod stats_accounting; mod tasks; @@ -17,43 +17,24 @@ use crate::*; use bucket::*; pub use bucket_entry::*; pub use debug::*; -pub use find_nodes::*; use hashlink::LruCache; pub use node_ref::*; pub use route_spec_store::*; pub use routing_domain_editor::*; pub use routing_domains::*; +pub use routing_table_inner::*; pub use stats_accounting::*; const RECENT_PEERS_TABLE_SIZE: usize = 64; ////////////////////////////////////////////////////////////////////////// -#[derive(Debug, Clone, Copy)] -pub struct RecentPeersEntry { - pub last_connection: ConnectionDescriptor, -} - -/// RoutingTable rwlock-internal data -struct RoutingTableInner { - /// Routing table buckets that hold entries - buckets: Vec, - /// A fast counter for the number of entries in the table, total - bucket_entry_count: usize, - /// The public internet routing domain - public_internet_routing_domain: PublicInternetRoutingDomainDetail, - /// The dial info we use on the local network - local_network_routing_domain: LocalNetworkRoutingDomainDetail, - /// Interim accounting mechanism for this node's RPC latency to any other node - self_latency_stats_accounting: LatencyStatsAccounting, - /// Interim accounting mechanism for the total bandwidth to/from this node - self_transfer_stats_accounting: TransferStatsAccounting, - /// Statistics about the total bandwidth to/from this node - self_transfer_stats: TransferStatsDownUp, - /// Peers we have recently communicated with - recent_peers: LruCache, - /// Storage for private/safety RouteSpecs - route_spec_store: RouteSpecStore, +pub type LowLevelProtocolPorts = BTreeSet<(LowLevelProtocolType, AddressType, u16)>; +pub type ProtocolToPortMapping = BTreeMap<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>; +#[derive(Clone, Debug)] +pub struct LowLevelPortInfo { + pub low_level_protocol_ports: LowLevelProtocolPorts, + pub protocol_to_port: ProtocolToPortMapping, } #[derive(Clone, Debug, Default)] @@ -90,19 +71,6 @@ pub struct RoutingTable { } impl RoutingTable { - fn new_inner(config: VeilidConfig) -> RoutingTableInner { - RoutingTableInner { - buckets: Vec::new(), - public_internet_routing_domain: PublicInternetRoutingDomainDetail::default(), - local_network_routing_domain: LocalNetworkRoutingDomainDetail::default(), - bucket_entry_count: 0, - self_latency_stats_accounting: LatencyStatsAccounting::new(), - self_transfer_stats_accounting: TransferStatsAccounting::new(), - self_transfer_stats: TransferStatsDownUp::default(), - recent_peers: LruCache::new(RECENT_PEERS_TABLE_SIZE), - route_spec_store: RouteSpecStore::new(config), - } - } fn new_unlocked_inner( config: VeilidConfig, network_manager: NetworkManager, @@ -120,10 +88,13 @@ impl RoutingTable { } pub fn new(network_manager: NetworkManager) -> Self { let config = network_manager.config(); + let unlocked_inner = Arc::new(Self::new_unlocked_inner(config, network_manager)); + let inner = Arc::new(RwLock::new(RoutingTableInner::new(unlocked_inner.clone()))); let this = Self { - inner: Arc::new(RwLock::new(Self::new_inner(config.clone()))), - unlocked_inner: Arc::new(Self::new_unlocked_inner(config, network_manager)), + inner, + unlocked_inner, }; + // Set rolling transfers tick task { let this2 = this.clone(); @@ -174,82 +145,76 @@ impl RoutingTable { self.unlocked_inner.node_id_secret } - fn routing_domain_for_address_inner( - inner: &RoutingTableInner, - address: Address, - ) -> Option { - for rd in RoutingDomain::all() { - let can_contain = - Self::with_routing_domain(inner, rd, |rdd| rdd.can_contain_address(address)); - if can_contain { - return Some(rd); - } - } - None + ///////////////////////////////////// + /// Initialization + + /// Called to initialize the routing table after it is created + pub async fn init(&self) -> EyreResult<()> { + let mut inner = self.inner.write(); + inner.init(self.clone()); + Ok(()) } + /// Called to shut down the routing table + pub async fn terminate(&self) { + debug!("starting routing table terminate"); + + // Cancel all tasks being ticked + debug!("stopping rolling transfers task"); + if let Err(e) = self.unlocked_inner.rolling_transfers_task.stop().await { + error!("rolling_transfers_task not stopped: {}", e); + } + debug!("stopping kick buckets task"); + if let Err(e) = self.unlocked_inner.kick_buckets_task.stop().await { + error!("kick_buckets_task not stopped: {}", e); + } + + let mut inner = self.inner.write(); + inner.terminate(); + *inner = RoutingTableInner::new(self.unlocked_inner.clone()); + + debug!("finished routing table terminate"); + } + + /// Set up the local network routing domain with our local routing table configuration + pub fn configure_local_network_routing_domain(&self, local_networks: Vec<(IpAddr, IpAddr)>) { + log_net!(debug "configure_local_network_routing_domain: {:#?}", local_networks); + self.inner + .write() + .configure_local_network_routing_domain(local_networks); + } + + ///////////////////////////////////// + /// Locked operations + pub fn routing_domain_for_address(&self, address: Address) -> Option { - let inner = self.inner.read(); - Self::routing_domain_for_address_inner(&*inner, address) - } - - fn with_routing_domain(inner: &RoutingTableInner, domain: RoutingDomain, f: F) -> R - where - F: FnOnce(&dyn RoutingDomainDetail) -> R, - { - match domain { - RoutingDomain::PublicInternet => f(&inner.public_internet_routing_domain), - RoutingDomain::LocalNetwork => f(&inner.local_network_routing_domain), - } - } - - fn with_routing_domain_mut( - inner: &mut RoutingTableInner, - domain: RoutingDomain, - f: F, - ) -> R - where - F: FnOnce(&mut dyn RoutingDomainDetail) -> R, - { - match domain { - RoutingDomain::PublicInternet => f(&mut inner.public_internet_routing_domain), - RoutingDomain::LocalNetwork => f(&mut inner.local_network_routing_domain), - } + self.inner.read().routing_domain_for_address(address) } pub fn with_route_spec_store_mut(&self, f: F) -> R where F: FnOnce(&mut RouteSpecStore, &mut RoutingTableInner) -> R, { - let inner = &mut *self.inner.write(); - f(&mut inner.route_spec_store, inner) + self.inner.write().with_route_spec_store_mut(f) } pub fn with_route_spec_store(&self, f: F) -> R where F: FnOnce(&RouteSpecStore, &RoutingTableInner) -> R, { - let inner = &*self.inner.read(); - f(&inner.route_spec_store, inner) + self.inner.read().with_route_spec_store(f) } pub fn relay_node(&self, domain: RoutingDomain) -> Option { - let inner = self.inner.read(); - Self::with_routing_domain(&*inner, domain, |rd| rd.common().relay_node()) + self.inner.read().relay_node(domain) } pub fn has_dial_info(&self, domain: RoutingDomain) -> bool { - let inner = self.inner.read(); - Self::with_routing_domain(&*inner, domain, |rd| { - !rd.common().dial_info_details().is_empty() - }) + self.inner.read().has_dial_info(domain) } pub fn dial_info_details(&self, domain: RoutingDomain) -> Vec { - let inner = self.inner.read(); - Self::with_routing_domain(&*inner, domain, |rd| { - rd.common().dial_info_details().clone() - }) + self.inner.read().dial_info_details(domain) } pub fn first_filtered_dial_info_detail( @@ -257,21 +222,9 @@ impl RoutingTable { routing_domain_set: RoutingDomainSet, filter: &DialInfoFilter, ) -> Option { - let inner = self.inner.read(); - for routing_domain in routing_domain_set { - let did = Self::with_routing_domain(&*inner, routing_domain, |rd| { - for did in rd.common().dial_info_details() { - if did.matches_filter(filter) { - return Some(did.clone()); - } - } - None - }); - if did.is_some() { - return did; - } - } - None + self.inner + .read() + .first_filtered_dial_info_detail(routing_domain_set, filter) } pub fn all_filtered_dial_info_details( @@ -279,39 +232,15 @@ impl RoutingTable { routing_domain_set: RoutingDomainSet, filter: &DialInfoFilter, ) -> Vec { - let inner = self.inner.read(); - let mut ret = Vec::new(); - for routing_domain in routing_domain_set { - Self::with_routing_domain(&*inner, routing_domain, |rd| { - for did in rd.common().dial_info_details() { - if did.matches_filter(filter) { - ret.push(did.clone()); - } - } - }); - } - ret.remove_duplicates(); - ret + self.inner + .read() + .all_filtered_dial_info_details(routing_domain_set, filter) } pub fn ensure_dial_info_is_valid(&self, domain: RoutingDomain, dial_info: &DialInfo) -> bool { - let address = dial_info.socket_address().address(); - let inner = self.inner.read(); - let can_contain_address = - Self::with_routing_domain(&*inner, domain, |rd| rd.can_contain_address(address)); - - if !can_contain_address { - log_rtab!(debug "can not add dial info to this routing domain"); - return false; - } - if !dial_info.is_valid() { - log_rtab!(debug - "shouldn't be registering invalid addresses: {:?}", - dial_info - ); - return false; - } - true + self.inner + .read() + .ensure_dial_info_is_valid(domain, dial_info) } pub fn node_info_is_valid_in_routing_domain( @@ -319,48 +248,9 @@ impl RoutingTable { routing_domain: RoutingDomain, node_info: &NodeInfo, ) -> bool { - // Should not be passing around nodeinfo with an invalid network class - if matches!(node_info.network_class, NetworkClass::Invalid) { - return false; - } - // Ensure all of the dial info works in this routing domain - for did in &node_info.dial_info_detail_list { - if !self.ensure_dial_info_is_valid(routing_domain, &did.dial_info) { - return false; - } - } - // Ensure the relay is also valid in this routing domain if it is provided - if let Some(relay_peer_info) = node_info.relay_peer_info.as_ref() { - let relay_ni = &relay_peer_info.signed_node_info.node_info; - if !self.node_info_is_valid_in_routing_domain(routing_domain, relay_ni) { - return false; - } - } - true - } - - #[instrument(level = "trace", skip(inner), ret)] - fn get_contact_method_inner( - inner: &RoutingTableInner, - routing_domain: RoutingDomain, - node_a_id: &DHTKey, - node_a: &NodeInfo, - node_b_id: &DHTKey, - node_b: &NodeInfo, - dial_info_filter: DialInfoFilter, - reliable: bool, - ) -> ContactMethod { - Self::with_routing_domain(inner, routing_domain, |rdd| { - rdd.get_contact_method( - inner, - node_a_id, - node_a, - node_b_id, - node_b, - dial_info_filter, - reliable, - ) - }) + self.inner + .read() + .node_info_is_valid_in_routing_domain(routing_domain, node_info) } /// Look up the best way for two nodes to reach each other over a specific routing domain @@ -375,9 +265,7 @@ impl RoutingTable { dial_info_filter: DialInfoFilter, reliable: bool, ) -> ContactMethod { - let inner = &*self.inner.read(); - Self::get_contact_method_inner( - inner, + self.inner.read().get_contact_method( routing_domain, node_a_id, node_a, @@ -388,307 +276,72 @@ impl RoutingTable { ) } - // Figure out how to reach a node from our own node over the best routing domain and reference the nodes we want to access - #[instrument(level = "trace", skip(self), ret)] - pub(crate) fn get_node_contact_method( - &self, - target_node_ref: NodeRef, - ) -> EyreResult { - // Lock the routing table for read to ensure the table doesn't change - let inner = &*self.inner.read(); - - // Figure out the best routing domain to get the contact method over - let routing_domain = match target_node_ref.best_routing_domain() { - Some(rd) => rd, - None => { - log_net!("no routing domain for node {:?}", target_node_ref); - return Ok(NodeContactMethod::Unreachable); - } - }; - - // Node A is our own node - let node_a = get_own_node_info_inner(inner, routing_domain); - let node_a_id = self.node_id(); - - // Node B is the target node - let node_b = target_node_ref.xxx operate(|_rti, e| e.node_info(routing_domain).unwrap()); - let node_b_id = target_node_ref.node_id(); - - // Dial info filter comes from the target node ref - let dial_info_filter = target_node_ref.dial_info_filter(); - let reliable = target_node_ref.reliable(); - - let cm = self.get_contact_method( - routing_domain, - &node_a_id, - &node_a, - &node_b_id, - node_b, - dial_info_filter, - reliable, - ); - - // Translate the raw contact method to a referenced contact method - Ok(match cm { - ContactMethod::Unreachable => NodeContactMethod::Unreachable, - ContactMethod::Existing => NodeContactMethod::Existing, - ContactMethod::Direct(di) => NodeContactMethod::Direct(di), - ContactMethod::SignalReverse(relay_key, target_key) => { - let relay_nr = Self::lookup_and_filter_noderef_inner(inner, self.clone(), relay_key, routing_domain.into(), dial_info_filter) - .ok_or_else(|| eyre!("couldn't look up relay"))?; - if target_node_ref.node_id() != target_key { - bail!("target noderef didn't match target key"); - } - NodeContactMethod::SignalReverse(relay_nr, target_node_ref) - } - ContactMethod::SignalHolePunch(relay_key, target_key) => { - let relay_nr = Self::lookup_and_filter_noderef_inner(inner, self.clone(), relay_key, routing_domain.into(), dial_info_filter) - .ok_or_else(|| eyre!("couldn't look up relay"))?; - if target_node_ref.node_id() != target_key { - bail!("target noderef didn't match target key"); - } - NodeContactMethod::SignalHolePunch(relay_nr, target_node_ref) - } - ContactMethod::InboundRelay(relay_key) => { - let relay_nr = Self::lookup_and_filter_noderef_nner(inner, self.clone(), relay_key, routing_domain.into(), dial_info_filter) - .ok_or_else(|| eyre!("couldn't look up relay"))?; - NodeContactMethod::InboundRelay(relay_nr) - } - ContactMethod::OutboundRelay(relay_key) => { - let relay_nr = Self::lookup_and_filter_noderef(inner, self.clone(), relay_key, routing_domain.into(), dial_info_filter) - .ok_or_else(|| eyre!("couldn't look up relay"))?; - NodeContactMethod::OutboundRelay(relay_nr) - } - }) - } - #[instrument(level = "debug", skip(self))] pub fn edit_routing_domain(&self, domain: RoutingDomain) -> RoutingDomainEditor { RoutingDomainEditor::new(self.clone(), domain) } - fn reset_all_seen_our_node_info(inner: &mut RoutingTableInner, routing_domain: RoutingDomain) { - let cur_ts = intf::get_timestamp(); - Self::with_entries_mut(inner, cur_ts, BucketEntryState::Dead, |rti, _, v| { - v.with_mut(rti, |_rti, e| { - e.set_seen_our_node_info(routing_domain, false); - }); - Option::<()>::None - }); - } - - fn reset_all_updated_since_last_network_change(inner: &mut RoutingTableInner) { - let cur_ts = intf::get_timestamp(); - Self::with_entries_mut(inner, cur_ts, BucketEntryState::Dead, |rti, _, v| { - v.with_mut(rti, |_rti, e| { - e.set_updated_since_last_network_change(false) - }); - Option::<()>::None - }); - } - /// Return a copy of our node's peerinfo pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo { - let inner = &*self.inner.read(); - Self::with_routing_domain(inner, routing_domain, |rdd| { - rdd.common().with_peer_info(|pi| pi.clone()) - }) + self.inner.read().get_own_peer_info(routing_domain) } /// Return a copy of our node's signednodeinfo pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedNodeInfo { - let inner = &*self.inner.read(); - Self::with_routing_domain(inner, routing_domain, |rdd| { - rdd.common() - .with_peer_info(|pi| pi.signed_node_info.clone()) - }) + self.inner.read().get_own_signed_node_info(routing_domain) } /// Return a copy of our node's nodeinfo - fn get_own_node_info_inner( - inner: &RoutingTableInner, - routing_domain: RoutingDomain, - ) -> NodeInfo { - Self::with_routing_domain(inner, routing_domain, |rdd| { - rdd.common() - .with_peer_info(|pi| pi.signed_node_info.node_info.clone()) - }) - } pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo { - let inner = &*self.inner.read(); - Self::get_own_node_info_inner(inner, routing_domain) + self.inner.read().get_own_node_info(routing_domain) } - /// Return our currently registered network class + /// If we have a valid network class in this routing domain, then our 'NodeInfo' is valid pub fn has_valid_own_node_info(&self, routing_domain: RoutingDomain) -> bool { - let inner = &*self.inner.read(); - Self::with_routing_domain(inner, routing_domain, |rdd| { - rdd.common().has_valid_own_node_info() - }) + self.inner.read().has_valid_own_node_info(routing_domain) } /// Return the domain's currently registered network class pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option { - let inner = &*self.inner.read(); - Self::with_routing_domain(inner, routing_domain, |rdd| rdd.common().network_class()) + self.inner.read().get_network_class(routing_domain) } /// Return the domain's filter for what we can receivein the form of a dial info filter pub fn get_inbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { - let inner = &*self.inner.read(); - Self::with_routing_domain(inner, routing_domain, |rdd| { - rdd.common().inbound_dial_info_filter() - }) + self.inner + .read() + .get_inbound_dial_info_filter(routing_domain) } /// Return the domain's filter for what we can receive in the form of a node ref filter pub fn get_inbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { - let dif = self.get_inbound_dial_info_filter(routing_domain); - NodeRefFilter::new() - .with_routing_domain(routing_domain) - .with_dial_info_filter(dif) + self.inner + .read() + .get_inbound_node_ref_filter(routing_domain) } /// Return the domain's filter for what we can send out in the form of a dial info filter pub fn get_outbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { - let inner = &*self.inner.read(); - Self::with_routing_domain(inner, routing_domain, |rdd| { - rdd.common().outbound_dial_info_filter() - }) + self.inner + .read() + .get_outbound_dial_info_filter(routing_domain) } /// Return the domain's filter for what we can receive in the form of a node ref filter pub fn get_outbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { - let dif = self.get_outbound_dial_info_filter(routing_domain); - NodeRefFilter::new() - .with_routing_domain(routing_domain) - .with_dial_info_filter(dif) - } - - fn bucket_depth(index: usize) -> usize { - match index { - 0 => 256, - 1 => 128, - 2 => 64, - 3 => 32, - 4 => 16, - 5 => 8, - 6 => 4, - 7 => 4, - 8 => 4, - 9 => 4, - _ => 4, - } - } - - pub async fn init(&self) -> EyreResult<()> { - let mut inner = self.inner.write(); - // Size the buckets (one per bit) - inner.buckets.reserve(DHT_KEY_LENGTH * 8); - for _ in 0..DHT_KEY_LENGTH * 8 { - let bucket = Bucket::new(self.clone()); - inner.buckets.push(bucket); - } - - Ok(()) - } - - pub async fn terminate(&self) { - debug!("starting routing table terminate"); - - // Cancel all tasks being ticked - debug!("stopping rolling transfers task"); - if let Err(e) = self.unlocked_inner.rolling_transfers_task.stop().await { - error!("rolling_transfers_task not stopped: {}", e); - } - debug!("stopping kick buckets task"); - if let Err(e) = self.unlocked_inner.kick_buckets_task.stop().await { - error!("kick_buckets_task not stopped: {}", e); - } - - *self.inner.write() = Self::new_inner(self.unlocked_inner.config.clone()); - - debug!("finished routing table terminate"); - } - - pub fn configure_local_network_routing_domain(&self, local_networks: Vec<(IpAddr, IpAddr)>) { - log_net!(debug "configure_local_network_routing_domain: {:#?}", local_networks); - - let mut inner = self.inner.write(); - let changed = inner - .local_network_routing_domain - .set_local_networks(local_networks); - - // If the local network topology has changed, nuke the existing local node info and let new local discovery happen - if changed { - let cur_ts = intf::get_timestamp(); - Self::with_entries_mut(&mut *inner, cur_ts, BucketEntryState::Dead, |rti, _, e| { - e.with_mut(rti, |_rti, e| { - e.clear_signed_node_info(RoutingDomain::LocalNetwork); - e.set_seen_our_node_info(RoutingDomain::LocalNetwork, false); - e.set_updated_since_last_network_change(false); - }); - Option::<()>::None - }); - } + self.inner + .read() + .get_outbound_node_ref_filter(routing_domain) } /// Attempt to empty the routing table /// should only be performed when there are no node_refs (detached) pub fn purge_buckets(&self) { - let mut inner = self.inner.write(); - let inner = &mut *inner; - log_rtab!( - "Starting routing table buckets purge. Table currently has {} nodes", - inner.bucket_entry_count - ); - for bucket in &inner.buckets { - bucket.kick(inner, 0); - } - log_rtab!(debug - "Routing table buckets purge complete. Routing table now has {} nodes", - inner.bucket_entry_count - ); + self.inner.write().purge_buckets(); } /// Attempt to remove last_connections from entries pub fn purge_last_connections(&self) { - let mut inner = self.inner.write(); - let inner = &mut *inner; - log_rtab!( - "Starting routing table last_connections purge. Table currently has {} nodes", - inner.bucket_entry_count - ); - for bucket in &inner.buckets { - for entry in bucket.entries() { - entry.1.with_mut(inner, |_rti, e| { - e.clear_last_connections(); - }); - } - } - log_rtab!(debug - "Routing table last_connections purge complete. Routing table now has {} nodes", - inner.bucket_entry_count - ); - } - - /// Attempt to settle buckets and remove entries down to the desired number - /// which may not be possible due extant NodeRefs - fn kick_bucket(inner: &mut RoutingTableInner, idx: usize) { - let bucket = &mut inner.buckets[idx]; - let bucket_depth = Self::bucket_depth(idx); - - if let Some(dead_node_ids) = bucket.kick(inner, bucket_depth) { - // Remove counts - inner.bucket_entry_count -= dead_node_ids.len(); - log_rtab!(debug "Routing table now has {} nodes", inner.bucket_entry_count); - - // Now purge the routing table inner vectors - //let filter = |k: &DHTKey| dead_node_ids.contains(k); - //inner.closest_reliable_nodes.retain(filter); - //inner.fastest_reliable_nodes.retain(filter); - //inner.closest_nodes.retain(filter); - //inner.fastest_nodes.retain(filter); - } + self.inner.write().purge_last_connections(); } fn find_bucket_index(&self, node_id: DHTKey) -> usize { @@ -702,65 +355,9 @@ impl RoutingTable { routing_domain_set: RoutingDomainSet, min_state: BucketEntryState, ) -> usize { - let inner = self.inner.read(); - Self::get_entry_count_inner(&*inner, routing_domain_set, min_state) - } - - fn get_entry_count_inner( - inner: &RoutingTableInner, - routing_domain_set: RoutingDomainSet, - min_state: BucketEntryState, - ) -> usize { - let mut count = 0usize; - let cur_ts = intf::get_timestamp(); - Self::with_entries(inner, cur_ts, min_state, |rti, _, e| { - if e.with(rti, |_rti, e| e.best_routing_domain(routing_domain_set)) - .is_some() - { - count += 1; - } - Option::<()>::None - }); - count - } - - fn with_entries) -> Option>( - inner: &RoutingTableInner, - cur_ts: u64, - min_state: BucketEntryState, - mut f: F, - ) -> Option { - for bucket in &inner.buckets { - for entry in bucket.entries() { - if entry.1.with(inner, |_rti, e| e.state(cur_ts) >= min_state) { - if let Some(out) = f(inner, *entry.0, entry.1.clone()) { - return Some(out); - } - } - } - } - None - } - - fn with_entries_mut< - T, - F: FnMut(&mut RoutingTableInner, DHTKey, Arc) -> Option, - >( - inner: &mut RoutingTableInner, - cur_ts: u64, - min_state: BucketEntryState, - mut f: F, - ) -> Option { - for bucket in &inner.buckets { - for entry in bucket.entries() { - if entry.1.with(inner, |_rti, e| e.state(cur_ts) >= min_state) { - if let Some(out) = f(inner, *entry.0, entry.1.clone()) { - return Some(out); - } - } - } - } - None + self.inner + .read() + .get_entry_count(routing_domain_set, min_state) } pub fn get_nodes_needing_updates( @@ -769,26 +366,9 @@ impl RoutingTable { cur_ts: u64, all: bool, ) -> Vec { - let inner = self.inner.read(); - let mut node_refs = Vec::::with_capacity(inner.bucket_entry_count); - Self::with_entries( - &*inner, - cur_ts, - BucketEntryState::Unreliable, - |rti, k, v| { - // Only update nodes that haven't seen our node info yet - if all || !v.with(rti, |_rti, e| e.has_seen_our_node_info(routing_domain)) { - node_refs.push(NodeRef::new( - self.clone(), - k, - v, - Some(NodeRefFilter::new().with_routing_domain(routing_domain)), - )); - } - Option::<()>::None - }, - ); - node_refs + self.inner + .read() + .get_nodes_needing_updates(self.clone(), routing_domain, cur_ts, all) } pub fn get_nodes_needing_ping( @@ -796,50 +376,14 @@ impl RoutingTable { routing_domain: RoutingDomain, cur_ts: u64, ) -> Vec { - let inner = self.inner.read(); - - // Collect relay nodes - let opt_relay_id = Self::with_routing_domain(&*inner, routing_domain, |rd| { - rd.common().relay_node().map(|rn| rn.node_id()) - }); - - // Collect all entries that are 'needs_ping' and have some node info making them reachable somehow - let mut node_refs = Vec::::with_capacity(inner.bucket_entry_count); - Self::with_entries( - &*inner, - cur_ts, - BucketEntryState::Unreliable, - |rti, k, v| { - if v.with(rti, |_rti, e| { - e.has_node_info(routing_domain.into()) - && e.needs_ping(cur_ts, opt_relay_id == Some(k)) - }) { - node_refs.push(NodeRef::new( - self.clone(), - k, - v, - Some(NodeRefFilter::new().with_routing_domain(routing_domain)), - )); - } - Option::<()>::None - }, - ); - node_refs + self.inner + .read() + .get_nodes_needing_ping(self.clone(), routing_domain, cur_ts) } pub fn get_all_nodes(&self, cur_ts: u64) -> Vec { let inner = self.inner.read(); - let mut node_refs = Vec::::with_capacity(inner.bucket_entry_count); - Self::with_entries( - &*inner, - cur_ts, - BucketEntryState::Unreliable, - |_rti, k, v| { - node_refs.push(NodeRef::new(self.clone(), k, v, None)); - Option::<()>::None - }, - ); - node_refs + inner.get_all_nodes(self.clone(), cur_ts) } fn queue_bucket_kick(&self, node_id: DHTKey) { @@ -854,77 +398,14 @@ impl RoutingTable { where F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner), { - // Ensure someone isn't trying register this node itself - if node_id == self.node_id() { - log_rtab!(debug "can't register own node"); - return None; - } - - // Lock this entire operation - let mut inner = self.inner.write(); - let inner = &mut *inner; - - // Look up existing entry - let idx = self.find_bucket_index(node_id); - let noderef = { - let bucket = &inner.buckets[idx]; - let entry = bucket.entry(&node_id); - entry.map(|e| NodeRef::new(self.clone(), node_id, e, None)) - }; - - // If one doesn't exist, insert into bucket, possibly evicting a bucket member - let noderef = match noderef { - None => { - // Make new entry - inner.bucket_entry_count += 1; - let cnt = inner.bucket_entry_count; - let bucket = &mut inner.buckets[idx]; - let nr = bucket.add_entry(node_id); - - // Update the entry - let entry = bucket.entry(&node_id).unwrap(); - entry.with_mut(inner, update_func); - - // Kick the bucket - self.unlocked_inner.kick_queue.lock().insert(idx); - log_rtab!(debug "Routing table now has {} nodes, {} live", cnt, Self::get_entry_count_inner(&mut *inner, RoutingDomainSet::all(), BucketEntryState::Unreliable)); - - nr - } - Some(nr) => { - // Update the entry - let bucket = &mut inner.buckets[idx]; - let entry = bucket.entry(&node_id).unwrap(); - entry.with_mut(inner, update_func); - - nr - } - }; - - Some(noderef) + self.inner + .write() + .create_node_ref(self.clone(), node_id, update_func) } /// Resolve an existing routing table entry and return a reference to it - fn lookup_node_ref_inner(inner: &RoutingTableInner, routing_table: RoutingTable, node_id: DHTKey) -> Option { - { - let idx = routing_table.find_bucket_index(node_id); - let bucket = &inner.buckets[idx]; - bucket - .entry(&node_id) - .map(|e| NodeRef::new(routing_table, node_id, e, None)) - } - pub fn lookup_node_ref(&self, node_id: DHTKey) -> Option { - if node_id == self.unlocked_inner.node_id { - log_rtab!(debug "can't look up own node id in routing table"); - return None; - } - let idx = self.find_bucket_index(node_id); - let inner = self.inner.read(); - let bucket = &inner.buckets[idx]; - bucket - .entry(&node_id) - .map(|e| NodeRef::new(self.clone(), node_id, e, None)) + self.inner.read().lookup_node_ref(self.clone(), node_id) } /// Resolve an existing routing table entry and return a filtered reference to it @@ -934,13 +415,11 @@ impl RoutingTable { routing_domain_set: RoutingDomainSet, dial_info_filter: DialInfoFilter, ) -> Option { - let nr = self.lookup_node_ref(node_id)?; - Some( - nr.filtered_clone( - NodeRefFilter::new() - .with_dial_info_filter(dial_info_filter) - .with_routing_domain_set(routing_domain_set), - ), + self.inner.read().lookup_and_filter_noderef( + self.clone(), + node_id, + routing_domain_set, + dial_info_filter, ) } @@ -954,43 +433,13 @@ impl RoutingTable { signed_node_info: SignedNodeInfo, allow_invalid: bool, ) -> Option { - //log_rtab!("register_node_with_signed_node_info: routing_domain: {:?}, node_id: {:?}, signed_node_info: {:?}, allow_invalid: {:?}", routing_domain, node_id, signed_node_info, allow_invalid ); - - // validate signed node info is not something malicious - if node_id == self.node_id() { - log_rtab!(debug "can't register own node id in routing table"); - return None; - } - if let Some(rpi) = &signed_node_info.node_info.relay_peer_info { - if rpi.node_id.key == node_id { - log_rtab!(debug "node can not be its own relay"); - return None; - } - } - if !allow_invalid { - // verify signature - if !signed_node_info.has_valid_signature() { - log_rtab!(debug "signed node info for {} has invalid signature", node_id); - return None; - } - // verify signed node info is valid in this routing domain - if !self - .node_info_is_valid_in_routing_domain(routing_domain, &signed_node_info.node_info) - { - log_rtab!(debug "signed node info for {} not valid in the {:?} routing domain", node_id, routing_domain); - return None; - } - } - - self.create_node_ref(node_id, |_rti, e| { - e.update_signed_node_info(routing_domain, signed_node_info); - }) - .map(|mut nr| { - nr.set_filter(Some( - NodeRefFilter::new().with_routing_domain(routing_domain), - )); - nr - }) + self.inner.write().register_node_with_signed_node_info( + self.clone(), + routing_domain, + node_id, + signed_node_info, + allow_invalid, + ) } /// Shortcut function to add a node to our routing table if it doesn't exist @@ -1001,15 +450,12 @@ impl RoutingTable { descriptor: ConnectionDescriptor, timestamp: u64, ) -> Option { - let out = self.create_node_ref(node_id, |_rti, e| { - // this node is live because it literally just connected to us - e.touch_last_seen(timestamp); - }); - if let Some(nr) = &out { - // set the most recent node address for connection finding and udp replies - nr.set_last_connection(descriptor, timestamp); - } - out + self.inner.write().register_node_with_existing_connection( + self.clone(), + node_id, + descriptor, + timestamp, + ) } /// Ticks about once per second @@ -1031,71 +477,436 @@ impl RoutingTable { // Routing Table Health Metrics pub fn get_routing_table_health(&self) -> RoutingTableHealth { - let mut health = RoutingTableHealth::default(); - let cur_ts = intf::get_timestamp(); - let inner = self.inner.read(); - let inner = &*inner; - for bucket in &inner.buckets { - for (_, v) in bucket.entries() { - match v.with(inner, |_rti, e| e.state(cur_ts)) { - BucketEntryState::Reliable => { - health.reliable_entry_count += 1; - } - BucketEntryState::Unreliable => { - health.unreliable_entry_count += 1; - } - BucketEntryState::Dead => { - health.dead_entry_count += 1; - } - } - } - } - health + self.inner.read().get_routing_table_health() } pub fn get_recent_peers(&self) -> Vec<(DHTKey, RecentPeersEntry)> { - let mut recent_peers = Vec::new(); - let mut dead_peers = Vec::new(); - let mut out = Vec::new(); - - // collect all recent peers - { - let inner = self.inner.read(); - for (k, _v) in &inner.recent_peers { - recent_peers.push(*k); - } - } - - // look up each node and make sure the connection is still live - // (uses same logic as send_data, ensuring last_connection works for UDP) - for e in &recent_peers { - let mut dead = true; - if let Some(nr) = self.lookup_node_ref(*e) { - if let Some(last_connection) = nr.last_connection() { - out.push((*e, RecentPeersEntry { last_connection })); - dead = false; - } - } - if dead { - dead_peers.push(e); - } - } - - // purge dead recent peers - { - let mut inner = self.inner.write(); - for d in dead_peers { - inner.recent_peers.remove(d); - } - } - - out + self.inner.write().get_recent_peers(self.clone()) } pub fn touch_recent_peer(&self, node_id: DHTKey, last_connection: ConnectionDescriptor) { - let mut inner = self.inner.write(); - inner - .recent_peers - .insert(node_id, RecentPeersEntry { last_connection }); + self.inner + .write() + .touch_recent_peer(node_id, last_connection) + } + + ////////////////////////////////////////////////////////////////////// + // Find Nodes + + /// Build a map of protocols to low level ports + /// This way we can get the set of protocols required to keep our NAT mapping alive for keepalive pings + /// Only one protocol per low level protocol/port combination is required + /// For example, if WS/WSS and TCP protocols are on the same low-level TCP port, only TCP keepalives will be required + /// and we do not need to do WS/WSS keepalive as well. If they are on different ports, then we will need WS/WSS keepalives too. + pub fn get_low_level_port_info(&self) -> LowLevelPortInfo { + let mut low_level_protocol_ports = + BTreeSet::<(LowLevelProtocolType, AddressType, u16)>::new(); + let mut protocol_to_port = + BTreeMap::<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>::new(); + let our_dids = self.all_filtered_dial_info_details( + RoutingDomain::PublicInternet.into(), + &DialInfoFilter::all(), + ); + for did in our_dids { + low_level_protocol_ports.insert(( + did.dial_info.protocol_type().low_level_protocol_type(), + did.dial_info.address_type(), + did.dial_info.socket_address().port(), + )); + protocol_to_port.insert( + (did.dial_info.protocol_type(), did.dial_info.address_type()), + ( + did.dial_info.protocol_type().low_level_protocol_type(), + did.dial_info.socket_address().port(), + ), + ); + } + LowLevelPortInfo { + low_level_protocol_ports, + protocol_to_port, + } + } + + /// Makes a filter that finds nodes with a matching inbound dialinfo + pub fn make_inbound_dial_info_entry_filter( + routing_domain: RoutingDomain, + dial_info_filter: DialInfoFilter, + ) -> impl FnMut(&RoutingTableInner, &BucketEntryInner) -> bool { + // does it have matching public dial info? + move |_rti, e| { + if let Some(ni) = e.node_info(routing_domain) { + if ni + .first_filtered_dial_info_detail(DialInfoDetail::NO_SORT, |did| { + did.matches_filter(&dial_info_filter) + }) + .is_some() + { + return true; + } + } + false + } + } + + /// Makes a filter that finds nodes capable of dialing a particular outbound dialinfo + pub fn make_outbound_dial_info_entry_filter<'s>( + routing_domain: RoutingDomain, + dial_info: DialInfo, + ) -> impl FnMut(&RoutingTableInner, &'s BucketEntryInner) -> bool { + // does the node's outbound capabilities match the dialinfo? + move |_rti, e| { + if let Some(ni) = e.node_info(routing_domain) { + let dif = DialInfoFilter::all() + .with_protocol_type_set(ni.outbound_protocols) + .with_address_type_set(ni.address_types); + if dial_info.matches_filter(&dif) { + return true; + } + } + false + } + } + + /// Make a filter that wraps another filter + pub fn combine_entry_filters<'a, 'b, F, G>( + mut f1: F, + mut f2: G, + ) -> impl FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool + where + F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, + G: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, + { + move |rti, e| { + if !f1(rti, e) { + return false; + } + if !f2(rti, e) { + return false; + } + true + } + } + + pub fn find_fast_public_nodes_filtered<'a, 'b, F>( + &self, + node_count: usize, + mut entry_filter: F, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, + { + self.inner + .read() + .find_fast_public_nodes_filtered(self.clone(), node_count, entry_filter) + } + + /// Retrieve up to N of each type of protocol capable nodes + pub fn find_bootstrap_nodes_filtered(&self, max_per_type: usize) -> Vec { + let protocol_types = vec![ + ProtocolType::UDP, + ProtocolType::TCP, + ProtocolType::WS, + ProtocolType::WSS, + ]; + let mut nodes_proto_v4 = vec![0usize, 0usize, 0usize, 0usize]; + let mut nodes_proto_v6 = vec![0usize, 0usize, 0usize, 0usize]; + + self.find_fastest_nodes( + // count + protocol_types.len() * 2 * max_per_type, + // filter + move |rti, _k: DHTKey, v: Option>| { + let entry = v.unwrap(); + entry.with(rti, |_rti, e| { + // skip nodes on our local network here + if e.has_node_info(RoutingDomain::LocalNetwork.into()) { + return false; + } + + // does it have some dial info we need? + let filter = |n: &NodeInfo| { + let mut keep = false; + for did in &n.dial_info_detail_list { + if matches!(did.dial_info.address_type(), AddressType::IPV4) { + for (n, protocol_type) in protocol_types.iter().enumerate() { + if nodes_proto_v4[n] < max_per_type + && did.dial_info.protocol_type() == *protocol_type + { + nodes_proto_v4[n] += 1; + keep = true; + } + } + } else if matches!(did.dial_info.address_type(), AddressType::IPV6) { + for (n, protocol_type) in protocol_types.iter().enumerate() { + if nodes_proto_v6[n] < max_per_type + && did.dial_info.protocol_type() == *protocol_type + { + nodes_proto_v6[n] += 1; + keep = true; + } + } + } + } + keep + }; + + e.node_info(RoutingDomain::PublicInternet) + .map(filter) + .unwrap_or(false) + }) + }, + // transform + |_rti, k: DHTKey, v: Option>| { + NodeRef::new(self.clone(), k, v.unwrap().clone(), None) + }, + ) + } + + pub fn find_peers_with_sort_and_filter<'a, 'b, F, C, T, O>( + &self, + node_count: usize, + cur_ts: u64, + mut filter: F, + compare: C, + mut transform: T, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + C: FnMut( + &'a RoutingTableInner, + &'b (DHTKey, Option>), + &'b (DHTKey, Option>), + ) -> core::cmp::Ordering, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + { + self.inner + .read() + .find_peers_with_sort_and_filter(node_count, cur_ts, filter, compare, transform) + } + + pub fn find_fastest_nodes<'a, T, F, O>( + &self, + node_count: usize, + mut filter: F, + transform: T, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + { + self.inner + .read() + .find_fastest_nodes(node_count, filter, transform) + } + + pub fn find_closest_nodes<'a, F, T, O>( + &self, + node_id: DHTKey, + filter: F, + mut transform: T, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + { + self.inner + .read() + .find_closest_nodes(node_id, filter, transform) + } + #[instrument(level = "trace", skip(self), ret)] + pub fn register_find_node_answer(&self, peers: Vec) -> Vec { + let node_id = self.node_id(); + + // register nodes we'd found + let mut out = Vec::::with_capacity(peers.len()); + for p in peers { + // if our own node if is in the list then ignore it, as we don't add ourselves to our own routing table + if p.node_id.key == node_id { + continue; + } + + // node can not be its own relay + if let Some(rpi) = &p.signed_node_info.node_info.relay_peer_info { + if rpi.node_id == p.node_id { + continue; + } + } + + // register the node if it's new + if let Some(nr) = self.register_node_with_signed_node_info( + RoutingDomain::PublicInternet, + p.node_id.key, + p.signed_node_info.clone(), + false, + ) { + out.push(nr); + } + } + out + } + + #[instrument(level = "trace", skip(self), ret, err)] + pub async fn find_node( + &self, + node_ref: NodeRef, + node_id: DHTKey, + ) -> EyreResult>> { + let rpc_processor = self.rpc_processor(); + + let res = network_result_try!( + rpc_processor + .clone() + .rpc_call_find_node(Destination::direct(node_ref), node_id) + .await? + ); + + // register nodes we'd found + Ok(NetworkResult::value( + self.register_find_node_answer(res.answer), + )) + } + + #[instrument(level = "trace", skip(self), ret, err)] + pub async fn find_self(&self, node_ref: NodeRef) -> EyreResult>> { + let node_id = self.node_id(); + self.find_node(node_ref, node_id).await + } + + #[instrument(level = "trace", skip(self), ret, err)] + pub async fn find_target(&self, node_ref: NodeRef) -> EyreResult>> { + let node_id = node_ref.node_id(); + self.find_node(node_ref, node_id).await + } + + #[instrument(level = "trace", skip(self))] + pub async fn reverse_find_node(&self, node_ref: NodeRef, wide: bool) { + // Ask bootstrap node to 'find' our own node so we can get some more nodes near ourselves + // and then contact those nodes to inform -them- that we exist + + // Ask bootstrap server for nodes closest to our own node + let closest_nodes = network_result_value_or_log!(debug match self.find_self(node_ref.clone()).await { + Err(e) => { + log_rtab!(error + "find_self failed for {:?}: {:?}", + &node_ref, e + ); + return; + } + Ok(v) => v, + } => { + return; + }); + + // Ask each node near us to find us as well + if wide { + for closest_nr in closest_nodes { + network_result_value_or_log!(debug match self.find_self(closest_nr.clone()).await { + Err(e) => { + log_rtab!(error + "find_self failed for {:?}: {:?}", + &closest_nr, e + ); + continue; + } + Ok(v) => v, + } => { + // Do nothing with non-values + continue; + }); + } + } + } + + pub fn make_public_internet_relay_node_filter(&self) -> impl Fn(&BucketEntryInner) -> bool { + // Get all our outbound protocol/address types + let outbound_dif = self.get_outbound_dial_info_filter(RoutingDomain::PublicInternet); + let mapped_port_info = self.get_low_level_port_info(); + + move |e: &BucketEntryInner| { + // Ensure this node is not on the local network + if e.has_node_info(RoutingDomain::LocalNetwork.into()) { + return false; + } + + // Disqualify nodes that don't cover all our inbound ports for tcp and udp + // as we need to be able to use the relay for keepalives for all nat mappings + let mut low_level_protocol_ports = mapped_port_info.low_level_protocol_ports.clone(); + + let can_serve_as_relay = e + .node_info(RoutingDomain::PublicInternet) + .map(|n| { + let dids = n.all_filtered_dial_info_details( + Some(DialInfoDetail::reliable_sort), // By default, choose reliable protocol for relay + |did| did.matches_filter(&outbound_dif), + ); + for did in &dids { + let pt = did.dial_info.protocol_type(); + let at = did.dial_info.address_type(); + if let Some((llpt, port)) = mapped_port_info.protocol_to_port.get(&(pt, at)) + { + low_level_protocol_ports.remove(&(*llpt, at, *port)); + } + } + low_level_protocol_ports.is_empty() + }) + .unwrap_or(false); + if !can_serve_as_relay { + return false; + } + + true + } + } + + #[instrument(level = "trace", skip(self), ret)] + pub fn find_inbound_relay( + &self, + routing_domain: RoutingDomain, + cur_ts: u64, + ) -> Option { + // Get relay filter function + let relay_node_filter = match routing_domain { + RoutingDomain::PublicInternet => self.make_public_internet_relay_node_filter(), + RoutingDomain::LocalNetwork => { + unimplemented!(); + } + }; + + // Go through all entries and find fastest entry that matches filter function + let inner = self.inner.read(); + let inner = &*inner; + let mut best_inbound_relay: Option<(DHTKey, Arc)> = None; + + // Iterate all known nodes for candidates + inner.with_entries(cur_ts, BucketEntryState::Unreliable, |rti, k, v| { + let v2 = v.clone(); + v.with(rti, |rti, e| { + // Ensure we have the node's status + if let Some(node_status) = e.node_status(routing_domain) { + // Ensure the node will relay + if node_status.will_relay() { + // Compare against previous candidate + if let Some(best_inbound_relay) = best_inbound_relay.as_mut() { + // Less is faster + let better = best_inbound_relay.1.with(rti, |_rti, best| { + BucketEntryInner::cmp_fastest_reliable(cur_ts, e, best) + == std::cmp::Ordering::Less + }); + // Now apply filter function and see if this node should be included + if better && relay_node_filter(e) { + *best_inbound_relay = (k, v2); + } + } else if relay_node_filter(e) { + // Always store the first candidate + best_inbound_relay = Some((k, v2)); + } + } + } + }); + // Don't end early, iterate through all entries + Option::<()>::None + }); + // Return the best inbound relay noderef + best_inbound_relay.map(|(k, e)| NodeRef::new(self.clone(), k, e, None)) } } diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index d3aa3ac5..19c14f9e 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -97,7 +97,6 @@ impl NodeRef { } // Operate on entry accessors - pub(super) fn operate(&self, f: F) -> T where F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, @@ -217,6 +216,9 @@ impl NodeRef { pub fn make_peer_info(&self, routing_domain: RoutingDomain) -> Option { self.operate(|_rti, e| e.make_peer_info(self.node_id(), routing_domain)) } + pub fn node_info(&self, routing_domain: RoutingDomain) -> Option { + self.operate(|_rti, e| e.node_info(routing_domain).cloned()) + } pub fn signed_node_info_has_valid_signature(&self, routing_domain: RoutingDomain) -> bool { self.operate(|_rti, e| { e.signed_node_info(routing_domain) @@ -371,26 +373,26 @@ impl NodeRef { pub fn stats_question_sent(&self, ts: u64, bytes: u64, expects_answer: bool) { self.operate_mut(|rti, e| { - rti.self_transfer_stats_accounting.add_up(bytes); + rti.transfer_stats_accounting().add_up(bytes); e.question_sent(ts, bytes, expects_answer); }) } pub fn stats_question_rcvd(&self, ts: u64, bytes: u64) { self.operate_mut(|rti, e| { - rti.self_transfer_stats_accounting.add_down(bytes); + rti.transfer_stats_accounting().add_down(bytes); e.question_rcvd(ts, bytes); }) } pub fn stats_answer_sent(&self, bytes: u64) { self.operate_mut(|rti, e| { - rti.self_transfer_stats_accounting.add_up(bytes); + rti.transfer_stats_accounting().add_up(bytes); e.answer_sent(bytes); }) } pub fn stats_answer_rcvd(&self, send_ts: u64, recv_ts: u64, bytes: u64) { self.operate_mut(|rti, e| { - rti.self_transfer_stats_accounting.add_down(bytes); - rti.self_latency_stats_accounting + rti.transfer_stats_accounting().add_down(bytes); + rti.latency_stats_accounting() .record_latency(recv_ts - send_ts); e.answer_rcvd(send_ts, recv_ts, bytes); }) diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 8228acb9..5dc639d8 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -14,7 +14,7 @@ pub struct SafetySpec { } /// Compiled route (safety route + private route) -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct CompiledRoute { /// The safety route attached to the private route safety_route: SafetyRoute, @@ -78,10 +78,6 @@ pub struct RouteSpecStoreCache { #[derive(Debug)] pub struct RouteSpecStore { - /// Our node id - node_id: DHTKey, - /// Our node id secret - node_id_secret: DHTKeySecret, /// Maximum number of hops in a route max_route_hop_count: usize, /// Default number of hops in a route @@ -179,8 +175,6 @@ impl RouteSpecStore { let c = config.get(); Self { - node_id: c.network.node_id, - node_id_secret: c.network.node_id_secret, max_route_hop_count: c.network.rpc.max_route_hop_count.into(), default_route_hop_count: c.network.rpc.default_route_hop_count.into(), content: RouteSpecStoreContent { @@ -198,8 +192,6 @@ impl RouteSpecStore { let rsstdb = table_store.open("RouteSpecStore", 1).await?; let content = rsstdb.load_cbor(0, b"content").await?.unwrap_or_default(); let mut rss = RouteSpecStore { - node_id: c.network.node_id, - node_id_secret: c.network.node_id_secret, max_route_hop_count: c.network.rpc.max_route_hop_count.into(), default_route_hop_count: c.network.rpc.default_route_hop_count.into(), content, @@ -295,6 +287,7 @@ impl RouteSpecStore { pub fn allocate_route( &mut self, rti: &RoutingTableInner, + routing_table: RoutingTable, reliable: bool, hop_count: usize, directions: DirectionSet, @@ -413,20 +406,12 @@ impl RouteSpecStore { }; // Pull the whole routing table in sorted order - let node_count = RoutingTable::get_entry_count_inner( - rti, + let node_count = rti.get_entry_count( RoutingDomain::PublicInternet.into(), BucketEntryState::Unreliable, ); - let nodes = RoutingTable::find_peers_with_sort_and_filter_inner( - rti, - self.node_id, - node_count, - cur_ts, - filter, - compare, - transform, - ); + let nodes = + rti.find_peers_with_sort_and_filter(node_count, cur_ts, filter, compare, transform); // If we couldn't find enough nodes, wait until we have more nodes in the routing table if nodes.len() < hop_count { @@ -450,15 +435,13 @@ impl RouteSpecStore { // Ensure this route is viable by checking that each node can contact the next one if directions.contains(Direction::Outbound) { - let our_node_info = - RoutingTable::get_own_node_info_inner(rti, RoutingDomain::PublicInternet); - let our_node_id = self.node_id; + let our_node_info = rti.get_own_node_info(RoutingDomain::PublicInternet); + let our_node_id = rti.node_id(); let mut previous_node = &(our_node_id, our_node_info); let mut reachable = true; for n in permutation { let current_node = nodes.get(*n).unwrap(); - let cm = RoutingTable::get_contact_method_inner( - rti, + let cm = rti.get_contact_method( RoutingDomain::PublicInternet, &previous_node.0, &previous_node.1, @@ -478,15 +461,13 @@ impl RouteSpecStore { } } if directions.contains(Direction::Inbound) { - let our_node_info = - RoutingTable::get_own_node_info_inner(rti, RoutingDomain::PublicInternet); - let our_node_id = self.node_id; + let our_node_info = rti.get_own_node_info(RoutingDomain::PublicInternet); + let our_node_id = rti.node_id(); let mut next_node = &(our_node_id, our_node_info); let mut reachable = true; for n in permutation.iter().rev() { let current_node = nodes.get(*n).unwrap(); - let cm = RoutingTable::get_contact_method_inner( - rti, + let cm = rti.get_contact_method( RoutingDomain::PublicInternet, &next_node.0, &next_node.1, @@ -522,7 +503,7 @@ impl RouteSpecStore { let hops = route_nodes.iter().map(|v| nodes[*v].0).collect(); let hop_node_refs = route_nodes .iter() - .map(|v| routing_table.lookup_node_ref(nodes[*v].0).unwrap()) + .map(|v| rti.lookup_node_ref(routing_table, nodes[*v].0).unwrap()) .collect(); let (public_key, secret_key) = generate_secret(); @@ -613,12 +594,15 @@ impl RouteSpecStore { ////////////////////////////////////////////////////////////////////// /// Compiles a safety route to the private route, with caching + /// Returns an Err() if the parameters are wrong + /// Returns Ok(None) if no allocation could happen at this time (not an error) pub fn compile_safety_route( &mut self, rti: &RoutingTableInner, + routing_table: RoutingTable, safety_spec: SafetySpec, private_route: PrivateRoute, - ) -> Result { + ) -> Result, RPCError> { let pr_hopcount = private_route.hop_count as usize; if pr_hopcount > self.max_route_hop_count { return Err(RPCError::internal("private route hop count too long")); @@ -647,7 +631,20 @@ impl RouteSpecStore { self.detail_mut(&sr_pubkey).unwrap() } else { // No route found, gotta allocate one - self.allocate_route(rti) + let sr_pubkey = match self + .allocate_route( + rti, + routing_table, + safety_spec.reliable, + safety_spec.hop_count, + Direction::Outbound.into(), + ) + .map_err(RPCError::internal)? + { + Some(pk) => pk, + None => return Ok(None), + }; + self.detail_mut(&sr_pubkey).unwrap() } }; @@ -787,7 +784,9 @@ impl RouteSpecStore { .detail_mut(&key) .ok_or_else(|| eyre!("route does not exist"))? .latency_stats_accounting; - self.detail_mut(&key).latency_stats = lsa.record_latency(latency); + self.detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .latency_stats = lsa.record_latency(latency); Ok(()) } diff --git a/veilid-core/src/routing_table/routing_domain_editor.rs b/veilid-core/src/routing_table/routing_domain_editor.rs index c507b3fa..65001c09 100644 --- a/veilid-core/src/routing_table/routing_domain_editor.rs +++ b/veilid-core/src/routing_table/routing_domain_editor.rs @@ -122,8 +122,7 @@ impl RoutingDomainEditor { let node_id = self.routing_table.node_id(); let mut inner = self.routing_table.inner.write(); - let inner = &mut *inner; - RoutingTable::with_routing_domain_mut(inner, self.routing_domain, |detail| { + inner.with_routing_domain_mut(self.routing_domain, |detail| { for change in self.changes { match change { RoutingDomainChange::ClearDialInfoDetails => { @@ -225,8 +224,8 @@ impl RoutingDomainEditor { } }); if changed { - RoutingTable::reset_all_seen_our_node_info(inner, self.routing_domain); - RoutingTable::reset_all_updated_since_last_network_change(inner); + inner.reset_all_seen_our_node_info(self.routing_domain); + inner.reset_all_updated_since_last_network_change(); } } if changed && self.send_node_info_updates { diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs new file mode 100644 index 00000000..f0bf9a24 --- /dev/null +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -0,0 +1,1033 @@ +use super::*; + +const RECENT_PEERS_TABLE_SIZE: usize = 64; + +////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, Copy)] +pub struct RecentPeersEntry { + pub last_connection: ConnectionDescriptor, +} + +/// RoutingTable rwlock-internal data +pub struct RoutingTableInner { + /// Extra pointer to unlocked members to simplify access + pub(super) unlocked_inner: Arc, + /// Routing table buckets that hold entries + pub(super) buckets: Vec, + /// A fast counter for the number of entries in the table, total + pub(super) bucket_entry_count: usize, + /// The public internet routing domain + pub(super) public_internet_routing_domain: PublicInternetRoutingDomainDetail, + /// The dial info we use on the local network + pub(super) local_network_routing_domain: LocalNetworkRoutingDomainDetail, + /// Interim accounting mechanism for this node's RPC latency to any other node + pub(super) self_latency_stats_accounting: LatencyStatsAccounting, + /// Interim accounting mechanism for the total bandwidth to/from this node + pub(super) self_transfer_stats_accounting: TransferStatsAccounting, + /// Statistics about the total bandwidth to/from this node + pub(super) self_transfer_stats: TransferStatsDownUp, + /// Peers we have recently communicated with + pub(super) recent_peers: LruCache, + /// Storage for private/safety RouteSpecs + pub(super) route_spec_store: RouteSpecStore, +} + +impl RoutingTableInner { + pub fn new(unlocked_inner: Arc) -> RoutingTableInner { + RoutingTableInner { + unlocked_inner, + buckets: Vec::new(), + public_internet_routing_domain: PublicInternetRoutingDomainDetail::default(), + local_network_routing_domain: LocalNetworkRoutingDomainDetail::default(), + bucket_entry_count: 0, + self_latency_stats_accounting: LatencyStatsAccounting::new(), + self_transfer_stats_accounting: TransferStatsAccounting::new(), + self_transfer_stats: TransferStatsDownUp::default(), + recent_peers: LruCache::new(RECENT_PEERS_TABLE_SIZE), + route_spec_store: RouteSpecStore::new(unlocked_inner.config.clone()), + } + } + + pub fn network_manager(&self) -> NetworkManager { + self.unlocked_inner.network_manager.clone() + } + pub fn rpc_processor(&self) -> RPCProcessor { + self.network_manager().rpc_processor() + } + + pub fn node_id(&self) -> DHTKey { + self.unlocked_inner.node_id + } + + pub fn node_id_secret(&self) -> DHTKeySecret { + self.unlocked_inner.node_id_secret + } + + pub fn config(&self) -> VeilidConfig { + self.unlocked_inner.config.clone() + } + + pub fn transfer_stats_accounting(&mut self) -> &TransferStatsAccounting { + &mut self.self_transfer_stats_accounting + } + pub fn latency_stats_accounting(&mut self) -> &LatencyStatsAccounting { + &mut self.self_latency_stats_accounting + } + + pub fn routing_domain_for_address(&self, address: Address) -> Option { + for rd in RoutingDomain::all() { + let can_contain = self.with_routing_domain(rd, |rdd| rdd.can_contain_address(address)); + if can_contain { + return Some(rd); + } + } + None + } + + pub fn with_routing_domain(&self, domain: RoutingDomain, f: F) -> R + where + F: FnOnce(&dyn RoutingDomainDetail) -> R, + { + match domain { + RoutingDomain::PublicInternet => f(&self.public_internet_routing_domain), + RoutingDomain::LocalNetwork => f(&self.local_network_routing_domain), + } + } + + pub fn with_routing_domain_mut(&mut self, domain: RoutingDomain, f: F) -> R + where + F: FnOnce(&mut dyn RoutingDomainDetail) -> R, + { + match domain { + RoutingDomain::PublicInternet => f(&mut self.public_internet_routing_domain), + RoutingDomain::LocalNetwork => f(&mut self.local_network_routing_domain), + } + } + + pub fn with_route_spec_store_mut(&mut self, f: F) -> R + where + F: FnOnce(&mut RouteSpecStore, &mut RoutingTableInner) -> R, + { + f(&mut self.route_spec_store, self) + } + + pub fn with_route_spec_store(&self, f: F) -> R + where + F: FnOnce(&RouteSpecStore, &RoutingTableInner) -> R, + { + f(&self.route_spec_store, self) + } + + pub fn relay_node(&self, domain: RoutingDomain) -> Option { + self.with_routing_domain(domain, |rd| rd.common().relay_node()) + } + + pub fn has_dial_info(&self, domain: RoutingDomain) -> bool { + self.with_routing_domain(domain, |rd| !rd.common().dial_info_details().is_empty()) + } + + pub fn dial_info_details(&self, domain: RoutingDomain) -> Vec { + self.with_routing_domain(domain, |rd| rd.common().dial_info_details().clone()) + } + + pub fn first_filtered_dial_info_detail( + &self, + routing_domain_set: RoutingDomainSet, + filter: &DialInfoFilter, + ) -> Option { + for routing_domain in routing_domain_set { + let did = self.with_routing_domain(routing_domain, |rd| { + for did in rd.common().dial_info_details() { + if did.matches_filter(filter) { + return Some(did.clone()); + } + } + None + }); + if did.is_some() { + return did; + } + } + None + } + + pub fn all_filtered_dial_info_details( + &self, + routing_domain_set: RoutingDomainSet, + filter: &DialInfoFilter, + ) -> Vec { + let mut ret = Vec::new(); + for routing_domain in routing_domain_set { + self.with_routing_domain(routing_domain, |rd| { + for did in rd.common().dial_info_details() { + if did.matches_filter(filter) { + ret.push(did.clone()); + } + } + }); + } + ret.remove_duplicates(); + ret + } + + pub fn ensure_dial_info_is_valid(&self, domain: RoutingDomain, dial_info: &DialInfo) -> bool { + let address = dial_info.socket_address().address(); + let can_contain_address = + self.with_routing_domain(domain, |rd| rd.can_contain_address(address)); + + if !can_contain_address { + log_rtab!(debug "can not add dial info to this routing domain"); + return false; + } + if !dial_info.is_valid() { + log_rtab!(debug + "shouldn't be registering invalid addresses: {:?}", + dial_info + ); + return false; + } + true + } + + pub fn node_info_is_valid_in_routing_domain( + &self, + routing_domain: RoutingDomain, + node_info: &NodeInfo, + ) -> bool { + // Should not be passing around nodeinfo with an invalid network class + if matches!(node_info.network_class, NetworkClass::Invalid) { + return false; + } + // Ensure all of the dial info works in this routing domain + for did in &node_info.dial_info_detail_list { + if !self.ensure_dial_info_is_valid(routing_domain, &did.dial_info) { + return false; + } + } + // Ensure the relay is also valid in this routing domain if it is provided + if let Some(relay_peer_info) = node_info.relay_peer_info.as_ref() { + let relay_ni = &relay_peer_info.signed_node_info.node_info; + if !self.node_info_is_valid_in_routing_domain(routing_domain, relay_ni) { + return false; + } + } + true + } + + #[instrument(level = "trace", skip(self), ret)] + pub fn get_contact_method( + &self, + routing_domain: RoutingDomain, + node_a_id: &DHTKey, + node_a: &NodeInfo, + node_b_id: &DHTKey, + node_b: &NodeInfo, + dial_info_filter: DialInfoFilter, + reliable: bool, + ) -> ContactMethod { + self.with_routing_domain(routing_domain, |rdd| { + rdd.get_contact_method( + self, + node_a_id, + node_a, + node_b_id, + node_b, + dial_info_filter, + reliable, + ) + }) + } + + pub fn reset_all_seen_our_node_info(&mut self, routing_domain: RoutingDomain) { + let cur_ts = intf::get_timestamp(); + self.with_entries_mut(cur_ts, BucketEntryState::Dead, |rti, _, v| { + v.with_mut(rti, |_rti, e| { + e.set_seen_our_node_info(routing_domain, false); + }); + Option::<()>::None + }); + } + + pub fn reset_all_updated_since_last_network_change(&mut self) { + let cur_ts = intf::get_timestamp(); + self.with_entries_mut(cur_ts, BucketEntryState::Dead, |rti, _, v| { + v.with_mut(rti, |_rti, e| { + e.set_updated_since_last_network_change(false) + }); + Option::<()>::None + }); + } + + /// Return a copy of our node's peerinfo + pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo { + self.with_routing_domain(routing_domain, |rdd| { + rdd.common().with_peer_info(|pi| pi.clone()) + }) + } + + /// Return a copy of our node's signednodeinfo + pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedNodeInfo { + self.with_routing_domain(routing_domain, |rdd| { + rdd.common() + .with_peer_info(|pi| pi.signed_node_info.clone()) + }) + } + + /// Return a copy of our node's nodeinfo + pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo { + self.with_routing_domain(routing_domain, |rdd| { + rdd.common() + .with_peer_info(|pi| pi.signed_node_info.node_info.clone()) + }) + } + + /// Return our currently registered network class + pub fn has_valid_own_node_info(&self, routing_domain: RoutingDomain) -> bool { + self.with_routing_domain(routing_domain, |rdd| rdd.common().has_valid_own_node_info()) + } + + /// Return the domain's currently registered network class + pub fn get_network_class(&self, routing_domain: RoutingDomain) -> Option { + self.with_routing_domain(routing_domain, |rdd| rdd.common().network_class()) + } + + /// Return the domain's filter for what we can receivein the form of a dial info filter + pub fn get_inbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { + self.with_routing_domain(routing_domain, |rdd| { + rdd.common().inbound_dial_info_filter() + }) + } + + /// Return the domain's filter for what we can receive in the form of a node ref filter + pub fn get_inbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { + let dif = self.get_inbound_dial_info_filter(routing_domain); + NodeRefFilter::new() + .with_routing_domain(routing_domain) + .with_dial_info_filter(dif) + } + + /// Return the domain's filter for what we can send out in the form of a dial info filter + pub fn get_outbound_dial_info_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { + self.with_routing_domain(routing_domain, |rdd| { + rdd.common().outbound_dial_info_filter() + }) + } + /// Return the domain's filter for what we can receive in the form of a node ref filter + pub fn get_outbound_node_ref_filter(&self, routing_domain: RoutingDomain) -> NodeRefFilter { + let dif = self.get_outbound_dial_info_filter(routing_domain); + NodeRefFilter::new() + .with_routing_domain(routing_domain) + .with_dial_info_filter(dif) + } + + fn bucket_depth(index: usize) -> usize { + match index { + 0 => 256, + 1 => 128, + 2 => 64, + 3 => 32, + 4 => 16, + 5 => 8, + 6 => 4, + 7 => 4, + 8 => 4, + 9 => 4, + _ => 4, + } + } + + pub fn init(&mut self, routing_table: RoutingTable) -> EyreResult<()> { + // Size the buckets (one per bit) + self.buckets.reserve(DHT_KEY_LENGTH * 8); + for _ in 0..DHT_KEY_LENGTH * 8 { + let bucket = Bucket::new(routing_table.clone()); + self.buckets.push(bucket); + } + Ok(()) + } + + pub fn terminate(&mut self) { + // + } + + pub fn configure_local_network_routing_domain( + &mut self, + local_networks: Vec<(IpAddr, IpAddr)>, + ) { + log_net!(debug "configure_local_network_routing_domain: {:#?}", local_networks); + + let changed = self + .local_network_routing_domain + .set_local_networks(local_networks); + + // If the local network topology has changed, nuke the existing local node info and let new local discovery happen + if changed { + let cur_ts = intf::get_timestamp(); + self.with_entries_mut(cur_ts, BucketEntryState::Dead, |rti, _, e| { + e.with_mut(rti, |_rti, e| { + e.clear_signed_node_info(RoutingDomain::LocalNetwork); + e.set_seen_our_node_info(RoutingDomain::LocalNetwork, false); + e.set_updated_since_last_network_change(false); + }); + Option::<()>::None + }); + } + } + + /// Attempt to empty the routing table + /// should only be performed when there are no node_refs (detached) + pub fn purge_buckets(&mut self) { + log_rtab!( + "Starting routing table buckets purge. Table currently has {} nodes", + self.bucket_entry_count + ); + for bucket in &self.buckets { + bucket.kick(self, 0); + } + log_rtab!(debug + "Routing table buckets purge complete. Routing table now has {} nodes", + self.bucket_entry_count + ); + } + + /// Attempt to remove last_connections from entries + pub fn purge_last_connections(&mut self) { + log_rtab!( + "Starting routing table last_connections purge. Table currently has {} nodes", + self.bucket_entry_count + ); + for bucket in &self.buckets { + for entry in bucket.entries() { + entry.1.with_mut(self, |_rti, e| { + e.clear_last_connections(); + }); + } + } + log_rtab!(debug + "Routing table last_connections purge complete. Routing table now has {} nodes", + self.bucket_entry_count + ); + } + + /// Attempt to settle buckets and remove entries down to the desired number + /// which may not be possible due extant NodeRefs + pub fn kick_bucket(&mut self, idx: usize) { + let bucket = &mut self.buckets[idx]; + let bucket_depth = Self::bucket_depth(idx); + + if let Some(dead_node_ids) = bucket.kick(self, bucket_depth) { + // Remove counts + self.bucket_entry_count -= dead_node_ids.len(); + log_rtab!(debug "Routing table now has {} nodes", self.bucket_entry_count); + + // Now purge the routing table inner vectors + //let filter = |k: &DHTKey| dead_node_ids.contains(k); + //inner.closest_reliable_nodes.retain(filter); + //inner.fastest_reliable_nodes.retain(filter); + //inner.closest_nodes.retain(filter); + //inner.fastest_nodes.retain(filter); + } + } + + pub fn find_bucket_index(&self, node_id: DHTKey) -> usize { + distance(&node_id, &self.unlocked_inner.node_id) + .first_nonzero_bit() + .unwrap() + } + + pub fn get_entry_count( + &self, + routing_domain_set: RoutingDomainSet, + min_state: BucketEntryState, + ) -> usize { + let mut count = 0usize; + let cur_ts = intf::get_timestamp(); + self.with_entries(cur_ts, min_state, |rti, _, e| { + if e.with(rti, |_rti, e| e.best_routing_domain(routing_domain_set)) + .is_some() + { + count += 1; + } + Option::<()>::None + }); + count + } + + pub fn with_entries) -> Option>( + &self, + cur_ts: u64, + min_state: BucketEntryState, + mut f: F, + ) -> Option { + for bucket in &self.buckets { + for entry in bucket.entries() { + if entry.1.with(self, |_rti, e| e.state(cur_ts) >= min_state) { + if let Some(out) = f(self, *entry.0, entry.1.clone()) { + return Some(out); + } + } + } + } + None + } + + pub fn with_entries_mut< + T, + F: FnMut(&mut RoutingTableInner, DHTKey, Arc) -> Option, + >( + &mut self, + cur_ts: u64, + min_state: BucketEntryState, + mut f: F, + ) -> Option { + for bucket in &self.buckets { + for entry in bucket.entries() { + if entry.1.with(self, |_rti, e| e.state(cur_ts) >= min_state) { + if let Some(out) = f(self, *entry.0, entry.1.clone()) { + return Some(out); + } + } + } + } + None + } + + pub fn get_nodes_needing_updates( + &self, + outer_self: RoutingTable, + routing_domain: RoutingDomain, + cur_ts: u64, + all: bool, + ) -> Vec { + let mut node_refs = Vec::::with_capacity(self.bucket_entry_count); + self.with_entries(cur_ts, BucketEntryState::Unreliable, |rti, k, v| { + // Only update nodes that haven't seen our node info yet + if all || !v.with(rti, |_rti, e| e.has_seen_our_node_info(routing_domain)) { + node_refs.push(NodeRef::new( + outer_self.clone(), + k, + v, + Some(NodeRefFilter::new().with_routing_domain(routing_domain)), + )); + } + Option::<()>::None + }); + node_refs + } + + pub fn get_nodes_needing_ping( + &self, + outer_self: RoutingTable, + routing_domain: RoutingDomain, + cur_ts: u64, + ) -> Vec { + // Collect relay nodes + let opt_relay_id = self.with_routing_domain(routing_domain, |rd| { + rd.common().relay_node().map(|rn| rn.node_id()) + }); + + // Collect all entries that are 'needs_ping' and have some node info making them reachable somehow + let mut node_refs = Vec::::with_capacity(self.bucket_entry_count); + self.with_entries(cur_ts, BucketEntryState::Unreliable, |rti, k, v| { + if v.with(rti, |_rti, e| { + e.has_node_info(routing_domain.into()) + && e.needs_ping(cur_ts, opt_relay_id == Some(k)) + }) { + node_refs.push(NodeRef::new( + outer_self.clone(), + k, + v, + Some(NodeRefFilter::new().with_routing_domain(routing_domain)), + )); + } + Option::<()>::None + }); + node_refs + } + + pub fn get_all_nodes(&self, outer_self: RoutingTable, cur_ts: u64) -> Vec { + let mut node_refs = Vec::::with_capacity(self.bucket_entry_count); + self.with_entries(cur_ts, BucketEntryState::Unreliable, |_rti, k, v| { + node_refs.push(NodeRef::new(outer_self.clone(), k, v, None)); + Option::<()>::None + }); + node_refs + } + + /// Create a node reference, possibly creating a bucket entry + /// the 'update_func' closure is called on the node, and, if created, + /// in a locked fashion as to ensure the bucket entry state is always valid + pub fn create_node_ref( + &mut self, + outer_self: RoutingTable, + node_id: DHTKey, + update_func: F, + ) -> Option + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner), + { + // Ensure someone isn't trying register this node itself + if node_id == self.node_id() { + log_rtab!(debug "can't register own node"); + return None; + } + + // Look up existing entry + let idx = self.find_bucket_index(node_id); + let noderef = { + let bucket = &self.buckets[idx]; + let entry = bucket.entry(&node_id); + entry.map(|e| NodeRef::new(outer_self.clone(), node_id, e, None)) + }; + + // If one doesn't exist, insert into bucket, possibly evicting a bucket member + let noderef = match noderef { + None => { + // Make new entry + self.bucket_entry_count += 1; + let cnt = self.bucket_entry_count; + let bucket = &mut self.buckets[idx]; + let nr = bucket.add_entry(node_id); + + // Update the entry + let entry = bucket.entry(&node_id).unwrap(); + entry.with_mut(self, update_func); + + // Kick the bucket + self.unlocked_inner.kick_queue.lock().insert(idx); + log_rtab!(debug "Routing table now has {} nodes, {} live", cnt, self.get_entry_count(RoutingDomainSet::all(), BucketEntryState::Unreliable)); + + nr + } + Some(nr) => { + // Update the entry + let bucket = &mut self.buckets[idx]; + let entry = bucket.entry(&node_id).unwrap(); + entry.with_mut(self, update_func); + + nr + } + }; + + Some(noderef) + } + + /// Resolve an existing routing table entry and return a reference to it + pub fn lookup_node_ref(&self, outer_self: RoutingTable, node_id: DHTKey) -> Option { + if node_id == self.unlocked_inner.node_id { + log_rtab!(debug "can't look up own node id in routing table"); + return None; + } + let idx = self.find_bucket_index(node_id); + let bucket = &self.buckets[idx]; + bucket + .entry(&node_id) + .map(|e| NodeRef::new(outer_self, node_id, e, None)) + } + + /// Resolve an existing routing table entry and return a filtered reference to it + pub fn lookup_and_filter_noderef( + &self, + outer_self: RoutingTable, + node_id: DHTKey, + routing_domain_set: RoutingDomainSet, + dial_info_filter: DialInfoFilter, + ) -> Option { + let nr = self.lookup_node_ref(outer_self, node_id)?; + Some( + nr.filtered_clone( + NodeRefFilter::new() + .with_dial_info_filter(dial_info_filter) + .with_routing_domain_set(routing_domain_set), + ), + ) + } + + /// Shortcut function to add a node to our routing table if it doesn't exist + /// and add the dial info we have for it. Returns a noderef filtered to + /// the routing domain in which this node was registered for convenience. + pub fn register_node_with_signed_node_info( + &mut self, + outer_self: RoutingTable, + routing_domain: RoutingDomain, + node_id: DHTKey, + signed_node_info: SignedNodeInfo, + allow_invalid: bool, + ) -> Option { + // validate signed node info is not something malicious + if node_id == self.node_id() { + log_rtab!(debug "can't register own node id in routing table"); + return None; + } + if let Some(rpi) = &signed_node_info.node_info.relay_peer_info { + if rpi.node_id.key == node_id { + log_rtab!(debug "node can not be its own relay"); + return None; + } + } + if !allow_invalid { + // verify signature + if !signed_node_info.has_valid_signature() { + log_rtab!(debug "signed node info for {} has invalid signature", node_id); + return None; + } + // verify signed node info is valid in this routing domain + if !self + .node_info_is_valid_in_routing_domain(routing_domain, &signed_node_info.node_info) + { + log_rtab!(debug "signed node info for {} not valid in the {:?} routing domain", node_id, routing_domain); + return None; + } + } + + self.create_node_ref(outer_self, node_id, |_rti, e| { + e.update_signed_node_info(routing_domain, signed_node_info); + }) + .map(|mut nr| { + nr.set_filter(Some( + NodeRefFilter::new().with_routing_domain(routing_domain), + )); + nr + }) + } + + /// Shortcut function to add a node to our routing table if it doesn't exist + /// and add the last peer address we have for it, since that's pretty common + pub fn register_node_with_existing_connection( + &mut self, + outer_self: RoutingTable, + node_id: DHTKey, + descriptor: ConnectionDescriptor, + timestamp: u64, + ) -> Option { + let out = self.create_node_ref(outer_self, node_id, |_rti, e| { + // this node is live because it literally just connected to us + e.touch_last_seen(timestamp); + }); + if let Some(nr) = &out { + // set the most recent node address for connection finding and udp replies + nr.set_last_connection(descriptor, timestamp); + } + out + } + + ////////////////////////////////////////////////////////////////////// + // Routing Table Health Metrics + + pub fn get_routing_table_health(&self) -> RoutingTableHealth { + let mut health = RoutingTableHealth::default(); + let cur_ts = intf::get_timestamp(); + for bucket in &self.buckets { + for (_, v) in bucket.entries() { + match v.with(self, |_rti, e| e.state(cur_ts)) { + BucketEntryState::Reliable => { + health.reliable_entry_count += 1; + } + BucketEntryState::Unreliable => { + health.unreliable_entry_count += 1; + } + BucketEntryState::Dead => { + health.dead_entry_count += 1; + } + } + } + } + health + } + + pub fn get_recent_peers( + &mut self, + outer_self: RoutingTable, + ) -> Vec<(DHTKey, RecentPeersEntry)> { + let mut recent_peers = Vec::new(); + let mut dead_peers = Vec::new(); + let mut out = Vec::new(); + + // collect all recent peers + for (k, _v) in &self.recent_peers { + recent_peers.push(*k); + } + + // look up each node and make sure the connection is still live + // (uses same logic as send_data, ensuring last_connection works for UDP) + for e in &recent_peers { + let mut dead = true; + if let Some(nr) = self.lookup_node_ref(outer_self.clone(), *e) { + if let Some(last_connection) = nr.last_connection() { + out.push((*e, RecentPeersEntry { last_connection })); + dead = false; + } + } + if dead { + dead_peers.push(e); + } + } + + // purge dead recent peers + for d in dead_peers { + self.recent_peers.remove(d); + } + + out + } + + pub fn touch_recent_peer(&self, node_id: DHTKey, last_connection: ConnectionDescriptor) { + self.recent_peers + .insert(node_id, RecentPeersEntry { last_connection }); + } + + ////////////////////////////////////////////////////////////////////// + // Find Nodes + + // Retrieve the fastest nodes in the routing table matching an entry filter + pub fn find_fast_public_nodes_filtered<'a, 'b, F>( + &self, + outer_self: RoutingTable, + node_count: usize, + mut entry_filter: F, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, + { + self.find_fastest_nodes( + // count + node_count, + // filter + |rti, _k: DHTKey, v: Option>| { + let entry = v.unwrap(); + entry.with(rti, |rti, e| { + // skip nodes on local network + if e.node_info(RoutingDomain::LocalNetwork).is_some() { + return false; + } + // skip nodes not on public internet + if e.node_info(RoutingDomain::PublicInternet).is_none() { + return false; + } + // skip nodes that dont match entry filter + entry_filter(rti, e) + }) + }, + // transform + |_rti, k: DHTKey, v: Option>| { + NodeRef::new(outer_self.clone(), k, v.unwrap().clone(), None) + }, + ) + } + + pub fn filter_has_valid_signed_node_info( + &self, + routing_domain: RoutingDomain, + has_valid_own_node_info: bool, + v: Option>, + ) -> bool { + match v { + None => has_valid_own_node_info, + Some(entry) => entry.with(self, |_rti, e| { + e.signed_node_info(routing_domain.into()) + .map(|sni| sni.has_valid_signature()) + .unwrap_or(false) + }), + } + } + + pub fn transform_to_peer_info( + &self, + routing_domain: RoutingDomain, + own_peer_info: PeerInfo, + k: DHTKey, + v: Option>, + ) -> PeerInfo { + match v { + None => own_peer_info, + Some(entry) => entry.with(self, |_rti, e| e.make_peer_info(k, routing_domain).unwrap()), + } + } + + pub fn find_peers_with_sort_and_filter<'a, 'b, F, C, T, O>( + &self, + node_count: usize, + cur_ts: u64, + mut filter: F, + compare: C, + mut transform: T, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + C: FnMut( + &'a RoutingTableInner, + &'b (DHTKey, Option>), + &'b (DHTKey, Option>), + ) -> core::cmp::Ordering, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + { + // collect all the nodes for sorting + let mut nodes = + Vec::<(DHTKey, Option>)>::with_capacity(self.bucket_entry_count + 1); + + // add our own node (only one of there with the None entry) + if filter(self, self.unlocked_inner.node_id, None) { + nodes.push((self.unlocked_inner.node_id, None)); + } + + // add all nodes from buckets + self.with_entries(cur_ts, BucketEntryState::Unreliable, |rti, k, v| { + // Apply filter + if filter(rti, k, Some(v.clone())) { + nodes.push((k, Some(v.clone()))); + } + Option::<()>::None + }); + + // sort by preference for returning nodes + nodes.sort_by(|a, b| compare(self, a, b)); + + // return transformed vector for filtered+sorted nodes + let cnt = usize::min(node_count, nodes.len()); + let mut out = Vec::::with_capacity(cnt); + for node in nodes { + let val = transform(self, node.0, node.1); + out.push(val); + } + + out + } + + pub fn find_fastest_nodes<'a, T, F, O>( + &self, + node_count: usize, + mut filter: F, + transform: T, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + { + let cur_ts = intf::get_timestamp(); + let out = self.find_peers_with_sort_and_filter( + node_count, + cur_ts, + // filter + |rti, k, v| { + if let Some(entry) = &v { + // always filter out dead nodes + if entry.with(rti, |_rti, e| e.state(cur_ts) == BucketEntryState::Dead) { + false + } else { + filter(rti, k, v) + } + } else { + // always filter out self peer, as it is irrelevant to the 'fastest nodes' search + false + } + }, + // sort + |rti, (a_key, a_entry), (b_key, b_entry)| { + // same nodes are always the same + if a_key == b_key { + return core::cmp::Ordering::Equal; + } + // our own node always comes last (should not happen, here for completeness) + if a_entry.is_none() { + return core::cmp::Ordering::Greater; + } + if b_entry.is_none() { + return core::cmp::Ordering::Less; + } + // reliable nodes come first + let ae = a_entry.as_ref().unwrap(); + let be = b_entry.as_ref().unwrap(); + ae.with(rti, |rti, ae| { + be.with(rti, |_rti, be| { + let ra = ae.check_reliable(cur_ts); + let rb = be.check_reliable(cur_ts); + if ra != rb { + if ra { + return core::cmp::Ordering::Less; + } else { + return core::cmp::Ordering::Greater; + } + } + + // latency is the next metric, closer nodes first + let a_latency = match ae.peer_stats().latency.as_ref() { + None => { + // treat unknown latency as slow + return core::cmp::Ordering::Greater; + } + Some(l) => l, + }; + let b_latency = match be.peer_stats().latency.as_ref() { + None => { + // treat unknown latency as slow + return core::cmp::Ordering::Less; + } + Some(l) => l, + }; + // Sort by average latency + a_latency.average.cmp(&b_latency.average) + }) + }) + }, + // transform, + transform, + ); + out + } + + pub fn find_closest_nodes<'a, F, T, O>( + &self, + node_id: DHTKey, + filter: F, + mut transform: T, + ) -> Vec + where + F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, + T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + { + let cur_ts = intf::get_timestamp(); + let node_count = { + let config = self.config(); + let c = config.get(); + c.network.dht.max_find_node_count as usize + }; + let out = self.find_peers_with_sort_and_filter( + node_count, + cur_ts, + // filter + filter, + // sort + |rti, (a_key, a_entry), (b_key, b_entry)| { + // same nodes are always the same + if a_key == b_key { + return core::cmp::Ordering::Equal; + } + + // reliable nodes come first, pessimistically treating our own node as unreliable + let ra = a_entry + .as_ref() + .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); + let rb = b_entry + .as_ref() + .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); + if ra != rb { + if ra { + return core::cmp::Ordering::Less; + } else { + return core::cmp::Ordering::Greater; + } + } + + // distance is the next metric, closer nodes first + let da = distance(a_key, &node_id); + let db = distance(b_key, &node_id); + da.cmp(&db) + }, + // transform, + &mut transform, + ); + log_rtab!(">> find_closest_nodes: node count = {}", out.len()); + out + } +} diff --git a/veilid-core/src/routing_table/tasks.rs b/veilid-core/src/routing_table/tasks.rs index d43ed5a4..f651d30e 100644 --- a/veilid-core/src/routing_table/tasks.rs +++ b/veilid-core/src/routing_table/tasks.rs @@ -47,7 +47,7 @@ impl RoutingTable { .collect(); let mut inner = self.inner.write(); for idx in kick_queue { - Self::kick_bucket(&mut *inner, idx) + inner.kick_bucket(idx) } Ok(()) } diff --git a/veilid-core/src/rpc_processor/coders/private_safety_route.rs b/veilid-core/src/rpc_processor/coders/private_safety_route.rs index f96261cb..50604970 100644 --- a/veilid-core/src/rpc_processor/coders/private_safety_route.rs +++ b/veilid-core/src/rpc_processor/coders/private_safety_route.rs @@ -24,14 +24,42 @@ pub fn encode_route_hop_data( Ok(()) } +pub fn decode_route_hop_data( + reader: &veilid_capnp::route_hop_data::Reader, +) -> Result { + let nonce = decode_nonce( + &reader + .reborrow() + .get_nonce() + .map_err(RPCError::map_protocol("invalid nonce in route hop data"))?, + ); + + let blob = reader + .reborrow() + .get_blob() + .map_err(RPCError::map_protocol("invalid blob in route hop data"))? + .to_vec(); + + Ok(RouteHopData { nonce, blob }) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + pub fn encode_route_hop( route_hop: &RouteHop, builder: &mut veilid_capnp::route_hop::Builder, ) -> Result<(), RPCError> { - encode_node_dial_info( - &route_hop.dial_info, - &mut builder.reborrow().init_dial_info(), - )?; + let node_builder = builder.reborrow().init_node(); + match &route_hop.node { + RouteNode::NodeId(ni) => { + let ni_builder = node_builder.init_node_id(); + encode_public_key(&ni.key, &mut ni_builder)?; + } + RouteNode::PeerInfo(pi) => { + let pi_builder = node_builder.init_peer_info(); + encode_peer_info(&pi, &mut pi_builder)?; + } + } if let Some(rhd) = &route_hop.next_hop { let mut rhd_builder = builder.reborrow().init_next_hop(); encode_route_hop_data(rhd, &mut rhd_builder)?; @@ -39,6 +67,36 @@ pub fn encode_route_hop( Ok(()) } +pub fn decode_route_hop(reader: &veilid_capnp::route_hop::Reader) -> Result { + let n_reader = reader.reborrow().get_node(); + let node = match n_reader.which().map_err(RPCError::protocol)? { + veilid_capnp::route_hop::node::Which::NodeId(ni) => { + let ni_reader = ni.map_err(RPCError::protocol)?; + RouteNode::NodeId(NodeId::new(decode_public_key(&ni_reader))) + } + veilid_capnp::route_hop::node::Which::PeerInfo(pi) => { + let pi_reader = pi.map_err(RPCError::protocol)?; + RouteNode::PeerInfo( + decode_peer_info(&pi_reader, true) + .map_err(RPCError::map_protocol("invalid peer info in route hop"))?, + ) + } + }; + + let next_hop = if reader.has_next_hop() { + let rhd_reader = reader + .get_next_hop() + .map_err(RPCError::map_protocol("invalid next hop in route hop"))?; + Some(decode_route_hop_data(&rhd_reader)?) + } else { + None + }; + + Ok(RouteHop { node, next_hop }) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + pub fn encode_private_route( private_route: &PrivateRoute, builder: &mut veilid_capnp::private_route::Builder, @@ -48,7 +106,7 @@ pub fn encode_private_route( &mut builder.reborrow().init_public_key(), )?; builder.set_hop_count(private_route.hop_count); - if let Some(rh) = &private_route.hops { + if let Some(rh) = &private_route.first_hop { let mut rh_builder = builder.reborrow().init_first_hop(); encode_route_hop(rh, &mut rh_builder)?; }; @@ -56,6 +114,31 @@ pub fn encode_private_route( Ok(()) } +pub fn decode_private_route( + reader: &veilid_capnp::private_route::Reader, +) -> Result { + let public_key = decode_public_key(&reader.get_public_key().map_err( + RPCError::map_protocol("invalid public key in private route"), + )?); + let hop_count = reader.get_hop_count(); + let first_hop = if reader.has_first_hop() { + let rh_reader = reader + .get_first_hop() + .map_err(RPCError::map_protocol("invalid first hop in private route"))?; + Some(decode_route_hop(&rh_reader)?) + } else { + None + }; + + Ok(PrivateRoute { + public_key, + hop_count, + first_hop, + }) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + pub fn encode_safety_route( safety_route: &SafetyRoute, builder: &mut veilid_capnp::safety_route::Builder, @@ -80,71 +163,6 @@ pub fn encode_safety_route( Ok(()) } -pub fn decode_route_hop_data( - reader: &veilid_capnp::route_hop_data::Reader, -) -> Result { - let nonce = decode_nonce( - &reader - .reborrow() - .get_nonce() - .map_err(RPCError::map_protocol("invalid nonce in route hop data"))?, - ); - - let blob = reader - .reborrow() - .get_blob() - .map_err(RPCError::map_protocol("invalid blob in route hop data"))? - .to_vec(); - - Ok(RouteHopData { nonce, blob }) -} - -pub fn decode_route_hop(reader: &veilid_capnp::route_hop::Reader) -> Result { - let dial_info = decode_node_dial_info( - &reader - .reborrow() - .get_dial_info() - .map_err(RPCError::map_protocol("invalid dial info in route hop"))?, - )?; - - let next_hop = if reader.has_next_hop() { - let rhd_reader = reader - .get_next_hop() - .map_err(RPCError::map_protocol("invalid next hop in route hop"))?; - Some(decode_route_hop_data(&rhd_reader)?) - } else { - None - }; - - Ok(RouteHop { - dial_info, - next_hop, - }) -} - -pub fn decode_private_route( - reader: &veilid_capnp::private_route::Reader, -) -> Result { - let public_key = decode_public_key(&reader.get_public_key().map_err( - RPCError::map_protocol("invalid public key in private route"), - )?); - let hop_count = reader.get_hop_count(); - let hops = if reader.has_first_hop() { - let rh_reader = reader - .get_first_hop() - .map_err(RPCError::map_protocol("invalid first hop in private route"))?; - Some(decode_route_hop(&rh_reader)?) - } else { - None - }; - - Ok(PrivateRoute { - public_key, - hop_count, - hops, - }) -} - pub fn decode_safety_route( reader: &veilid_capnp::safety_route::Reader, ) -> Result { diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index 76d86990..152923cf 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -8,7 +8,7 @@ pub enum Destination { /// The node to send to target: NodeRef, /// Require safety route or not - safety: Option, + safety_spec: Option, }, /// Send to node for relay purposes Relay { @@ -17,14 +17,14 @@ pub enum Destination { /// The final destination the relay should send to target: DHTKey, /// Require safety route or not - safety: Option, + safety_spec: Option, }, /// Send to private route (privateroute) PrivateRoute { /// A private route to send to private_route: PrivateRoute, /// Require safety route or not - safety: Option, + safety_spec: Option, /// Prefer reliability or not reliable: bool, }, @@ -34,46 +34,49 @@ impl Destination { pub fn direct(target: NodeRef) -> Self { Self::Direct { target, - safety: None, + safety_spec: None, } } pub fn relay(relay: NodeRef, target: DHTKey) -> Self { Self::Relay { relay, target, - safety: None, + safety_spec: None, } } pub fn private_route(private_route: PrivateRoute, reliable: bool) -> Self { Self::PrivateRoute { private_route, - safety: None, + safety_spec: None, reliable, } } - pub fn with_safety(self, spec: SafetySpec) -> Self { + pub fn with_safety(self, safety_spec: SafetySpec) -> Self { match self { - Destination::Direct { target, safety: _ } => Self::Direct { + Destination::Direct { target, - safety: Some(spec), + safety_spec: _, + } => Self::Direct { + target, + safety_spec: Some(safety_spec), }, Destination::Relay { relay, target, - safety: _, + safety_spec: _, } => Self::Relay { relay, target, - safety: Some(spec), + safety_spec: Some(safety_spec), }, Destination::PrivateRoute { private_route, - safety: _, + safety_spec: _, reliable, } => Self::PrivateRoute { private_route, - safety: Some(spec), + safety_spec: Some(safety_spec), reliable, }, } @@ -83,26 +86,29 @@ impl Destination { impl fmt::Display for Destination { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - Destination::Direct { target, safety } => { - let sr = if safety.is_some() { "+SR" } else { "" }; + Destination::Direct { + target, + safety_spec, + } => { + let sr = if safety_spec.is_some() { "+SR" } else { "" }; write!(f, "{}{}", target, sr) } Destination::Relay { relay, target, - safety, + safety_spec, } => { - let sr = if safety.is_some() { "+SR" } else { "" }; + let sr = if safety_spec.is_some() { "+SR" } else { "" }; write!(f, "{}@{}{}", target.encode(), relay, sr) } Destination::PrivateRoute { private_route, - safety, + safety_spec, reliable, } => { - let sr = if safety.is_some() { "+SR" } else { "" }; + let sr = if safety_spec.is_some() { "+SR" } else { "" }; let rl = if *reliable { "+RL" } else { "" }; write!(f, "{}{}{}", private_route, sr, rl) diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index dab096a6..0edc535c 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -1,7 +1,6 @@ mod coders; mod destination; mod operation_waiter; -mod private_route; mod rpc_app_call; mod rpc_app_message; mod rpc_cancel_tunnel; @@ -24,7 +23,6 @@ mod rpc_watch_value; pub use destination::*; pub use operation_waiter::*; -pub use private_route::*; pub use rpc_error::*; use super::*; @@ -398,7 +396,7 @@ impl RPCProcessor { } // Wrap an operation with a private route inside a safety route - pub(super) fn wrap_with_route( + pub(super) fn wrap_with_route(xxx continue here &self, safety_spec: SafetySpec, private_route: PrivateRoute, @@ -406,7 +404,7 @@ impl RPCProcessor { ) -> Result { let compiled_route: CompiledRoute = self.routing_table().with_route_spec_store(|rss| { // Compile the safety route with the private route - rss.compile_safety_route(safety_spec, private_route) + rss.compile_safety_route(self.safety_spec, private_route) })?; // Encrypt routed operation diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index 2448b9b8..b5674b70 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -72,8 +72,7 @@ impl RPCProcessor { find_node_q.node_id, // filter |rti, _k, v| { - RoutingTable::filter_has_valid_signed_node_info_inner( - rti, + rti.filter_has_valid_signed_node_info( RoutingDomain::PublicInternet, has_valid_own_node_info, v, @@ -81,11 +80,9 @@ impl RPCProcessor { }, // transform |rti, k, v| { - let own_peer_info = own_peer_info.clone(); - RoutingTable::transform_to_peer_info_inner( - rti, + rti.transform_to_peer_info( RoutingDomain::PublicInternet, - own_peer_info, + own_peer_info.clone(), k, v, ) diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 96f46ae1..e990ebfc 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -341,7 +341,7 @@ impl VeilidAPI { nr.merge_filter(NodeRefFilter::new().with_routing_domain(routing_domain)) } - let cm = routing_table + let cm = network_manager .get_node_contact_method(nr) .map_err(VeilidAPIError::internal)?; diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index ea37ecfd..dafd0c25 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -11,7 +11,7 @@ pub struct RouteHopData { #[derive(Clone, Debug)] pub enum RouteNode { - NodeId(DHTKey), + NodeId(NodeId), PeerInfo(PeerInfo), } impl fmt::Display for RouteNode { @@ -20,7 +20,7 @@ impl fmt::Display for RouteNode { f, "{}", match self { - RouteNode::NodeId(x) => x.encode(), + RouteNode::NodeId(x) => x.key.encode(), RouteNode::PeerInfo(pi) => pi.node_id.key.encode(), } ) diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index 215a8b2c..e0c3a8b4 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -11,7 +11,7 @@ pub struct RoutingContextInner {} pub struct RoutingContextUnlockedInner { /// Enforce use of private routing - privacy: bool, + privacy: usize, /// Choose reliable protocols over unreliable/faster protocols when available reliable: bool, } @@ -41,21 +41,45 @@ impl RoutingContext { api, inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - privacy: false, + privacy: 0, reliable: false, }), } } - pub fn with_privacy(self) -> Self { - Self { + pub fn with_default_privacy(self) -> Result { + let config = self.api.config()?; + let c = config.get(); + Ok(Self { api: self.api.clone(), inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - privacy: true, + privacy: c.network.rpc.default_route_hop_count as usize, reliable: self.unlocked_inner.reliable, }), - } + }) + } + pub fn with_privacy(self, hops: usize) -> Result { + let config = self.api.config()?; + let c = config.get(); + + let privacy = if hops > 0 && hops <= c.network.rpc.max_route_hop_count as usize { + hops + } else { + return Err(VeilidAPIError::invalid_argument( + "hops value is too large", + "hops", + hops, + )); + }; + Ok(Self { + api: self.api.clone(), + inner: Arc::new(Mutex::new(RoutingContextInner {})), + unlocked_inner: Arc::new(RoutingContextUnlockedInner { + privacy, + reliable: self.unlocked_inner.reliable, + }), + }) } pub fn with_reliability(self) -> Self { @@ -93,12 +117,20 @@ impl RoutingContext { } Ok(rpc_processor::Destination::Direct { target: nr, - safety: self.unlocked_inner.privacy, + safety_spec: Some(routing_table::SafetySpec { + preferred_route: None, + hop_count: self.unlocked_inner.privacy, + reliable: self.unlocked_inner.reliable, + }), }) } Target::PrivateRoute(pr) => Ok(rpc_processor::Destination::PrivateRoute { private_route: pr, - safety: self.unlocked_inner.privacy, + safety_spec: Some(routing_table::SafetySpec { + preferred_route: None, + hop_count: self.unlocked_inner.privacy, + reliable: self.unlocked_inner.reliable, + }), reliable: self.unlocked_inner.reliable, }), } diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index f40dd2ea..6aed0875 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -81,7 +81,7 @@ core: max_timestamp_behind_ms: 10000 max_timestamp_ahead_ms: 10000 timeout_ms: 10000 - max_route_hop_count: 7 + max_route_hop_count: 4 default_route_hop_count: 2 dht: resolve_node_timeout: @@ -1491,7 +1491,7 @@ mod tests { assert_eq!(s.core.network.rpc.max_timestamp_behind_ms, Some(10_000u32)); assert_eq!(s.core.network.rpc.max_timestamp_ahead_ms, Some(10_000u32)); assert_eq!(s.core.network.rpc.timeout_ms, 10_000u32); - assert_eq!(s.core.network.rpc.max_route_hop_count, 7); + assert_eq!(s.core.network.rpc.max_route_hop_count, 4); assert_eq!(s.core.network.rpc.default_route_hop_count, 2); // assert_eq!(s.core.network.dht.resolve_node_timeout_ms, None); From fc6eb6e84aade9d5edda24d16455a040d1d99a2b Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 20 Oct 2022 15:09:04 -0400 Subject: [PATCH 18/67] checkpoint --- .../src/routing_table/route_spec_store.rs | 55 ++++++---- veilid-core/src/rpc_processor/mod.rs | 101 +++++++++++------- veilid-core/src/veilid_api/privacy.rs | 23 ++++ 3 files changed, 119 insertions(+), 60 deletions(-) diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 5dc639d8..ef0854b2 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -17,9 +17,9 @@ pub struct SafetySpec { #[derive(Clone, Debug)] pub struct CompiledRoute { /// The safety route attached to the private route - safety_route: SafetyRoute, + pub safety_route: SafetyRoute, /// The secret used to encrypt the message payload - secret: DHTKeySecret, + pub secret: DHTKeySecret, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -600,7 +600,7 @@ impl RouteSpecStore { &mut self, rti: &RoutingTableInner, routing_table: RoutingTable, - safety_spec: SafetySpec, + safety_spec: Option, private_route: PrivateRoute, ) -> Result, RPCError> { let pr_hopcount = private_route.hop_count as usize; @@ -608,7 +608,18 @@ impl RouteSpecStore { return Err(RPCError::internal("private route hop count too long")); } + // See if we are using a safety route, if not, short circuit this operation + if safety_spec.is_none() { + // Safety route stub with the node's public key as the safety route key since it's the 0th hop + return Ok(Some(CompiledRoute { + safety_route: SafetyRoute::new_stub(routing_table.node_id(), private_route), + secret: routing_table.node_id_secret(), + })); + } + let safety_spec = safety_spec.unwrap(); + // See if the preferred route is here + let safety_route_public_key; let opt_safety_rsd: Option<&mut RouteSpecDetail> = if let Some(preferred_route) = safety_spec.preferred_route { self.detail_mut(&preferred_route) @@ -650,19 +661,21 @@ impl RouteSpecStore { // xxx implement caching first! - // xxx implement, ensure we handle hops == 0 for our safetyspec - // Ensure the total hop count isn't too long for our config let sr_hopcount = safety_spec.hop_count; - if sr_hopcount > self.max_route_hop_count { - return Err(RPCError::internal("private route hop count too long")); + if sr_hopcount == 0 { + return Err(RPCError::internal("safety route hop count is zero")); } - let total_hopcount = sr_hopcount + pr_hopcount; + if sr_hopcount > self.max_route_hop_count { + return Err(RPCError::internal("safety route hop count too long")); + } + + // See if we can optimize this compilation yet + // We don't want to include full nodeinfo if we don't have to + //let optimize = safety_rsd. // Create hops - let hops = if sr_hopcount == 0 { - SafetyRouteHops::Private(private_route) - } else { + let hops = { // start last blob-to-encrypt data off as private route let mut blob_data = { let mut pr_message = ::capnp::message::Builder::new_default(); @@ -681,16 +694,13 @@ impl RouteSpecStore { // (outer hop is a RouteHopData, not a RouteHop). // Each loop mutates 'nonce', and 'blob_data' let mut nonce = Crypto::get_random_nonce(); + let crypto = routing_table.network_manager().crypto(); for h in (1..sr_hopcount).rev() { // Get blob to encrypt for next hop blob_data = { // Encrypt the previous blob ENC(nonce, DH(PKhop,SKsr)) - let dh_secret = self - .crypto - .cached_dh( - &safety_route_spec.hops[h].dial_info.node_id.key, - &safety_route_spec.secret_key, - ) + let dh_secret = crypto + .cached_dh(&safety_rsd.hops[h], &safety_rsd.secret_key) .map_err(RPCError::map_internal("dh failed"))?; let enc_msg_data = Crypto::encrypt_aead(blob_data.as_slice(), &nonce, &dh_secret, None) @@ -704,7 +714,7 @@ impl RouteSpecStore { // Make route hop let route_hop = RouteHop { - dial_info: safety_route_spec.hops[h].dial_info.clone(), + node: safety_route_spec.hops[h].dial_info.clone(), next_hop: Some(route_hop_data), }; @@ -744,12 +754,15 @@ impl RouteSpecStore { // Build safety route let safety_route = SafetyRoute { - public_key: safety_route_spec.public_key, - hop_count: safety_route_spec.hops.len() as u8, + public_key: safety_rsd. + hop_count: safety_spec.hop_count as u8, hops, }; - Ok(safety_route) + Ok(Some(CompiledRoute { + safety_route, + secret: todo!(), + })) } /// Mark route as published diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 0edc535c..4947a0a3 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -21,6 +21,7 @@ mod rpc_validate_dial_info; mod rpc_value_changed; mod rpc_watch_value; +pub use coders::*; pub use destination::*; pub use operation_waiter::*; pub use rpc_error::*; @@ -29,7 +30,6 @@ use super::*; use crate::dht::*; use crate::xx::*; use capnp::message::ReaderSegments; -use coders::*; use futures_util::StreamExt; use network_manager::*; use receipt_manager::*; @@ -55,6 +55,8 @@ struct RPCMessageHeader { connection_descriptor: ConnectionDescriptor, /// The routing domain the message was sent through routing_domain: RoutingDomain, + /// The private route the message was received through + private_route: Option, } #[derive(Debug)] @@ -134,7 +136,7 @@ struct RenderedOperation { message: Vec, // The rendered operation bytes node_id: DHTKey, // Destination node id we're sending to node_ref: NodeRef, // Node to send envelope to (may not be destination node id in case of relay) - hop_count: usize, // Total safety + private route hop count + hop_count: usize, // Total safety + private route hop count + 1 hop for the initial send } ///////////////////////////////////////////////////////////////////// @@ -396,16 +398,25 @@ impl RPCProcessor { } // Wrap an operation with a private route inside a safety route - pub(super) fn wrap_with_route(xxx continue here + pub(super) fn wrap_with_route( &self, - safety_spec: SafetySpec, + safety_spec: Option, private_route: PrivateRoute, message_data: Vec, - ) -> Result { - let compiled_route: CompiledRoute = self.routing_table().with_route_spec_store(|rss| { - // Compile the safety route with the private route - rss.compile_safety_route(self.safety_spec, private_route) - })?; + ) -> Result, RPCError> { + let routing_table = self.routing_table(); + let compiled_route: CompiledRoute = + match self.routing_table().with_route_spec_store(|rss, rti| { + // Compile the safety route with the private route + rss.compile_safety_route(rti, routing_table, safety_spec, private_route) + })? { + Some(cr) => cr, + None => { + return Ok(NetworkResult::no_connection_other( + "private route could not be compiled at this time", + )) + } + }; // Encrypt routed operation // Xmsg + ENC(Xmsg, DH(PKapr, SKbsr)) @@ -421,30 +432,42 @@ impl RPCProcessor { let operation = RoutedOperation::new(nonce, enc_msg_data); // Prepare route operation - let route = RPCOperationRoute { - safety_route, + let route_operation = RPCOperationRoute { + safety_route: compiled_route.safety_route, operation, }; - let operation = - RPCOperation::new_statement(RPCStatement::new(RPCStatementDetail::Route(route)), None); + let operation = RPCOperation::new_statement( + RPCStatement::new(RPCStatementDetail::Route(route_operation)), + None, + ); // Convert message to bytes and return it let mut route_msg = ::capnp::message::Builder::new_default(); let mut route_operation = route_msg.init_root::(); operation.encode(&mut route_operation)?; - let out = builder_to_vec(route_msg)?; + let out_message = builder_to_vec(route_msg)?; - // out_node_id = sr - // .hops - // .first() - // .ok_or_else(RPCError::else_internal("no hop in safety route"))? - // .dial_info - // .node_id - // .key; + // Get the first hop this is going to + let out_node_id = compiled_route + .safety_route + .hops + .first() + .ok_or_else(RPCError::else_internal("no hop in safety route"))? + .dial_info + .node_id + .key; - //out_hop_count = 1 + sr.hops.len(); + let out_hop_count = + (1 + compiled_route.safety_route.hop_count + private_route.hop_count) as usize; - Ok(out) + let out = RenderedOperation { + message: out_message, + node_id: out_node_id, + node_ref: compiled_route.first_hop, + hop_count: out_hop_count, + }; + + Ok(NetworkResult::value(out)) } /// Produce a byte buffer that represents the wire encoding of the entire @@ -455,8 +478,8 @@ impl RPCProcessor { &self, dest: Destination, operation: &RPCOperation, - ) -> Result { - let out: RenderedOperation; + ) -> Result, RPCError> { + let out: NetworkResult; // Encode message to a builder and make a message reader for it // Then produce the message as an unencrypted byte buffer @@ -471,12 +494,12 @@ impl RPCProcessor { match dest { Destination::Direct { target: ref node_ref, - safety, + safety_spec, } | Destination::Relay { relay: ref node_ref, target: _, - safety, + safety_spec, } => { // Send to a node without a private route // -------------------------------------- @@ -485,7 +508,7 @@ impl RPCProcessor { let (node_ref, node_id) = if let Destination::Relay { relay: _, target: ref dht_key, - safety: _, + safety_spec: _, } = dest { (node_ref.clone(), dht_key.clone()) @@ -495,36 +518,36 @@ impl RPCProcessor { }; // Handle the existence of safety route - match safety { - false => { + match safety_spec { + None => { // If no safety route is being used, and we're not sending to a private // route, we can use a direct envelope instead of routing - out = RenderedOperation { + out = NetworkResult::value(RenderedOperation { message, node_id, node_ref, hop_count: 1, - }; + }); } - true => { + Some(safety_spec) => { // No private route was specified for the request // but we are using a safety route, so we must create an empty private route let private_route = PrivateRoute::new_stub(node_id); // Wrap with safety route - out = self.wrap_with_route(true, private_route, message)?; + out = self.wrap_with_route(Some(safety_spec), private_route, message)?; } }; } Destination::PrivateRoute { private_route, - safety, + safety_spec, reliable, } => { // Send to private route // --------------------- // Reply with 'route' operation - out = self.wrap_with_route(safety, private_route, message)?; + out = self.wrap_with_route(safety_spec, private_route, message)?; } } @@ -602,7 +625,7 @@ impl RPCProcessor { node_id, node_ref, hop_count, - } = self.render_operation(dest, &operation)?; + } = network_result_try!(self.render_operation(dest, &operation)?); // Calculate answer timeout // Timeout is number of hops times the timeout per hop @@ -665,7 +688,7 @@ impl RPCProcessor { node_id, node_ref, hop_count: _, - } = self.render_operation(dest, &operation)?; + } = network_result_try!(self.render_operation(dest, &operation)?); // Send statement let bytes = message.len() as u64; @@ -749,7 +772,7 @@ impl RPCProcessor { node_id, node_ref, hop_count: _, - } = self.render_operation(dest, &operation)?; + } = network_result_try!(self.render_operation(dest, &operation)?); // Send the reply let bytes = message.len() as u64; diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index dafd0c25..9bdd05f8 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -48,6 +48,19 @@ impl PrivateRoute { first_hop: None, } } + pub fn simplify(self) -> Self { + Self { + public_key: self.public_key, + hop_count: self.hop_count, + first_hop: self.first_hop.map(|h| RouteHop { + node: match h.node { + RouteNode::NodeId(ni) => RouteNode::NodeId(ni), + RouteNode::PeerInfo(pi) => RouteNode::NodeId(pi.node_id), + }, + next_hop: h.next_hop, + }), + } + } } impl fmt::Display for PrivateRoute { @@ -79,6 +92,16 @@ pub struct SafetyRoute { pub hops: SafetyRouteHops, } +impl SafetyRoute { + pub fn new_stub(public_key: DHTKey, private_route: PrivateRoute) -> Self { + Self { + public_key, + hop_count: 0, + hops: SafetyRouteHops::Private(private_route), + } + } +} + impl fmt::Display for SafetyRoute { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( From c8ba88fb99015badb5f5277fa966d2e9f57aa347 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 20 Oct 2022 23:11:41 -0400 Subject: [PATCH 19/67] more route work --- veilid-core/src/network_manager/mod.rs | 4 +- veilid-core/src/routing_table/bucket.rs | 2 +- veilid-core/src/routing_table/bucket_entry.rs | 4 +- .../src/routing_table/route_spec_store.rs | 136 ++++++++++++------ .../src/routing_table/routing_table_inner.rs | 23 ++- veilid-core/src/rpc_processor/destination.rs | 19 +++ veilid-core/src/rpc_processor/mod.rs | 58 ++++---- veilid-core/src/veilid_api/privacy.rs | 2 - 8 files changed, 171 insertions(+), 77 deletions(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 6bca250c..1204a4a1 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -197,7 +197,7 @@ impl NetworkManager { block_store: BlockStore, crypto: Crypto, ) -> NetworkManagerUnlockedInner { - let c = config.get(); + let min_peer_refresh_time_ms = config.get().network.dht.min_peer_refresh_time_ms; NetworkManagerUnlockedInner { config, protected_store, @@ -211,7 +211,7 @@ impl NetworkManager { relay_management_task: TickTask::new(RELAY_MANAGEMENT_INTERVAL_SECS), private_route_management_task: TickTask::new(PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS), bootstrap_task: TickTask::new(1), - peer_minimum_refresh_task: TickTask::new_ms(c.network.dht.min_peer_refresh_time_ms), + peer_minimum_refresh_task: TickTask::new_ms(min_peer_refresh_time_ms), ping_validator_task: TickTask::new(1), public_address_check_task: TickTask::new(PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS), node_info_update_single_future: MustJoinSingleFuture::new(), diff --git a/veilid-core/src/routing_table/bucket.rs b/veilid-core/src/routing_table/bucket.rs index 88051c0c..ba48563c 100644 --- a/veilid-core/src/routing_table/bucket.rs +++ b/veilid-core/src/routing_table/bucket.rs @@ -57,7 +57,7 @@ impl Bucket { } pub(super) fn kick( - &self, + &mut self, inner: &mut RoutingTableInner, bucket_depth: usize, ) -> Option> { diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index fd563f7c..94428da9 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -658,7 +658,7 @@ impl BucketEntry { // immutable reference to RoutingTableInner must be passed in to get this // This ensures that an operation on the routing table can not change entries // while it is being read from - pub(super) fn with(&self, rti: &RoutingTableInner, f: F) -> R + pub fn with(&self, rti: &RoutingTableInner, f: F) -> R where F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> R, { @@ -668,7 +668,7 @@ impl BucketEntry { // Note, that this requires -also- holding the RoutingTable write lock, as a // mutable reference to RoutingTableInner must be passed in to get this - pub(super) fn with_mut(&self, rti: &mut RoutingTableInner, f: F) -> R + pub fn with_mut(&self, rti: &mut RoutingTableInner, f: F) -> R where F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> R, { diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index ef0854b2..fd1d80ab 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -20,6 +20,8 @@ pub struct CompiledRoute { pub safety_route: SafetyRoute, /// The secret used to encrypt the message payload pub secret: DHTKeySecret, + /// The node ref to the first hop in the compiled route + pub first_hop: NodeRef, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -46,6 +48,9 @@ struct RouteSpecDetail { /// Not serialized because all routes should be re-published when restarting #[serde(skip)] published: bool, + // Can optimize the rendering of this route, using node ids only instead of full peer info + #[serde(skip)] + reachable: bool, /// Timestamp of when the route was created created_ts: u64, /// Timestamp of when the route was last checked for validity @@ -124,7 +129,7 @@ fn get_route_permutation_count(hop_count: usize) -> usize { /// get the route permutation at particular 'perm' index, starting at the 'start' index /// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum /// number of permutations is given by get_route_permutation_count() -fn with_route_permutations(hop_count: usize, start: usize, f: F) -> bool +fn with_route_permutations(hop_count: usize, start: usize, mut f: F) -> bool where F: FnMut(&[usize]) -> bool, { @@ -142,7 +147,7 @@ where } // heaps algorithm, but skipping the first element - fn heaps_permutation(permutation: &mut [usize], size: usize, f: F) -> bool + fn heaps_permutation(permutation: &mut [usize], size: usize, mut f: F) -> bool where F: FnMut(&[usize]) -> bool, { @@ -154,7 +159,7 @@ where } for i in 0..size { - if heaps_permutation(permutation, size - 1, f) { + if heaps_permutation(permutation, size - 1, &mut f) { return true; } if size % 2 == 1 { @@ -224,7 +229,7 @@ impl RouteSpecStore { } // Rebuild the routespecstore cache - rss.rebuild_cache(routing_table); + rss.rebuild_cache(); Ok(rss) } @@ -251,28 +256,28 @@ impl RouteSpecStore { Ok(()) } - fn add_to_cache(&mut self, cache_key: Vec, rsd: &RouteSpecDetail) { - if !self.cache.hop_cache.insert(cache_key) { + fn add_to_cache(cache: &mut RouteSpecStoreCache, cache_key: Vec, rsd: &RouteSpecDetail) { + if !cache.hop_cache.insert(cache_key) { panic!("route should never be inserted twice"); } for h in &rsd.hops { - self.cache + cache .used_nodes .entry(*h) .and_modify(|e| *e += 1) .or_insert(1); } - self.cache + cache .used_end_nodes .entry(*rsd.hops.last().unwrap()) .and_modify(|e| *e += 1) .or_insert(1); } - fn rebuild_cache(&mut self, routing_table: RoutingTable) { + fn rebuild_cache(&mut self) { for v in self.content.details.values() { let cache_key = route_hops_to_hop_cache(&v.hops); - self.add_to_cache(cache_key, &v); + Self::add_to_cache(&mut self.cache, cache_key, &v); } } @@ -304,12 +309,7 @@ impl RouteSpecStore { // Get list of all nodes, and sort them for selection let cur_ts = intf::get_timestamp(); - let dial_info_sort = if reliable { - Some(DialInfoDetail::reliable_sort) - } else { - None - }; - let filter = |rti, k: DHTKey, v: Option>| -> bool { + let filter = |rti, _k: DHTKey, v: Option>| -> bool { // Exclude our own node from routes if v.is_none() { return false; @@ -503,7 +503,10 @@ impl RouteSpecStore { let hops = route_nodes.iter().map(|v| nodes[*v].0).collect(); let hop_node_refs = route_nodes .iter() - .map(|v| rti.lookup_node_ref(routing_table, nodes[*v].0).unwrap()) + .map(|v| { + rti.lookup_node_ref(routing_table.clone(), nodes[*v].0) + .unwrap() + }) .collect(); let (public_key, secret_key) = generate_secret(); @@ -517,6 +520,7 @@ impl RouteSpecStore { latency_stats_accounting: Default::default(), transfer_stats_accounting: Default::default(), published: false, + reachable: false, created_ts: cur_ts, last_checked_ts: None, last_used_ts: None, @@ -525,7 +529,7 @@ impl RouteSpecStore { }; // Add to cache - self.add_to_cache(cache_key, &rsd); + Self::add_to_cache(&mut self.cache, cache_key, &rsd); // Keep route in spec store self.content.details.insert(public_key, rsd); @@ -543,7 +547,7 @@ impl RouteSpecStore { // Remove from used nodes cache for h in &detail.hops { match self.cache.used_nodes.entry(*h) { - std::collections::hash_map::Entry::Occupied(o) => { + std::collections::hash_map::Entry::Occupied(mut o) => { *o.get_mut() -= 1; if *o.get() == 0 { o.remove(); @@ -556,7 +560,7 @@ impl RouteSpecStore { } // Remove from end nodes cache match self.cache.used_nodes.entry(*detail.hops.last().unwrap()) { - std::collections::hash_map::Entry::Occupied(o) => { + std::collections::hash_map::Entry::Occupied(mut o) => { *o.get_mut() -= 1; if *o.get() == 0 { o.remove(); @@ -611,23 +615,44 @@ impl RouteSpecStore { // See if we are using a safety route, if not, short circuit this operation if safety_spec.is_none() { // Safety route stub with the node's public key as the safety route key since it's the 0th hop + if private_route.first_hop.is_none() { + return Err(RPCError::internal("can't compile zero length route")); + } + let first_hop = private_route.first_hop.as_ref().unwrap(); + let opt_first_hop_noderef = match &first_hop.node { + RouteNode::NodeId(id) => rti.lookup_node_ref(routing_table, id.key), + RouteNode::PeerInfo(pi) => rti.register_node_with_signed_node_info( + routing_table.clone(), + RoutingDomain::PublicInternet, + pi.node_id.key, + pi.signed_node_info, + false, + ), + }; + if opt_first_hop_noderef.is_none() { + // Can't reach this private route any more + log_rtab!(debug "can't reach private route any more"); + return Ok(None); + } + return Ok(Some(CompiledRoute { safety_route: SafetyRoute::new_stub(routing_table.node_id(), private_route), secret: routing_table.node_id_secret(), + first_hop: opt_first_hop_noderef.unwrap(), })); } let safety_spec = safety_spec.unwrap(); // See if the preferred route is here - let safety_route_public_key; - let opt_safety_rsd: Option<&mut RouteSpecDetail> = + let opt_safety_rsd: Option<(&mut RouteSpecDetail, DHTKey)> = if let Some(preferred_route) = safety_spec.preferred_route { self.detail_mut(&preferred_route) + .map(|rsd| (rsd, preferred_route)) } else { // Preferred safety route was not requested None }; - let safety_rsd: &mut RouteSpecDetail = if let Some(safety_rsd) = opt_safety_rsd { + let (safety_rsd, sr_pubkey) = if let Some(safety_rsd) = opt_safety_rsd { // Safety route exists safety_rsd } else { @@ -639,13 +664,13 @@ impl RouteSpecStore { Direction::Outbound.into(), ) { // Found a route to use - self.detail_mut(&sr_pubkey).unwrap() + (self.detail_mut(&sr_pubkey).unwrap(), sr_pubkey) } else { // No route found, gotta allocate one let sr_pubkey = match self .allocate_route( rti, - routing_table, + routing_table.clone(), safety_spec.reliable, safety_spec.hop_count, Direction::Outbound.into(), @@ -655,12 +680,10 @@ impl RouteSpecStore { Some(pk) => pk, None => return Ok(None), }; - self.detail_mut(&sr_pubkey).unwrap() + (self.detail_mut(&sr_pubkey).unwrap(), sr_pubkey) } }; - // xxx implement caching first! - // Ensure the total hop count isn't too long for our config let sr_hopcount = safety_spec.hop_count; if sr_hopcount == 0 { @@ -672,7 +695,9 @@ impl RouteSpecStore { // See if we can optimize this compilation yet // We don't want to include full nodeinfo if we don't have to - //let optimize = safety_rsd. + let optimize = safety_rsd.reachable; + + // xxx implement caching here! // Create hops let hops = { @@ -714,7 +739,27 @@ impl RouteSpecStore { // Make route hop let route_hop = RouteHop { - node: safety_route_spec.hops[h].dial_info.clone(), + node: match optimize { + // Optimized, no peer info, just the dht key + true => RouteNode::NodeId(NodeId::new(safety_rsd.hops[h])), + // Full peer info, required until we are sure the route has been fully established + false => { + let node_id = safety_rsd.hops[h]; + let pi = rti + .with_node_entry(node_id, |entry| { + entry.with(rti, |rti, e| { + e.make_peer_info(node_id, RoutingDomain::PublicInternet) + }) + }) + .flatten(); + if pi.is_none() { + return Err(RPCError::internal( + "peer info should exist for route but doesn't", + )); + } + RouteNode::PeerInfo(pi.unwrap()) + } + }, next_hop: Some(route_hop_data), }; @@ -734,12 +779,8 @@ impl RouteSpecStore { } // Encode first RouteHopData - let dh_secret = self - .crypto - .cached_dh( - &safety_route_spec.hops[0].dial_info.node_id.key, - &safety_route_spec.secret_key, - ) + let dh_secret = crypto + .cached_dh(&safety_rsd.hops[0], &safety_rsd.secret_key) .map_err(RPCError::map_internal("dh failed"))?; let enc_msg_data = Crypto::encrypt_aead(blob_data.as_slice(), &nonce, &dh_secret, None) .map_err(RPCError::map_internal("encryption failed"))?; @@ -754,15 +795,20 @@ impl RouteSpecStore { // Build safety route let safety_route = SafetyRoute { - public_key: safety_rsd. + public_key: sr_pubkey, hop_count: safety_spec.hop_count as u8, hops, }; - Ok(Some(CompiledRoute { + let compiled_route = CompiledRoute { safety_route, - secret: todo!(), - })) + secret: safety_rsd.secret_key, + first_hop: safety_rsd.hop_node_refs.first().unwrap().clone(), + }; + + // xxx: add cache here + + Ok(Some(compiled_route)) } /// Mark route as published @@ -775,6 +821,16 @@ impl RouteSpecStore { Ok(()) } + /// Mark route as reachable + /// When first deserialized, routes must be re-tested for reachability + /// This can be used to determine if routes need to be sent with full peerinfo or can just use a node id + pub fn mark_route_reachable(&mut self, key: &DHTKey) -> EyreResult<()> { + self.detail_mut(&key) + .ok_or_else(|| eyre!("route does not exist"))? + .published = true; + Ok(()) + } + /// Mark route as checked pub fn touch_route_checked(&mut self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { self.detail_mut(&key) diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index f0bf9a24..aded6846 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -68,10 +68,10 @@ impl RoutingTableInner { self.unlocked_inner.config.clone() } - pub fn transfer_stats_accounting(&mut self) -> &TransferStatsAccounting { + pub fn transfer_stats_accounting(&mut self) -> &mut TransferStatsAccounting { &mut self.self_transfer_stats_accounting } - pub fn latency_stats_accounting(&mut self) -> &LatencyStatsAccounting { + pub fn latency_stats_accounting(&mut self) -> &mut LatencyStatsAccounting { &mut self.self_latency_stats_accounting } @@ -616,7 +616,7 @@ impl RoutingTableInner { /// Resolve an existing routing table entry and return a reference to it pub fn lookup_node_ref(&self, outer_self: RoutingTable, node_id: DHTKey) -> Option { if node_id == self.unlocked_inner.node_id { - log_rtab!(debug "can't look up own node id in routing table"); + log_rtab!(error "can't look up own node id in routing table"); return None; } let idx = self.find_bucket_index(node_id); @@ -644,6 +644,23 @@ impl RoutingTableInner { ) } + /// Resolve an existing routing table entry and call a function on its entry without using a noderef + pub fn with_node_entry(&self, node_id: DHTKey, f: F) -> Option + where + F: FnOnce(Arc) -> R, + { + if node_id == self.unlocked_inner.node_id { + log_rtab!(error "can't look up own node id in routing table"); + return None; + } + let idx = self.find_bucket_index(node_id); + let bucket = &self.buckets[idx]; + if let Some(e) = bucket.entry(&node_id) { + return Some(f(e)); + } + None + } + /// Shortcut function to add a node to our routing table if it doesn't exist /// and add the dial info we have for it. Returns a noderef filtered to /// the routing domain in which this node was registered for convenience. diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index 152923cf..a611275b 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -81,6 +81,25 @@ impl Destination { }, } } + + pub fn get_safety_spec(&self) -> &Option { + match self { + Destination::Direct { + target: _, + safety_spec, + } => safety_spec, + Destination::Relay { + relay: _, + target: _, + safety_spec, + } => safety_spec, + Destination::PrivateRoute { + private_route: _, + safety_spec, + reliable: _, + } => safety_spec, + } + } } impl fmt::Display for Destination { diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 4947a0a3..499fc543 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -55,8 +55,8 @@ struct RPCMessageHeader { connection_descriptor: ConnectionDescriptor, /// The routing domain the message was sent through routing_domain: RoutingDomain, - /// The private route the message was received through - private_route: Option, + // The private route the message was received through + //private_route: Option, } #[derive(Debug)] @@ -87,7 +87,7 @@ pub(crate) struct RPCMessage { opt_sender_nr: Option, } -fn builder_to_vec<'a, T>(builder: capnp::message::Builder) -> Result, RPCError> +pub fn builder_to_vec<'a, T>(builder: capnp::message::Builder) -> Result, RPCError> where T: capnp::message::Allocator + 'a, { @@ -185,10 +185,10 @@ impl RPCProcessor { // set up channel let mut concurrency = c.network.rpc.concurrency; - let mut queue_size = c.network.rpc.queue_size; - let mut timeout = ms_to_us(c.network.rpc.timeout_ms); - let mut max_route_hop_count = c.network.rpc.max_route_hop_count as usize; - let mut default_route_hop_count = c.network.rpc.default_route_hop_count as usize; + let queue_size = c.network.rpc.queue_size; + let timeout = ms_to_us(c.network.rpc.timeout_ms); + let max_route_hop_count = c.network.rpc.max_route_hop_count as usize; + let default_route_hop_count = c.network.rpc.default_route_hop_count as usize; if concurrency == 0 { concurrency = intf::get_concurrency() / 2; if concurrency == 0 { @@ -405,8 +405,11 @@ impl RPCProcessor { message_data: Vec, ) -> Result, RPCError> { let routing_table = self.routing_table(); + let pr_hop_count = private_route.hop_count; + let pr_pubkey = private_route.public_key; + let compiled_route: CompiledRoute = - match self.routing_table().with_route_spec_store(|rss, rti| { + match self.routing_table().with_route_spec_store_mut(|rss, rti| { // Compile the safety route with the private route rss.compile_safety_route(rti, routing_table, safety_spec, private_route) })? { @@ -423,7 +426,7 @@ impl RPCProcessor { let nonce = Crypto::get_random_nonce(); let dh_secret = self .crypto - .cached_dh(&private_route.public_key, &compiled_route.secret) + .cached_dh(&pr_pubkey, &compiled_route.secret) .map_err(RPCError::map_internal("dh failed"))?; let enc_msg_data = Crypto::encrypt_aead(&message_data, &nonce, &dh_secret, None) .map_err(RPCError::map_internal("encryption failed"))?; @@ -432,6 +435,7 @@ impl RPCProcessor { let operation = RoutedOperation::new(nonce, enc_msg_data); // Prepare route operation + let sr_hop_count = compiled_route.safety_route.hop_count; let route_operation = RPCOperationRoute { safety_route: compiled_route.safety_route, operation, @@ -448,17 +452,8 @@ impl RPCProcessor { let out_message = builder_to_vec(route_msg)?; // Get the first hop this is going to - let out_node_id = compiled_route - .safety_route - .hops - .first() - .ok_or_else(RPCError::else_internal("no hop in safety route"))? - .dial_info - .node_id - .key; - - let out_hop_count = - (1 + compiled_route.safety_route.hop_count + private_route.hop_count) as usize; + let out_node_id = compiled_route.first_hop.node_id(); + let out_hop_count = (1 + sr_hop_count + pr_hop_count) as usize; let out = RenderedOperation { message: out_message, @@ -560,20 +555,21 @@ impl RPCProcessor { // as far as we can tell this is the only domain that will really benefit fn get_sender_signed_node_info(&self, dest: &Destination) -> Option { // Don't do this if the sender is to remain private - if dest.safety_route_spec().is_some() { + // Otherwise we would be attaching the original sender's identity to the final destination, + // thus defeating the purpose of the safety route entirely :P + if dest.get_safety_spec().is_some() { return None; } // Don't do this if our own signed node info isn't valid yet let routing_table = self.routing_table(); - let network_manager = self.network_manager(); - if !RoutingTable::has_valid_own_node_info(network_manager, RoutingDomain::PublicInternet) { + if !routing_table.has_valid_own_node_info(RoutingDomain::PublicInternet) { return None; } match dest { Destination::Direct { target, - safety_route_spec: _, + safety_spec: _, } => { // If the target has seen our node info already don't do this if target.has_seen_our_node_info(RoutingDomain::PublicInternet) { @@ -584,7 +580,7 @@ impl RPCProcessor { Destination::Relay { relay: _, target, - safety_route_spec: _, + safety_spec: _, } => { if let Some(target) = routing_table.lookup_node_ref(*target) { if target.has_seen_our_node_info(RoutingDomain::PublicInternet) { @@ -597,7 +593,8 @@ impl RPCProcessor { } Destination::PrivateRoute { private_route: _, - safety_route_spec: _, + safety_spec: _, + reliable: _, } => None, } } @@ -742,7 +739,14 @@ impl RPCProcessor { Destination::relay(peer_noderef, sender_id) } } - RespondTo::PrivateRoute(pr) => Destination::private_route(pr.clone()), + RespondTo::PrivateRoute(pr) => Destination::private_route( + pr.clone(), + request + .header + .connection_descriptor + .protocol_type() + .is_connection_oriented(), + ), } } diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index 9bdd05f8..a47721a0 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -116,5 +116,3 @@ impl fmt::Display for SafetyRoute { ) } } - -// xxx impl to_blob and from_blob using capnp here From a1b40c79f17c421979b0a79696401e85f1195d6a Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 21 Oct 2022 10:35:03 -0400 Subject: [PATCH 20/67] more route work --- veilid-core/src/network_manager/mod.rs | 4 +- veilid-core/src/routing_table/mod.rs | 7 +- veilid-core/src/routing_table/node_ref.rs | 59 ++++--- .../src/routing_table/route_spec_store.rs | 147 +++++++++++------- .../src/routing_table/routing_domains.rs | 8 +- .../src/routing_table/routing_table_inner.rs | 4 +- veilid-core/src/rpc_processor/mod.rs | 8 +- veilid-core/src/veilid_api/mod.rs | 67 +++++++- 8 files changed, 207 insertions(+), 97 deletions(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 1204a4a1..4bb5178a 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1182,7 +1182,7 @@ impl NetworkManager { // Dial info filter comes from the target node ref let dial_info_filter = target_node_ref.dial_info_filter(); - let reliable = target_node_ref.reliable(); + let sequencing = target_node_ref.sequencing(); let cm = routing_table.get_contact_method( routing_domain, @@ -1191,7 +1191,7 @@ impl NetworkManager { &node_b_id, &node_b, dial_info_filter, - reliable, + sequencing, ); // Translate the raw contact method to a referenced contact method diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index efbdd077..3c1ed52c 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -263,7 +263,7 @@ impl RoutingTable { node_b_id: &DHTKey, node_b: &NodeInfo, dial_info_filter: DialInfoFilter, - reliable: bool, + sequencing: Sequencing, ) -> ContactMethod { self.inner.read().get_contact_method( routing_domain, @@ -272,7 +272,7 @@ impl RoutingTable { node_b_id, node_b, dial_info_filter, - reliable, + sequencing, ) } @@ -836,7 +836,7 @@ impl RoutingTable { .node_info(RoutingDomain::PublicInternet) .map(|n| { let dids = n.all_filtered_dial_info_details( - Some(DialInfoDetail::reliable_sort), // By default, choose reliable protocol for relay + Some(DialInfoDetail::ordered_sequencing_sort), // By default, choose connection-oriented protocol for relay |did| did.matches_filter(&outbound_dif), ); for did in &dids { @@ -889,6 +889,7 @@ impl RoutingTable { if let Some(best_inbound_relay) = best_inbound_relay.as_mut() { // Less is faster let better = best_inbound_relay.1.with(rti, |_rti, best| { + // choose low latency stability for relays BucketEntryInner::cmp_fastest_reliable(cur_ts, e, best) == std::cmp::Ordering::Less }); diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 19c14f9e..1e4385d8 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -71,7 +71,7 @@ pub struct NodeRef { node_id: DHTKey, entry: Arc, filter: Option, - reliable: bool, + sequencing: Sequencing, #[cfg(feature = "tracking")] track_id: usize, } @@ -90,7 +90,7 @@ impl NodeRef { node_id, entry, filter, - reliable: false, + sequencing: Sequencing::NoPreference, #[cfg(feature = "tracking")] track_id: entry.track(), } @@ -127,11 +127,11 @@ impl NodeRef { self.filter = filter } - pub fn set_reliable(&mut self) { - self.reliable = true; + pub fn set_sequencing(&mut self, sequencing: Sequencing) { + self.sequencing = sequencing; } - pub fn reliable(&self) -> bool { - self.reliable + pub fn sequencing(&self) -> Sequencing { + self.sequencing } pub fn merge_filter(&mut self, filter: NodeRefFilter) { @@ -278,10 +278,18 @@ impl NodeRef { let routing_domain_set = self.routing_domain_set(); let dial_info_filter = self.dial_info_filter(); - let sort = if self.reliable { - Some(DialInfoDetail::reliable_sort) - } else { - None + let (sort, dial_info_filter) = match self.sequencing { + Sequencing::NoPreference => (None, dial_info_filter), + Sequencing::PreferOrdered => ( + Some(DialInfoDetail::ordered_sequencing_sort), + dial_info_filter, + ), + Sequencing::EnsureOrdered => ( + Some(DialInfoDetail::ordered_sequencing_sort), + dial_info_filter.filtered( + &DialInfoFilter::all().with_protocol_type_set(ProtocolType::all_ordered_set()), + ), + ), }; self.operate(|_rt, e| { @@ -301,10 +309,18 @@ impl NodeRef { let routing_domain_set = self.routing_domain_set(); let dial_info_filter = self.dial_info_filter(); - let sort = if self.reliable { - Some(DialInfoDetail::reliable_sort) - } else { - None + let (sort, dial_info_filter) = match self.sequencing { + Sequencing::NoPreference => (None, dial_info_filter), + Sequencing::PreferOrdered => ( + Some(DialInfoDetail::ordered_sequencing_sort), + dial_info_filter, + ), + Sequencing::EnsureOrdered => ( + Some(DialInfoDetail::ordered_sequencing_sort), + dial_info_filter.filtered( + &DialInfoFilter::all().with_protocol_type_set(ProtocolType::all_ordered_set()), + ), + ), }; let mut out = Vec::new(); @@ -418,20 +434,20 @@ impl Clone for NodeRef { node_id: self.node_id, entry: self.entry.clone(), filter: self.filter.clone(), - reliable: self.reliable, + sequencing: self.sequencing, #[cfg(feature = "tracking")] track_id: e.track(), } } } -impl PartialEq for NodeRef { - fn eq(&self, other: &Self) -> bool { - self.node_id == other.node_id - } -} +// impl PartialEq for NodeRef { +// fn eq(&self, other: &Self) -> bool { +// self.node_id == other.node_id +// } +// } -impl Eq for NodeRef {} +// impl Eq for NodeRef {} impl fmt::Display for NodeRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -444,6 +460,7 @@ impl fmt::Debug for NodeRef { f.debug_struct("NodeRef") .field("node_id", &self.node_id) .field("filter", &self.filter) + .field("sequencing", &self.sequencing) .finish() } } diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index fd1d80ab..9d6e6485 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -9,8 +9,10 @@ pub struct SafetySpec { pub preferred_route: Option, /// 0 = no safety route, just use node's node id, more hops is safer but slower pub hop_count: usize, - /// prefer more reliable protocols and relays over faster ones - pub reliable: bool, + /// prefer reliability over speed + pub stability: Stability, + /// prefer connection-oriented sequenced protocols + pub sequencing: Sequencing, } /// Compiled route (safety route + private route) @@ -59,8 +61,10 @@ struct RouteSpecDetail { last_used_ts: Option, /// Directions this route is guaranteed to work in directions: DirectionSet, - /// Reliability - reliable: bool, + /// Stability preference (prefer reliable nodes over faster) + stability: Stability, + /// Sequencing preference (connection oriented protocols vs datagram) + sequencing: Sequencing, } /// The core representation of the RouteSpecStore that can be serialized @@ -81,6 +85,7 @@ pub struct RouteSpecStoreCache { hop_cache: HashSet>, } +/// The routing table's storage for private/safety routes #[derive(Debug)] pub struct RouteSpecStore { /// Maximum number of hops in a route @@ -93,6 +98,15 @@ pub struct RouteSpecStore { cache: RouteSpecStoreCache, } +/// The choice of safety route including in compiled routes +#[derive(Debug, Clone)] +pub enum SafetySelection { + /// Don't use a safety route, only specify the sequencing preference + Unsafe(Sequencing), + /// Use a safety route and parameters specified by a SafetySpec + Safe(SafetySpec), +} + fn route_hops_to_hop_cache(hops: &[DHTKey]) -> Vec { let mut cache: Vec = Vec::with_capacity(hops.len() * DHT_KEY_LENGTH); for hop in hops { @@ -293,7 +307,8 @@ impl RouteSpecStore { &mut self, rti: &RoutingTableInner, routing_table: RoutingTable, - reliable: bool, + stability: Stability, + sequencing: Sequencing, hop_count: usize, directions: DirectionSet, ) -> EyreResult> { @@ -327,7 +342,7 @@ impl RouteSpecStore { // Exclude nodes with no publicinternet nodeinfo, or incompatible nodeinfo or node status won't route v.with(rti, |_rti, e| { let node_info_ok = if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) { - ni.has_any_dial_info() + ni.has_sequencing_matched_dial_info(sequencing) } else { false }; @@ -383,13 +398,16 @@ impl RouteSpecStore { // always prioritize reliable nodes, but sort by oldest or fastest let cmpout = v1.1.as_ref().unwrap().with(rti, |rti, e1| { - v2.1.as_ref().unwrap().with(rti, |_rti, e2| { - if reliable { - BucketEntryInner::cmp_oldest_reliable(cur_ts, e1, e2) - } else { - BucketEntryInner::cmp_fastest_reliable(cur_ts, e1, e2) - } - }) + v2.1.as_ref() + .unwrap() + .with(rti, |_rti, e2| match stability { + Stability::LowLatency => { + BucketEntryInner::cmp_fastest_reliable(cur_ts, e1, e2) + } + Stability::Reliable => { + BucketEntryInner::cmp_oldest_reliable(cur_ts, e1, e2) + } + }) }); cmpout }; @@ -448,7 +466,7 @@ impl RouteSpecStore { ¤t_node.0, ¤t_node.1, DialInfoFilter::all(), - reliable, + sequencing, ); if matches!(cm, ContactMethod::Unreachable) { reachable = false; @@ -474,7 +492,7 @@ impl RouteSpecStore { ¤t_node.0, ¤t_node.1, DialInfoFilter::all(), - reliable, + sequencing, ); if matches!(cm, ContactMethod::Unreachable) { reachable = false; @@ -525,7 +543,8 @@ impl RouteSpecStore { last_checked_ts: None, last_used_ts: None, directions, - reliable, + stability, + sequencing, }; // Add to cache @@ -575,15 +594,18 @@ impl RouteSpecStore { } } + /// Find first matching unpublished route that fits into the selection criteria pub fn first_unpublished_route( &mut self, - reliable: bool, min_hop_count: usize, max_hop_count: usize, + stability: Stability, + sequencing: Sequencing, directions: DirectionSet, ) -> Option { for detail in &self.content.details { - if detail.1.reliable == reliable + if detail.1.stability >= stability + && detail.1.sequencing >= sequencing && detail.1.hops.len() >= min_hop_count && detail.1.hops.len() <= max_hop_count && detail.1.directions.is_subset(directions) @@ -602,46 +624,54 @@ impl RouteSpecStore { /// Returns Ok(None) if no allocation could happen at this time (not an error) pub fn compile_safety_route( &mut self, - rti: &RoutingTableInner, + rti: &mut RoutingTableInner, routing_table: RoutingTable, - safety_spec: Option, + safety_selection: SafetySelection, private_route: PrivateRoute, ) -> Result, RPCError> { let pr_hopcount = private_route.hop_count as usize; - if pr_hopcount > self.max_route_hop_count { + let max_route_hop_count = self.max_route_hop_count; + if pr_hopcount > max_route_hop_count { return Err(RPCError::internal("private route hop count too long")); } // See if we are using a safety route, if not, short circuit this operation - if safety_spec.is_none() { - // Safety route stub with the node's public key as the safety route key since it's the 0th hop - if private_route.first_hop.is_none() { - return Err(RPCError::internal("can't compile zero length route")); - } - let first_hop = private_route.first_hop.as_ref().unwrap(); - let opt_first_hop_noderef = match &first_hop.node { - RouteNode::NodeId(id) => rti.lookup_node_ref(routing_table, id.key), - RouteNode::PeerInfo(pi) => rti.register_node_with_signed_node_info( - routing_table.clone(), - RoutingDomain::PublicInternet, - pi.node_id.key, - pi.signed_node_info, - false, - ), - }; - if opt_first_hop_noderef.is_none() { - // Can't reach this private route any more - log_rtab!(debug "can't reach private route any more"); - return Ok(None); - } + let safety_spec = match safety_selection { + SafetySelection::Unsafe(sequencing) => { + // Safety route stub with the node's public key as the safety route key since it's the 0th hop + if private_route.first_hop.is_none() { + return Err(RPCError::internal("can't compile zero length route")); + } + let first_hop = private_route.first_hop.as_ref().unwrap(); + let opt_first_hop = match &first_hop.node { + RouteNode::NodeId(id) => rti.lookup_node_ref(routing_table.clone(), id.key), + RouteNode::PeerInfo(pi) => rti.register_node_with_signed_node_info( + routing_table.clone(), + RoutingDomain::PublicInternet, + pi.node_id.key, + pi.signed_node_info.clone(), + false, + ), + }; + if opt_first_hop.is_none() { + // Can't reach this private route any more + log_rtab!(debug "can't reach private route any more"); + return Ok(None); + } + let mut first_hop = opt_first_hop.unwrap(); - return Ok(Some(CompiledRoute { - safety_route: SafetyRoute::new_stub(routing_table.node_id(), private_route), - secret: routing_table.node_id_secret(), - first_hop: opt_first_hop_noderef.unwrap(), - })); - } - let safety_spec = safety_spec.unwrap(); + // Set sequencing requirement + first_hop.set_sequencing(sequencing); + + // Return the compiled safety route + return Ok(Some(CompiledRoute { + safety_route: SafetyRoute::new_stub(routing_table.node_id(), private_route), + secret: routing_table.node_id_secret(), + first_hop, + })); + } + SafetySelection::Safe(safety_spec) => safety_spec, + }; // See if the preferred route is here let opt_safety_rsd: Option<(&mut RouteSpecDetail, DHTKey)> = @@ -658,9 +688,10 @@ impl RouteSpecStore { } else { // Select a safety route from the pool or make one if we don't have one that matches if let Some(sr_pubkey) = self.first_unpublished_route( - safety_spec.reliable, safety_spec.hop_count, safety_spec.hop_count, + safety_spec.stability, + safety_spec.sequencing, Direction::Outbound.into(), ) { // Found a route to use @@ -671,7 +702,8 @@ impl RouteSpecStore { .allocate_route( rti, routing_table.clone(), - safety_spec.reliable, + safety_spec.stability, + safety_spec.sequencing, safety_spec.hop_count, Direction::Outbound.into(), ) @@ -689,7 +721,7 @@ impl RouteSpecStore { if sr_hopcount == 0 { return Err(RPCError::internal("safety route hop count is zero")); } - if sr_hopcount > self.max_route_hop_count { + if sr_hopcount > max_route_hop_count { return Err(RPCError::internal("safety route hop count too long")); } @@ -747,7 +779,7 @@ impl RouteSpecStore { let node_id = safety_rsd.hops[h]; let pi = rti .with_node_entry(node_id, |entry| { - entry.with(rti, |rti, e| { + entry.with(rti, |_rti, e| { e.make_peer_info(node_id, RoutingDomain::PublicInternet) }) }) @@ -800,14 +832,21 @@ impl RouteSpecStore { hops, }; + let mut first_hop = safety_rsd.hop_node_refs.first().unwrap().clone(); + + // Ensure sequencing requirement is set on first hop + first_hop.set_sequencing(safety_spec.sequencing); + + // Build compiled route let compiled_route = CompiledRoute { safety_route, secret: safety_rsd.secret_key, - first_hop: safety_rsd.hop_node_refs.first().unwrap().clone(), + first_hop, }; // xxx: add cache here + // Return compiled route Ok(Some(compiled_route)) } diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 6475cd01..0e164b34 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -179,7 +179,7 @@ pub trait RoutingDomainDetail { node_b_id: &DHTKey, node_b: &NodeInfo, dial_info_filter: DialInfoFilter, - reliable: bool, + sequencing: Sequencing, ) -> ContactMethod; } @@ -204,7 +204,7 @@ fn first_filtered_dial_info_detail( from_node: &NodeInfo, to_node: &NodeInfo, dial_info_filter: &DialInfoFilter, - reliable: bool, + reliable: bool, xxx continue here ) -> Option { let direct_dial_info_filter = dial_info_filter.clone().filtered( &DialInfoFilter::all() @@ -214,7 +214,7 @@ fn first_filtered_dial_info_detail( // Get first filtered dialinfo let sort = if reliable { - Some(DialInfoDetail::reliable_sort) + Some(DialInfoDetail::ordered_sequencing_sort) } else { None }; @@ -428,7 +428,7 @@ impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { // Get first filtered dialinfo let sort = if reliable { - Some(DialInfoDetail::reliable_sort) + Some(DialInfoDetail::ordered_sequencing_sort) } else { None }; diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index aded6846..10918799 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -224,7 +224,7 @@ impl RoutingTableInner { node_b_id: &DHTKey, node_b: &NodeInfo, dial_info_filter: DialInfoFilter, - reliable: bool, + sequencing: Sequencing, ) -> ContactMethod { self.with_routing_domain(routing_domain, |rdd| { rdd.get_contact_method( @@ -234,7 +234,7 @@ impl RoutingTableInner { node_b_id, node_b, dial_info_filter, - reliable, + sequencing, ) }) } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 499fc543..709ab9b1 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -402,6 +402,7 @@ impl RPCProcessor { &self, safety_spec: Option, private_route: PrivateRoute, + reliable: bool, message_data: Vec, ) -> Result, RPCError> { let routing_table = self.routing_table(); @@ -411,7 +412,7 @@ impl RPCProcessor { let compiled_route: CompiledRoute = match self.routing_table().with_route_spec_store_mut(|rss, rti| { // Compile the safety route with the private route - rss.compile_safety_route(rti, routing_table, safety_spec, private_route) + rss.compile_safety_route(rti, routing_table, safety_spec, private_route, reliable) })? { Some(cr) => cr, None => { @@ -455,6 +456,7 @@ impl RPCProcessor { let out_node_id = compiled_route.first_hop.node_id(); let out_hop_count = (1 + sr_hop_count + pr_hop_count) as usize; + let out = RenderedOperation { message: out_message, node_id: out_node_id, @@ -537,12 +539,12 @@ impl RPCProcessor { Destination::PrivateRoute { private_route, safety_spec, - reliable, + reliable, xxxx does this need to be here? what about None safety spec, reliable is in there, does it need to not be? or something? } => { // Send to private route // --------------------- // Reply with 'route' operation - out = self.wrap_with_route(safety_spec, private_route, message)?; + out = self.wrap_with_route(safety_spec, private_route, reliable, message)?; } } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 92f6fede..8f2ac241 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -405,6 +405,21 @@ impl DialInfoClass { } } +// Ordering here matters, >= is used to check strength of sequencing requirement +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum Sequencing { + NoPreference, + PreferOrdered, + EnsureOrdered, +} + +// Ordering here matters, >= is used to check strength of stability requirement +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum Stability { + LowLatency, + Reliable, +} + // Keep member order appropriate for sorting < preference #[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize, Hash)] pub struct DialInfoDetail { @@ -419,14 +434,14 @@ impl MatchesDialInfoFilter for DialInfoDetail { } impl DialInfoDetail { - pub fn reliable_sort(a: &DialInfoDetail, b: &DialInfoDetail) -> core::cmp::Ordering { + pub fn ordered_sequencing_sort(a: &DialInfoDetail, b: &DialInfoDetail) -> core::cmp::Ordering { if a.class < b.class { return core::cmp::Ordering::Less; } if a.class > b.class { return core::cmp::Ordering::Greater; } - DialInfo::reliable_sort(&a.dial_info, &b.dial_info) + DialInfo::ordered_sequencing_sort(&a.dial_info, &b.dial_info) } pub const NO_SORT: std::option::Option< for<'r, 's> fn( @@ -592,6 +607,39 @@ impl NodeInfo { .unwrap_or_default() } + pub fn has_sequencing_matched_dial_info(&self, sequencing: Sequencing) -> bool { + // Check our dial info + for did in &self.dial_info_detail_list { + match sequencing { + Sequencing::NoPreference | Sequencing::PreferOrdered => return true, + Sequencing::EnsureOrdered => { + if did.dial_info.protocol_type().is_connection_oriented() { + return true; + } + } + } + } + // Check our relay if we have one + return self + .relay_peer_info + .as_ref() + .map(|rpi| { + let relay_ni = &rpi.signed_node_info.node_info; + for did in relay_ni.dial_info_detail_list { + match sequencing { + Sequencing::NoPreference | Sequencing::PreferOrdered => return true, + Sequencing::EnsureOrdered => { + if did.dial_info.protocol_type().is_connection_oriented() { + return true; + } + } + } + } + false + }) + .unwrap_or_default(); + } + pub fn has_direct_dial_info(&self) -> bool { !self.dial_info_detail_list.is_empty() } @@ -693,31 +741,31 @@ impl ProtocolType { ProtocolType::TCP | ProtocolType::WS | ProtocolType::WSS => LowLevelProtocolType::TCP, } } - pub fn sort_order(&self, reliable: bool) -> usize { + pub fn sort_order(&self, sequencing: Sequencing) -> usize { match self { ProtocolType::UDP => { - if reliable { + if sequencing != Sequencing::NoPreference { 3 } else { 0 } } ProtocolType::TCP => { - if reliable { + if sequencing != Sequencing::NoPreference { 0 } else { 1 } } ProtocolType::WS => { - if reliable { + if sequencing != Sequencing::NoPreference { 1 } else { 2 } } ProtocolType::WSS => { - if reliable { + if sequencing != Sequencing::NoPreference { 2 } else { 3 @@ -725,6 +773,9 @@ impl ProtocolType { } } } + pub fn all_ordered_set() -> ProtocolTypeSet { + ProtocolType::TCP | ProtocolType::WS | ProtocolType::WSS + } } pub type ProtocolTypeSet = EnumSet; @@ -1499,7 +1550,7 @@ impl DialInfo { } } - pub fn reliable_sort(a: &DialInfo, b: &DialInfo) -> core::cmp::Ordering { + pub fn ordered_sequencing_sort(a: &DialInfo, b: &DialInfo) -> core::cmp::Ordering { let ca = a.protocol_type().sort_order(true); let cb = b.protocol_type().sort_order(true); if ca < cb { From be55a42878182341ebbc96649d20e966e5066fbc Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 21 Oct 2022 21:27:07 -0400 Subject: [PATCH 21/67] route work --- veilid-core/src/network_manager/mod.rs | 4 +- veilid-core/src/receipt_manager.rs | 2 +- .../src/routing_table/route_spec_store.rs | 21 ----- .../src/routing_table/routing_domains.rs | 63 ++++++++++----- veilid-core/src/rpc_processor/destination.rs | 76 +++++++++--------- veilid-core/src/rpc_processor/mod.rs | 45 ++++++----- veilid-core/src/rpc_processor/rpc_route.rs | 3 + veilid-core/src/veilid_api/mod.rs | 26 +++++- veilid-core/src/veilid_api/routing_context.rs | 79 +++++++++---------- 9 files changed, 175 insertions(+), 144 deletions(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 4bb5178a..83a9ed73 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1021,7 +1021,7 @@ impl NetworkManager { // We expect the inbound noderef to be the same as the target noderef // if they aren't the same, we should error on this and figure out what then hell is up - if target_nr != inbound_nr { + if target_nr.node_id() != inbound_nr.node_id() { bail!("unexpected noderef mismatch on reverse connect"); } @@ -1122,7 +1122,7 @@ impl NetworkManager { // We expect the inbound noderef to be the same as the target noderef // if they aren't the same, we should error on this and figure out what then hell is up - if target_nr != inbound_nr { + if target_nr.node_id() != inbound_nr.node_id() { bail!( "unexpected noderef mismatch on hole punch {}, expected {}", inbound_nr, diff --git a/veilid-core/src/receipt_manager.rs b/veilid-core/src/receipt_manager.rs index 4ad56db9..02bf8a75 100644 --- a/veilid-core/src/receipt_manager.rs +++ b/veilid-core/src/receipt_manager.rs @@ -7,7 +7,7 @@ use routing_table::*; use stop_token::future::FutureExt; use xx::*; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug)] pub enum ReceiptEvent { ReturnedOutOfBand, ReturnedInBand { inbound_noderef: NodeRef }, diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 9d6e6485..b0805baa 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -2,18 +2,6 @@ use super::*; use crate::veilid_api::*; use serde::*; -/// Options for safety routes (sender privacy) -#[derive(Copy, Clone, Debug, Serialize, Deserialize)] -pub struct SafetySpec { - /// preferred safety route if it still exists - pub preferred_route: Option, - /// 0 = no safety route, just use node's node id, more hops is safer but slower - pub hop_count: usize, - /// prefer reliability over speed - pub stability: Stability, - /// prefer connection-oriented sequenced protocols - pub sequencing: Sequencing, -} /// Compiled route (safety route + private route) #[derive(Clone, Debug)] @@ -98,15 +86,6 @@ pub struct RouteSpecStore { cache: RouteSpecStoreCache, } -/// The choice of safety route including in compiled routes -#[derive(Debug, Clone)] -pub enum SafetySelection { - /// Don't use a safety route, only specify the sequencing preference - Unsafe(Sequencing), - /// Use a safety route and parameters specified by a SafetySpec - Safe(SafetySpec), -} - fn route_hops_to_hop_cache(hops: &[DHTKey]) -> Vec { let mut cache: Vec = Vec::with_capacity(hops.len() * DHT_KEY_LENGTH); for hop in hops { diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 0e164b34..7a7cc0e8 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -204,21 +204,34 @@ fn first_filtered_dial_info_detail( from_node: &NodeInfo, to_node: &NodeInfo, dial_info_filter: &DialInfoFilter, - reliable: bool, xxx continue here + sequencing: Sequencing, ) -> Option { - let direct_dial_info_filter = dial_info_filter.clone().filtered( + let dial_info_filter = dial_info_filter.clone().filtered( &DialInfoFilter::all() .with_address_type_set(from_node.address_types) .with_protocol_type_set(from_node.outbound_protocols), ); // Get first filtered dialinfo - let sort = if reliable { - Some(DialInfoDetail::ordered_sequencing_sort) - } else { - None + let (sort, dial_info_filter) = match sequencing { + Sequencing::NoPreference => (None, dial_info_filter), + Sequencing::PreferOrdered => ( + Some(DialInfoDetail::ordered_sequencing_sort), + dial_info_filter, + ), + Sequencing::EnsureOrdered => ( + Some(DialInfoDetail::ordered_sequencing_sort), + dial_info_filter.filtered( + &DialInfoFilter::all().with_protocol_type_set(ProtocolType::all_ordered_set()), + ), + ), }; - let direct_filter = |did: &DialInfoDetail| did.matches_filter(&direct_dial_info_filter); + // If the filter is dead then we won't be able to connect + if dial_info_filter.is_dead() { + return None; + } + + let direct_filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); // Get the best match dial info for node B if we have it to_node.first_filtered_dial_info_detail(sort, direct_filter) @@ -242,11 +255,11 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { node_b_id: &DHTKey, node_b: &NodeInfo, dial_info_filter: DialInfoFilter, - reliable: bool, + sequencing: Sequencing, ) -> ContactMethod { // Get the best match dial info for node B if we have it if let Some(target_did) = - first_filtered_dial_info_detail(node_a, node_b, &dial_info_filter, reliable) + first_filtered_dial_info_detail(node_a, node_b, &dial_info_filter, sequencing) { // Do we need to signal before going inbound? if !target_did.class.requires_signal() { @@ -267,7 +280,7 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { node_a, &inbound_relay.signed_node_info.node_info, &dial_info_filter, - reliable, + sequencing, ) .is_some() { @@ -280,7 +293,7 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { node_b, node_a, &dial_info_filter, - reliable, + sequencing, ) { // Ensure we aren't on the same public IP address (no hairpin nat) if reverse_did.dial_info.to_ip_addr() @@ -306,14 +319,14 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { node_a, node_b, &udp_dial_info_filter, - reliable, + sequencing, ) { // Does node A have a direct udp dialinfo that node B can reach? if let Some(reverse_udp_did) = first_filtered_dial_info_detail( node_b, node_a, &udp_dial_info_filter, - reliable, + sequencing, ) { // Ensure we aren't on the same public IP address (no hairpin nat) if reverse_udp_did.dial_info.to_ip_addr() @@ -341,7 +354,7 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { node_a, &inbound_relay.signed_node_info.node_info, &dial_info_filter, - reliable, + sequencing, ) .is_some() { @@ -412,7 +425,7 @@ impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { _node_b_id: &DHTKey, node_b: &NodeInfo, dial_info_filter: DialInfoFilter, - reliable: bool, + sequencing: Sequencing, ) -> ContactMethod { // Scope the filter down to protocols node A can do outbound let dial_info_filter = dial_info_filter.filtered( @@ -421,17 +434,25 @@ impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { .with_protocol_type_set(node_a.outbound_protocols), ); + // Get first filtered dialinfo + let (sort, dial_info_filter) = match sequencing { + Sequencing::NoPreference => (None, dial_info_filter), + Sequencing::PreferOrdered => ( + Some(DialInfoDetail::ordered_sequencing_sort), + dial_info_filter, + ), + Sequencing::EnsureOrdered => ( + Some(DialInfoDetail::ordered_sequencing_sort), + dial_info_filter.filtered( + &DialInfoFilter::all().with_protocol_type_set(ProtocolType::all_ordered_set()), + ), + ), + }; // If the filter is dead then we won't be able to connect if dial_info_filter.is_dead() { return ContactMethod::Unreachable; } - // Get first filtered dialinfo - let sort = if reliable { - Some(DialInfoDetail::ordered_sequencing_sort) - } else { - None - }; let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); let opt_target_did = node_b.first_filtered_dial_info_detail(sort, filter); diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index a611275b..d68ba5f4 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -8,7 +8,7 @@ pub enum Destination { /// The node to send to target: NodeRef, /// Require safety route or not - safety_spec: Option, + safety_selection: SafetySelection, }, /// Send to node for relay purposes Relay { @@ -17,16 +17,14 @@ pub enum Destination { /// The final destination the relay should send to target: DHTKey, /// Require safety route or not - safety_spec: Option, + safety_selection: SafetySelection, }, /// Send to private route (privateroute) PrivateRoute { /// A private route to send to private_route: PrivateRoute, /// Require safety route or not - safety_spec: Option, - /// Prefer reliability or not - reliable: bool, + safety_selection: SafetySelection, }, } @@ -34,70 +32,66 @@ impl Destination { pub fn direct(target: NodeRef) -> Self { Self::Direct { target, - safety_spec: None, + safety_selection: SafetySelection::Unsafe(target.sequencing()), } } pub fn relay(relay: NodeRef, target: DHTKey) -> Self { Self::Relay { relay, target, - safety_spec: None, + safety_selection: SafetySelection::Unsafe(relay.sequencing()), } } - pub fn private_route(private_route: PrivateRoute, reliable: bool) -> Self { + pub fn private_route(private_route: PrivateRoute, safety_selection: SafetySelection) -> Self { Self::PrivateRoute { private_route, - safety_spec: None, - reliable, + safety_selection, } } - pub fn with_safety(self, safety_spec: SafetySpec) -> Self { + pub fn with_safety(self, safety_selection: SafetySelection) -> Self { match self { Destination::Direct { target, - safety_spec: _, + safety_selection: _, } => Self::Direct { target, - safety_spec: Some(safety_spec), + safety_selection, }, Destination::Relay { relay, target, - safety_spec: _, + safety_selection: _, } => Self::Relay { relay, target, - safety_spec: Some(safety_spec), + safety_selection, }, Destination::PrivateRoute { private_route, - safety_spec: _, - reliable, + safety_selection: _, } => Self::PrivateRoute { private_route, - safety_spec: Some(safety_spec), - reliable, + safety_selection, }, } } - pub fn get_safety_spec(&self) -> &Option { + pub fn get_safety_selection(&self) -> &SafetySelection { match self { Destination::Direct { target: _, - safety_spec, - } => safety_spec, + safety_selection, + } => safety_selection, Destination::Relay { relay: _, target: _, - safety_spec, - } => safety_spec, + safety_selection, + } => safety_selection, Destination::PrivateRoute { private_route: _, - safety_spec, - reliable: _, - } => safety_spec, + safety_selection, + } => safety_selection, } } } @@ -107,30 +101,40 @@ impl fmt::Display for Destination { match self { Destination::Direct { target, - safety_spec, + safety_selection, } => { - let sr = if safety_spec.is_some() { "+SR" } else { "" }; + let sr = if matches!(safety_selection, SafetySelection::Safe(_)) { + "+SR" + } else { + "" + }; write!(f, "{}{}", target, sr) } Destination::Relay { relay, target, - safety_spec, + safety_selection, } => { - let sr = if safety_spec.is_some() { "+SR" } else { "" }; + let sr = if matches!(safety_selection, SafetySelection::Safe(_)) { + "+SR" + } else { + "" + }; write!(f, "{}@{}{}", target.encode(), relay, sr) } Destination::PrivateRoute { private_route, - safety_spec, - reliable, + safety_selection, } => { - let sr = if safety_spec.is_some() { "+SR" } else { "" }; - let rl = if *reliable { "+RL" } else { "" }; + let sr = if matches!(safety_selection, SafetySelection::Safe(_)) { + "+SR" + } else { + "" + }; - write!(f, "{}{}{}", private_route, sr, rl) + write!(f, "{}{}", private_route, sr) } } } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 709ab9b1..7064b256 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -400,9 +400,8 @@ impl RPCProcessor { // Wrap an operation with a private route inside a safety route pub(super) fn wrap_with_route( &self, - safety_spec: Option, + safety_selection: SafetySelection, private_route: PrivateRoute, - reliable: bool, message_data: Vec, ) -> Result, RPCError> { let routing_table = self.routing_table(); @@ -412,7 +411,7 @@ impl RPCProcessor { let compiled_route: CompiledRoute = match self.routing_table().with_route_spec_store_mut(|rss, rti| { // Compile the safety route with the private route - rss.compile_safety_route(rti, routing_table, safety_spec, private_route, reliable) + rss.compile_safety_route(rti, routing_table, safety_selection, private_route) })? { Some(cr) => cr, None => { @@ -456,7 +455,6 @@ impl RPCProcessor { let out_node_id = compiled_route.first_hop.node_id(); let out_hop_count = (1 + sr_hop_count + pr_hop_count) as usize; - let out = RenderedOperation { message: out_message, node_id: out_node_id, @@ -491,12 +489,12 @@ impl RPCProcessor { match dest { Destination::Direct { target: ref node_ref, - safety_spec, + safety_selection, } | Destination::Relay { relay: ref node_ref, target: _, - safety_spec, + safety_selection, } => { // Send to a node without a private route // -------------------------------------- @@ -505,7 +503,7 @@ impl RPCProcessor { let (node_ref, node_id) = if let Destination::Relay { relay: _, target: ref dht_key, - safety_spec: _, + safety_selection: _, } = dest { (node_ref.clone(), dht_key.clone()) @@ -515,8 +513,13 @@ impl RPCProcessor { }; // Handle the existence of safety route - match safety_spec { - None => { + match safety_selection { + SafetySelection::Unsafe(sequencing) => { + // Apply safety selection sequencing requirement if it is more strict than the node_ref's sequencing requirement + if sequencing > node_ref.sequencing() { + node_ref.set_sequencing(sequencing) + } + // If no safety route is being used, and we're not sending to a private // route, we can use a direct envelope instead of routing out = NetworkResult::value(RenderedOperation { @@ -526,25 +529,24 @@ impl RPCProcessor { hop_count: 1, }); } - Some(safety_spec) => { + SafetySelection::Safe(_) => { // No private route was specified for the request // but we are using a safety route, so we must create an empty private route let private_route = PrivateRoute::new_stub(node_id); // Wrap with safety route - out = self.wrap_with_route(Some(safety_spec), private_route, message)?; + out = self.wrap_with_route(safety_selection, private_route, message)?; } }; } Destination::PrivateRoute { private_route, - safety_spec, - reliable, xxxx does this need to be here? what about None safety spec, reliable is in there, does it need to not be? or something? + safety_selection, } => { // Send to private route // --------------------- // Reply with 'route' operation - out = self.wrap_with_route(safety_spec, private_route, reliable, message)?; + out = self.wrap_with_route(safety_selection, private_route, message)?; } } @@ -559,8 +561,11 @@ impl RPCProcessor { // Don't do this if the sender is to remain private // Otherwise we would be attaching the original sender's identity to the final destination, // thus defeating the purpose of the safety route entirely :P - if dest.get_safety_spec().is_some() { - return None; + match dest.get_safety_selection() { + SafetySelection::Unsafe(_) => {} + SafetySelection::Safe(_) => { + return None; + } } // Don't do this if our own signed node info isn't valid yet let routing_table = self.routing_table(); @@ -571,7 +576,7 @@ impl RPCProcessor { match dest { Destination::Direct { target, - safety_spec: _, + safety_selection: _, } => { // If the target has seen our node info already don't do this if target.has_seen_our_node_info(RoutingDomain::PublicInternet) { @@ -582,7 +587,7 @@ impl RPCProcessor { Destination::Relay { relay: _, target, - safety_spec: _, + safety_selection: _, } => { if let Some(target) = routing_table.lookup_node_ref(*target) { if target.has_seen_our_node_info(RoutingDomain::PublicInternet) { @@ -595,8 +600,7 @@ impl RPCProcessor { } Destination::PrivateRoute { private_route: _, - safety_spec: _, - reliable: _, + safety_selection: _, } => None, } } @@ -741,6 +745,7 @@ impl RPCProcessor { Destination::relay(peer_noderef, sender_id) } } + //xxx needs to know what route the request came in on in order to reply over that same route as the preferred safety route RespondTo::PrivateRoute(pr) => Destination::private_route( pr.clone(), request diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 29729bb7..3833318a 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -7,6 +7,9 @@ impl RPCProcessor { pub(crate) async fn process_route(&self, msg: RPCMessage) -> Result<(), RPCError> { // xxx do not process latency for routed messages // tracing::Span::current().record("res", &tracing::field::display(res)); + + xxx continue here + Err(RPCError::unimplemented("process_route")) } } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 8f2ac241..9d16b15b 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -420,6 +420,28 @@ pub enum Stability { Reliable, } +/// The choice of safety route including in compiled routes +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum SafetySelection { + /// Don't use a safety route, only specify the sequencing preference + Unsafe(Sequencing), + /// Use a safety route and parameters specified by a SafetySpec + Safe(SafetySpec), +} + +/// Options for safety routes (sender privacy) +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct SafetySpec { + /// preferred safety route if it still exists + pub preferred_route: Option, + /// 0 = no safety route, just use node's node id, more hops is safer but slower + pub hop_count: usize, + /// prefer reliability over speed + pub stability: Stability, + /// prefer connection-oriented sequenced protocols + pub sequencing: Sequencing, +} + // Keep member order appropriate for sorting < preference #[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize, Hash)] pub struct DialInfoDetail { @@ -1551,8 +1573,8 @@ impl DialInfo { } pub fn ordered_sequencing_sort(a: &DialInfo, b: &DialInfo) -> core::cmp::Ordering { - let ca = a.protocol_type().sort_order(true); - let cb = b.protocol_type().sort_order(true); + let ca = a.protocol_type().sort_order(Sequencing::EnsureOrdered); + let cb = b.protocol_type().sort_order(Sequencing::EnsureOrdered); if ca < cb { return core::cmp::Ordering::Less; } diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index e0c3a8b4..47efb369 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -10,10 +10,8 @@ pub enum Target { pub struct RoutingContextInner {} pub struct RoutingContextUnlockedInner { - /// Enforce use of private routing - privacy: usize, - /// Choose reliable protocols over unreliable/faster protocols when available - reliable: bool, + /// Safety routing requirements + safety_selection: SafetySelection, } impl Drop for RoutingContextInner { @@ -41,8 +39,7 @@ impl RoutingContext { api, inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - privacy: 0, - reliable: false, + safety_selection: SafetySelection::Unsafe(Sequencing::NoPreference), }), } } @@ -54,44 +51,54 @@ impl RoutingContext { api: self.api.clone(), inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - privacy: c.network.rpc.default_route_hop_count as usize, - reliable: self.unlocked_inner.reliable, + safety_selection: SafetySelection::Safe(SafetySpec { + preferred_route: None, + hop_count: c.network.rpc.default_route_hop_count as usize, + stability: Stability::LowLatency, + sequencing: Sequencing::NoPreference, + }), }), }) } - pub fn with_privacy(self, hops: usize) -> Result { - let config = self.api.config()?; - let c = config.get(); - - let privacy = if hops > 0 && hops <= c.network.rpc.max_route_hop_count as usize { - hops - } else { - return Err(VeilidAPIError::invalid_argument( - "hops value is too large", - "hops", - hops, - )); - }; + pub fn with_privacy(self, safety_spec: SafetySpec) -> Result { Ok(Self { api: self.api.clone(), inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - privacy, - reliable: self.unlocked_inner.reliable, + safety_selection: SafetySelection::Safe(safety_spec), }), }) } - pub fn with_reliability(self) -> Self { + pub fn with_sequencing(self, sequencing: Sequencing) -> Self { Self { api: self.api.clone(), inner: Arc::new(Mutex::new(RoutingContextInner {})), unlocked_inner: Arc::new(RoutingContextUnlockedInner { - privacy: self.unlocked_inner.privacy, - reliable: true, + safety_selection: match self.unlocked_inner.safety_selection { + SafetySelection::Unsafe(_) => SafetySelection::Unsafe(sequencing), + SafetySelection::Safe(safety_spec) => SafetySelection::Safe(SafetySpec { + preferred_route: safety_spec.preferred_route, + hop_count: safety_spec.hop_count, + stability: safety_spec.stability, + sequencing, + }), + }, }), } } + pub fn sequencing(&self) -> Sequencing { + match self.unlocked_inner.safety_selection { + SafetySelection::Unsafe(sequencing) => sequencing, + SafetySelection::Safe(safety_spec) => safety_spec.sequencing, + } + } + pub fn safety_spec(&self) -> Option { + match self.unlocked_inner.safety_selection { + SafetySelection::Unsafe(_) => None, + SafetySelection::Safe(safety_spec) => Some(safety_spec.clone()), + } + } pub fn api(&self) -> VeilidAPI { self.api.clone() @@ -111,27 +118,17 @@ impl RoutingContext { Ok(None) => return Err(VeilidAPIError::NodeNotFound { node_id }), Err(e) => return Err(e.into()), }; - // Apply reliability sort - if self.unlocked_inner.reliable { - nr.set_reliable(); - } + // Apply sequencing to match safety selection + nr.set_sequencing(self.sequencing()); + Ok(rpc_processor::Destination::Direct { target: nr, - safety_spec: Some(routing_table::SafetySpec { - preferred_route: None, - hop_count: self.unlocked_inner.privacy, - reliable: self.unlocked_inner.reliable, - }), + safety_selection: self.unlocked_inner.safety_selection, }) } Target::PrivateRoute(pr) => Ok(rpc_processor::Destination::PrivateRoute { private_route: pr, - safety_spec: Some(routing_table::SafetySpec { - preferred_route: None, - hop_count: self.unlocked_inner.privacy, - reliable: self.unlocked_inner.reliable, - }), - reliable: self.unlocked_inner.reliable, + safety_selection: self.unlocked_inner.safety_selection, }), } } From d335b56571021c5098e369078ac0f743a610584b Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 28 Oct 2022 22:26:21 -0400 Subject: [PATCH 22/67] route work --- veilid-core/proto/veilid.capnp | 10 +- .../src/routing_table/route_spec_store.rs | 3 +- .../coders/private_safety_route.rs | 20 +- veilid-core/src/rpc_processor/rpc_route.rs | 232 +++++++++++++++++- veilid-core/src/veilid_api/privacy.rs | 4 +- 5 files changed, 246 insertions(+), 23 deletions(-) diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index b972b191..09cce2a4 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -136,22 +136,22 @@ struct RouteHop { nodeId @0 :NodeID; # node id only for established routes peerInfo @1 :PeerInfo; # full peer info for this hop to establish the route } - nextHop @2 :RouteHopData; # Optional: next hop in encrypted blob - # Null means no next hop, at destination (only used in private route, safety routes must enclose a stub private route) + nextHop @2 :RouteHopData; # Next hop in encrypted blob } struct PrivateRoute { publicKey @0 :RoutePublicKey; # private route public key (unique per private route) - hopCount @1 :UInt8; # Count of hops left in the private route - firstHop @2 :RouteHop; # Optional: first hop in the private route + hopCount @1 :UInt8; # Count of hops left in the private route (for timeout calculation purposes only) + firstHop @2 :RouteHop; # Optional: first hop in the private route, if empty, this is the last hop and payload should be decrypted and processed. } struct SafetyRoute { publicKey @0 :RoutePublicKey; # safety route public key (unique per safety route) - hopCount @1 :UInt8; # Count of hops left in the safety route + hopCount @1 :UInt8; # Count of hops left in the safety route (for timeout calculation purposes only) hops :union { data @2 :RouteHopData; # safety route has more hops private @3 :PrivateRoute; # safety route has ended and private route follows + xxx find better representation for privateroute stub (going straight to node) } } diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index b0805baa..febe51f6 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -2,7 +2,6 @@ use super::*; use crate::veilid_api::*; use serde::*; - /// Compiled route (safety route + private route) #[derive(Clone, Debug)] pub struct CompiledRoute { @@ -771,7 +770,7 @@ impl RouteSpecStore { RouteNode::PeerInfo(pi.unwrap()) } }, - next_hop: Some(route_hop_data), + next_hop: route_hop_data, }; // Make next blob from route hop diff --git a/veilid-core/src/rpc_processor/coders/private_safety_route.rs b/veilid-core/src/rpc_processor/coders/private_safety_route.rs index 50604970..a7714353 100644 --- a/veilid-core/src/rpc_processor/coders/private_safety_route.rs +++ b/veilid-core/src/rpc_processor/coders/private_safety_route.rs @@ -60,10 +60,10 @@ pub fn encode_route_hop( encode_peer_info(&pi, &mut pi_builder)?; } } - if let Some(rhd) = &route_hop.next_hop { - let mut rhd_builder = builder.reborrow().init_next_hop(); - encode_route_hop_data(rhd, &mut rhd_builder)?; - } + + let mut rhd_builder = builder.reborrow().init_next_hop(); + encode_route_hop_data(&route_hop.next_hop, &mut rhd_builder)?; + Ok(()) } @@ -83,14 +83,10 @@ pub fn decode_route_hop(reader: &veilid_capnp::route_hop::Reader) -> Result Result<(), RPCError> { + // Get next hop node ref + let next_hop_nr = match route_hop.node { + RouteNode::NodeId(id) => { + // + self.routing_table + .lookup_node_ref(id.key) + .ok_or_else(|| RPCError::network(format!("node hop {} not found", id.key))) + } + RouteNode::PeerInfo(pi) => { + // + self.routing_table + .register_node_with_signed_node_info( + RoutingDomain::PublicInternet, + pi.node_id.key, + pi.signed_node_info, + false, + ) + .ok_or_else(|| { + RPCError::network(format!( + "node hop {} could not be registered", + pi.node_id.key + )) + }) + } + }?; + + // Pass along the route + let next_hop_route = RPCOperationRoute { + safety_route: SafetyRoute { + public_key: route.safety_route.public_key, + hop_count: route.safety_route.hop_count - 1, + hops: SafetyRouteHops::Data(route_hop.next_hop), + }, + operation: route.operation, + }; + let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route)); + + // Send the next route statement + network_result_try!( + self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt) + .await? + ); + + Ok(()) + } + + async fn process_route_safety_route_private_route_hop( + &self, + route: &RPCOperationRoute, + private_route: &PrivateRoute, + ) -> Result<(), RPCError> { + // + let route_hop = private_route.first_hop.unwrap(); + + // Pass along the route + let next_hop_route = RPCOperationRoute { + safety_route: SafetyRoute { + public_key: route.safety_route.public_key, + hop_count: 0, + hops: SafetyRouteHops::PrivateRoute(Private), + }, + operation: route.operation, + }; + let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route)); + + // Send the next route statement + network_result_try!( + self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt) + .await? + ); + + Ok(()) + } + async fn process_routed_operation( + &self, + route: &RPCOperationRoute, + private_route: &PrivateRoute, + ) -> Result<(), RPCError> { + // + Ok(()) + } #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] pub(crate) async fn process_route(&self, msg: RPCMessage) -> Result<(), RPCError> { // xxx do not process latency for routed messages // tracing::Span::current().record("res", &tracing::field::display(res)); - xxx continue here + // Get the statement + let route = match msg.operation.kind() { + RPCOperationKind::Statement(s) => match s.detail() { + RPCStatementDetail::Route(s) => s, + _ => panic!("not a route statement"), + }, + _ => panic!("not a statement"), + }; - Err(RPCError::unimplemented("process_route")) + // See what kind of safety route we have going on here + match &route.safety_route.hops { + // There is a safety route hop + SafetyRouteHops::Data(d) => { + // See if this is last hop in safety route, if so, we're decoding a PrivateRoute not a RouteHop + let (blob_tag, blob_data) = if let Some(b) = d.blob.last() { + (*b, &d.blob[0..d.blob.len() - 1]) + } else { + return Err(RPCError::protocol("no bytes in blob")); + }; + + // Decrypt the blob with DEC(nonce, DH(the SR's public key, this hop's secret) + let node_id_secret = self.routing_table.node_id_secret(); + let dh_secret = self + .crypto + .cached_dh(&route.safety_route.public_key, &node_id_secret) + .map_err(RPCError::protocol)?; + let dec_blob_data = Crypto::decrypt_aead(blob_data, &d.nonce, &dh_secret, None) + .map_err(RPCError::map_internal("encryption failed"))?; + let dec_blob_reader = capnp::message::Reader::new( + RPCMessageData { + contents: dec_blob_data, + }, + Default::default(), + ); + + // Decode the blob appropriately + if blob_tag == 0 { + // PrivateRoute + let private_route = { + let pr_reader = dec_blob_reader + .get_root::() + .map_err(RPCError::protocol)?; + decode_private_route(&pr_reader)? + }; + + // Make sure hop count makes sense + if route.safety_route.hop_count as usize != 0 { + return Err(RPCError::protocol( + "Safety hop count should be zero if switched to private route", + )); + } + + // Get the next hop node ref + if private_route.first_hop.is_some() { + // Make sure hop count makes sense + if private_route.hop_count as usize + > self.unlocked_inner.max_route_hop_count + { + return Err(RPCError::protocol( + "Private route hop count too high to process", + )); + } + if private_route.hop_count == 0 { + return Err(RPCError::protocol( + "Private route hop count should not be zero if there are more hops", + )); + } + + // Switching to private route from safety route + self.process_route_safety_route_private_route_hop(route, &private_route) + .await?; + } else { + // Make sure hop count makes sense + if private_route.hop_count != 0 { + return Err(RPCError::protocol( + "Private route hop count should be zero if we are at the end", + )); + } + + // Private route was a stub, process routed operation + self.process_routed_operation(route, &private_route).await?; + } + } else if blob_tag == 1 { + // RouteHop + let route_hop = { + let rh_reader = dec_blob_reader + .get_root::() + .map_err(RPCError::protocol)?; + decode_route_hop(&rh_reader)? + }; + + // Make sure hop count makes sense + if route.safety_route.hop_count as usize + > self.unlocked_inner.max_route_hop_count + { + return Err(RPCError::protocol( + "Safety route hop count too high to process", + )); + } + if route.safety_route.hop_count == 0 { + return Err(RPCError::protocol( + "Safety route hop count should not be zero if there are more hops", + )); + } + + self.process_route_safety_route_hop(route, route_hop) + .await?; + } else { + return Err(RPCError::protocol("invalid blob tag")); + } + } + // Safety route has ended, now do private route + SafetyRouteHops::Private(private_route) => { + if private_route.first_hop.is_some() { + // Make sure hop count makes sense + if private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { + return Err(RPCError::protocol( + "Private route hop count too high to process", + )); + } + if private_route.hop_count == 0 { + return Err(RPCError::protocol( + "Private route hop count should not be zero if there are more hops", + )); + } + + // There are some hops left + self.process_route_safety_route_private_route_hop(route, private_route) + .await?; + } else { + // Make sure hop count makes sense + if private_route.hop_count != 0 { + return Err(RPCError::protocol( + "Private route hop count should be zero if we are at the end", + )); + } + + // No hops left, time to process the routed operation + self.process_routed_operation(route, private_route).await?; + } + } + } + + Ok(()) } } diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index a47721a0..a167339a 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -30,7 +30,7 @@ impl fmt::Display for RouteNode { #[derive(Clone, Debug)] pub struct RouteHop { pub node: RouteNode, - pub next_hop: Option, + pub next_hop: RouteHopData, } #[derive(Clone, Debug)] @@ -81,7 +81,9 @@ impl fmt::Display for PrivateRoute { #[derive(Clone, Debug)] pub enum SafetyRouteHops { + /// Has >= 1 safety route hops Data(RouteHopData), + /// Has 0 safety route hops Private(PrivateRoute), } From d94a023c322d2e9ef5824c10fae6ded2fd495d94 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 29 Oct 2022 22:15:50 -0400 Subject: [PATCH 23/67] route work --- veilid-core/proto/veilid.capnp | 14 +- veilid-core/src/network_manager/mod.rs | 4 +- .../src/routing_table/route_spec_store.rs | 16 +- .../coders/private_safety_route.rs | 20 +- veilid-core/src/rpc_processor/mod.rs | 148 ++++++-- veilid-core/src/rpc_processor/rpc_route.rs | 342 +++++++++++++----- veilid-core/src/veilid_api/privacy.rs | 20 +- 7 files changed, 420 insertions(+), 144 deletions(-) diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 09cce2a4..0d1ec7e3 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -126,9 +126,11 @@ struct SignalInfoReverseConnect { struct RouteHopData { nonce @0 :Nonce; # nonce for encrypted blob blob @1 :Data; # encrypted blob with ENC(nonce,DH(PK,SK)) - # can be one of: - # if more hops remain in this route: RouteHop (0 byte appended as key) - # if end of safety route and starting private route: PrivateRoute (1 byte appended as key) + # if this is a safety route RouteHopData, there is a single byte tag appended to the end of the encrypted blob + # it can be one of: + # if more hops remain in this route: RouteHop (0 byte appended as tag) + # if end of safety route and starting private route: PrivateRoute (1 byte appended as tag) + # if this is a private route RouteHopData, only can decode to RouteHop, no tag is appended } struct RouteHop { @@ -136,14 +138,15 @@ struct RouteHop { nodeId @0 :NodeID; # node id only for established routes peerInfo @1 :PeerInfo; # full peer info for this hop to establish the route } - nextHop @2 :RouteHopData; # Next hop in encrypted blob + nextHop @2 :RouteHopData; # Optional: if the private route is a stub, it contains no route hop data, just the target node for the routed operation. + # if this is a safety route routehop, this field is not optional and must exist } struct PrivateRoute { publicKey @0 :RoutePublicKey; # private route public key (unique per private route) hopCount @1 :UInt8; # Count of hops left in the private route (for timeout calculation purposes only) firstHop @2 :RouteHop; # Optional: first hop in the private route, if empty, this is the last hop and payload should be decrypted and processed. -} +} struct SafetyRoute { publicKey @0 :RoutePublicKey; # safety route public key (unique per safety route) @@ -151,7 +154,6 @@ struct SafetyRoute { hops :union { data @2 :RouteHopData; # safety route has more hops private @3 :PrivateRoute; # safety route has ended and private route follows - xxx find better representation for privateroute stub (going straight to node) } } diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 83a9ed73..75969de0 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1569,12 +1569,12 @@ impl NetworkManager { // xxx: deal with spoofing and flooding here? // Pass message to RPC system - rpc.enqueue_message( + rpc.enqueue_direct_message( envelope, - body, source_noderef, connection_descriptor, routing_domain, + body, )?; // Inform caller that we dealt with the envelope locally diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index febe51f6..369c3be6 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -17,9 +17,9 @@ pub struct CompiledRoute { struct RouteSpecDetail { /// Secret key #[serde(skip)] - secret_key: DHTKeySecret, + pub secret_key: DHTKeySecret, /// Route hops - hops: Vec, + pub hops: Vec, /// Route noderefs #[serde(skip)] hop_node_refs: Vec, @@ -273,6 +273,9 @@ impl RouteSpecStore { } } + fn detail(&self, public_key: &DHTKey) -> Option<&RouteSpecDetail> { + self.content.details.get(&public_key) + } fn detail_mut(&mut self, public_key: &DHTKey) -> Option<&mut RouteSpecDetail> { self.content.details.get_mut(&public_key) } @@ -534,6 +537,13 @@ impl RouteSpecStore { Ok(Some(public_key)) } + pub fn with_route_spec_detail(&self, public_key: &DHTKey, f: F) -> Option + where + F: FnOnce(&RouteSpecDetail) -> R, + { + self.detail(&public_key).map(|rsd| f(rsd)) + } + pub fn release_route(&mut self, public_key: DHTKey) { if let Some(detail) = self.content.details.remove(&public_key) { // Remove from hop cache @@ -770,7 +780,7 @@ impl RouteSpecStore { RouteNode::PeerInfo(pi.unwrap()) } }, - next_hop: route_hop_data, + next_hop: Some(route_hop_data), }; // Make next blob from route hop diff --git a/veilid-core/src/rpc_processor/coders/private_safety_route.rs b/veilid-core/src/rpc_processor/coders/private_safety_route.rs index a7714353..50604970 100644 --- a/veilid-core/src/rpc_processor/coders/private_safety_route.rs +++ b/veilid-core/src/rpc_processor/coders/private_safety_route.rs @@ -60,10 +60,10 @@ pub fn encode_route_hop( encode_peer_info(&pi, &mut pi_builder)?; } } - - let mut rhd_builder = builder.reborrow().init_next_hop(); - encode_route_hop_data(&route_hop.next_hop, &mut rhd_builder)?; - + if let Some(rhd) = &route_hop.next_hop { + let mut rhd_builder = builder.reborrow().init_next_hop(); + encode_route_hop_data(rhd, &mut rhd_builder)?; + } Ok(()) } @@ -83,10 +83,14 @@ pub fn decode_route_hop(reader: &veilid_capnp::route_hop::Reader) -> Result, +} + +#[derive(Debug, Clone)] +struct RPCMessageHeaderDetailPrivateRoute { + + /// The private route we received the rpc over + private_route: DHTKey, + // The safety selection for replying to this private routed rpc + safety_selection: SafetySelection, +} + +#[derive(Debug, Clone)] +enum RPCMessageHeaderDetail { + Direct(RPCMessageHeaderDetailDirect), + PrivateRoute(RPCMessageHeaderDetailPrivateRoute), +} + +/// The decoded header of an RPC message +#[derive(Debug, Clone)] +struct RPCMessageHeader { + + version: u8, + min_version: u8, + max_version: u8, + timestamp: u64,???? do we need a header for private routed messages? write process_rpc_message + + /// Time the message was received, not sent + timestamp: u64, + /// The length in bytes of the rpc message body + body_len: u64, + /// The header detail depending on which way the message was received + detail: RPCMessageHeaderDetail, } #[derive(Debug)] @@ -532,7 +557,17 @@ impl RPCProcessor { SafetySelection::Safe(_) => { // No private route was specified for the request // but we are using a safety route, so we must create an empty private route - let private_route = PrivateRoute::new_stub(node_id); + let peer_info = match node_ref.make_peer_info(RoutingDomain::PublicInternet) + { + None => { + return Ok(NetworkResult::no_connection_other( + "No PublicInternet peer info for stub private route", + )) + } + Some(pi) => pi, + }; + let private_route = + PrivateRoute::new_stub(node_id, RouteNode::PeerInfo(peer_info)); // Wrap with safety route out = self.wrap_with_route(safety_selection, private_route, message)?; @@ -719,7 +754,7 @@ impl RPCProcessor { } // Convert the 'RespondTo' into a 'Destination' for a response - fn get_respond_to_destination(&self, request: &RPCMessage) -> Destination { + fn get_respond_to_destination(&self, request: &RPCMessage) -> NetworkResult { // Get the question 'respond to' let respond_to = match request.operation.kind() { RPCOperationKind::Question(q) => q.respond_to(), @@ -731,29 +766,47 @@ impl RPCProcessor { // To where should we respond? match respond_to { RespondTo::Sender => { + // Parse out the header detail from the question + let detail = match &request.header.detail { + RPCMessageHeaderDetail::Direct(detail) => detail, + RPCMessageHeaderDetail::PrivateRoute(_) => { + // If this was sent via a private route, we don't know what the sender was, so drop this + return NetworkResult::invalid_message( + "not responding directly to question from private route", + ); + } + }; + // Reply directly to the request's source - let sender_id = request.header.envelope.get_sender_id(); + let sender_id = detail.envelope.get_sender_id(); // This may be a different node's reference than the 'sender' in the case of a relay - let peer_noderef = request.header.peer_noderef.clone(); + let peer_noderef = detail.peer_noderef.clone(); // If the sender_id is that of the peer, then this is a direct reply // else it is a relayed reply through the peer if peer_noderef.node_id() == sender_id { - Destination::direct(peer_noderef) + NetworkResult::value(Destination::direct(peer_noderef)) } else { - Destination::relay(peer_noderef, sender_id) + NetworkResult::value(Destination::relay(peer_noderef, sender_id)) } } - //xxx needs to know what route the request came in on in order to reply over that same route as the preferred safety route - RespondTo::PrivateRoute(pr) => Destination::private_route( - pr.clone(), - request - .header - .connection_descriptor - .protocol_type() - .is_connection_oriented(), - ), + RespondTo::PrivateRoute(pr) => { + let detail = match &request.header.detail { + RPCMessageHeaderDetail::Direct(_) => { + // If this was sent directly, don't respond to a private route as this could give away this node's safety routes + return NetworkResult::invalid_message( + "not responding to private route from direct question", + ); + } + RPCMessageHeaderDetail::PrivateRoute(detail) => detail, + }; + + NetworkResult::value(Destination::private_route( + pr.clone(), + detail.safety_selection.clone(), + )) + } } } @@ -766,7 +819,7 @@ impl RPCProcessor { answer: RPCAnswer, ) -> Result, RPCError> { // Extract destination from respond_to - let dest = self.get_respond_to_destination(&request); + let dest = network_result_try!(self.get_respond_to_destination(&request)); // Get sender info if we should send that let opt_sender_info = self.get_sender_signed_node_info(&dest); @@ -956,22 +1009,57 @@ impl RPCProcessor { } #[instrument(level = "trace", skip(self, body), err)] - pub fn enqueue_message( + pub fn enqueue_direct_message( &self, envelope: Envelope, - body: Vec, peer_noderef: NodeRef, connection_descriptor: ConnectionDescriptor, routing_domain: RoutingDomain, + body: Vec, ) -> EyreResult<()> { let msg = RPCMessageEncoded { header: RPCMessageHeader { + detail: RPCMessageHeaderDetail::Direct(RPCMessageHeaderDetailDirect { + envelope, + peer_noderef, + connection_descriptor, + routing_domain, + }), + timestamp: intf::get_timestamp(), + body_len: body.len() as u64, + }, + data: RPCMessageData { contents: body }, + }; + let send_channel = { + let inner = self.inner.lock(); + inner.send_channel.as_ref().unwrap().clone() + }; + let span_id = Span::current().id(); + send_channel + .try_send((span_id, msg)) + .wrap_err("failed to enqueue received RPC message")?; + Ok(()) + } + + #[instrument(level = "trace", skip(self, body), err)] + pub fn enqueue_private_route_message( + &self, + + xx need rpc version somewhere! the rpc version to decode should be packaged in with the body, and the allocator should ensure the version is compatible at the end node. can append to body, and then pop the last u8. + xx try to write the whole process_rpc_message pipeline first + + private_route: DHTKey, + safety_selection: SafetySelection, + body: Vec, + ) -> EyreResult<()> { + let msg = RPCMessageEncoded { + header: RPCMessageHeader { + detail: RPCMessageHeaderDetail::PrivateRoute(RPCMessageHeaderDetailPrivateRoute { + private_route, + safety_selection, + }), timestamp: intf::get_timestamp(), - envelope, body_len: body.len() as u64, - peer_noderef, - connection_descriptor, - routing_domain, }, data: RPCMessageData { contents: body }, }; diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 9999383c..001bf2bf 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -1,11 +1,27 @@ use super::*; impl RPCProcessor { + #[instrument(level = "trace", skip_all, err)] async fn process_route_safety_route_hop( &self, - route: &RPCOperationRoute, + route: RPCOperationRoute, route_hop: RouteHop, ) -> Result<(), RPCError> { + // Make sure hop count makes sense + if route.safety_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { + return Err(RPCError::protocol( + "Safety route hop count too high to process", + )); + } + if route.safety_route.hop_count == 0 { + return Err(RPCError::protocol( + "Safety route hop count should not be zero if there are more hops", + )); + } + if route_hop.next_hop.is_none() { + return Err(RPCError::protocol("Safety route hop must have next hop")); + } + // Get next hop node ref let next_hop_nr = match route_hop.node { RouteNode::NodeId(id) => { @@ -37,65 +53,214 @@ impl RPCProcessor { safety_route: SafetyRoute { public_key: route.safety_route.public_key, hop_count: route.safety_route.hop_count - 1, - hops: SafetyRouteHops::Data(route_hop.next_hop), + hops: SafetyRouteHops::Data(route_hop.next_hop.unwrap()), }, operation: route.operation, }; let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route)); // Send the next route statement - network_result_try!( + network_result_value_or_log!(debug self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt) - .await? + .await? => { + return Err(RPCError::network("unable to send route statement for next safety route hop")); + } ); - Ok(()) } - async fn process_route_safety_route_private_route_hop( + #[instrument(level = "trace", skip_all, err)] + async fn process_route_private_route_hop( &self, - route: &RPCOperationRoute, - private_route: &PrivateRoute, + route: RPCOperationRoute, + private_route: PrivateRoute, ) -> Result<(), RPCError> { - // - let route_hop = private_route.first_hop.unwrap(); + // Make sure hop count makes sense + if route.safety_route.hop_count != 0 { + return Err(RPCError::protocol( + "Safety hop count should be zero if switched to private route", + )); + } + if private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { + return Err(RPCError::protocol( + "Private route hop count too high to process", + )); + } + if private_route.hop_count == 0 { + return Err(RPCError::protocol( + "Private route hop count should not be zero if there are more hops", + )); + } + + // Get private route first hop (this is validated to not be None before calling this function) + let first_hop = private_route.first_hop.as_ref().unwrap(); + + // Get next hop node ref + let next_hop_nr = match &first_hop.node { + RouteNode::NodeId(id) => { + // + self.routing_table + .lookup_node_ref(id.key) + .ok_or_else(|| RPCError::network(format!("node hop {} not found", id.key))) + } + RouteNode::PeerInfo(pi) => { + // + self.routing_table + .register_node_with_signed_node_info( + RoutingDomain::PublicInternet, + pi.node_id.key, + pi.signed_node_info.clone(), + false, + ) + .ok_or_else(|| { + RPCError::network(format!( + "node hop {} could not be registered", + pi.node_id.key + )) + }) + } + }?; // Pass along the route let next_hop_route = RPCOperationRoute { safety_route: SafetyRoute { public_key: route.safety_route.public_key, hop_count: 0, - hops: SafetyRouteHops::PrivateRoute(Private), + hops: SafetyRouteHops::Private(private_route), }, operation: route.operation, }; let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route)); // Send the next route statement - network_result_try!( + network_result_value_or_log!(debug self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt) - .await? + .await? => { + return Err(RPCError::network("unable to send route statement for private route hop")); + } ); Ok(()) } + + #[instrument(level = "trace", skip_all, err)] async fn process_routed_operation( &self, - route: &RPCOperationRoute, + sender_id: DHTKey, + route: RPCOperationRoute, private_route: &PrivateRoute, ) -> Result<(), RPCError> { - // + // Make sure hop count makes sense + if route.safety_route.hop_count != 0 { + return Err(RPCError::protocol( + "Safety hop count should be zero if switched to private route", + )); + } + if private_route.hop_count != 0 { + return Err(RPCError::protocol( + "Private route hop count should be zero if we are at the end", + )); + } + + let routed_operation = &route.operation; + + // Get sequencing preference + if route. + + // If the private route public key is our node id, then this was sent via safety route to our node directly + // so there will be no signatures to validate + let opt_pr_info = if private_route.public_key == self.routing_table.node_id() { + // the private route was a stub to our own node's secret + // return our secret key + Some(( + self.routing_table.node_id_secret(), // Figure out how we'd reply to this if it were a question + SafetySelection::Unsafe(sequencing), + )) + } else { + // Look up the private route and ensure it's one in our spec store + let opt_signatures_valid = self.routing_table.with_route_spec_store(|rss, rti| { + rss.with_route_spec_detail(&private_route.public_key, |rsd| { + // Ensure we have the right number of signatures + if routed_operation.signatures.len() != rsd.hops.len() - 1 { + // Wrong number of signatures + log_rpc!(debug "wrong number of signatures ({} should be {}) for routed operation on private route {}", routed_operation.signatures.len(), rsd.hops.len() - 1, private_route.public_key); + return None; + } + // Validate signatures to ensure the route was handled by the nodes and not messed with + for (hop_n, hop_public_key) in rsd.hops.iter().enumerate() { + // The last hop is not signed, as the whole packet is signed + if hop_n == routed_operation.signatures.len() { + // Verify the node we received the routed operation from is the last hop in our route + if *hop_public_key != sender_id { + log_rpc!(debug "received routed operation from the wrong hop ({} should be {}) on private route {}", hop_public_key.encode(), sender_id.encode(), private_route.public_key); + return None; + } + } else { + // Verify a signature for a hop node along the route + if let Err(e) = verify( + hop_public_key, + &routed_operation.data, + &routed_operation.signatures[hop_n], + ) { + log_rpc!(debug "failed to verify signature for hop {} at {} on private route {}", hop_n, hop_public_key, private_route.public_key); + return None; + } + } + } + // Correct signatures + Some(( + rsd.secret_key, + SafetySelection::Safe(SafetySpec { preferred_route: todo!(), hop_count: todo!(), stability: todo!(), sequencing: todo!() }) + )) + }) + }); + opt_signatures_valid.ok_or_else(|| { + RPCError::protocol("routed operation received on unallocated private route") + })? + }; + if opt_pr_info.is_none() { + return Err(RPCError::protocol( + "signatures did not validate for private route", + )); + } + let (secret_key, safety_selection) = opt_pr_info.unwrap(); + + // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret) + // xxx: punish nodes that send messages that fail to decrypt eventually + let dh_secret = self + .crypto + .cached_dh(&route.safety_route.public_key, &secret_key) + .map_err(RPCError::protocol)?; + let body = Crypto::decrypt_aead( + &routed_operation.data, + &routed_operation.nonce, + &dh_secret, + None, + ) + .map_err(RPCError::map_internal( + "decryption of routed operation failed", + ))?; + + // Pass message to RPC system + self.enqueue_private_route_message(private_route.public_key, safety_selection, body) + .map_err(RPCError::internal)?; + Ok(()) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] + #[instrument(level = "trace", skip(self, msg), err)] pub(crate) async fn process_route(&self, msg: RPCMessage) -> Result<(), RPCError> { // xxx do not process latency for routed messages - // tracing::Span::current().record("res", &tracing::field::display(res)); + + // Get header detail, must be direct and not inside a route itself + let (envelope, peer_noderef, connection_descriptor, routing_domain) = match msg.header.detail { + RPCMessageHeaderDetail::Direct { envelope, peer_noderef, connection_descriptor, routing_domain } => (envelope, peer_noderef, connection_descriptor, routing_domain), + RPCMessageHeaderDetail::PrivateRoute { private_route, safety_selection } => { return Err(RPCError::protocol("route operation can not be inside route")) }, + }; // Get the statement - let route = match msg.operation.kind() { - RPCOperationKind::Statement(s) => match s.detail() { + let route = match msg.operation.into_kind() { + RPCOperationKind::Statement(s) => match s.into_detail() { RPCStatementDetail::Route(s) => s, _ => panic!("not a route statement"), }, @@ -103,9 +268,9 @@ impl RPCProcessor { }; // See what kind of safety route we have going on here - match &route.safety_route.hops { + match route.safety_route.hops { // There is a safety route hop - SafetyRouteHops::Data(d) => { + SafetyRouteHops::Data(ref d) => { // See if this is last hop in safety route, if so, we're decoding a PrivateRoute not a RouteHop let (blob_tag, blob_data) = if let Some(b) = d.blob.last() { (*b, &d.blob[0..d.blob.len() - 1]) @@ -120,7 +285,9 @@ impl RPCProcessor { .cached_dh(&route.safety_route.public_key, &node_id_secret) .map_err(RPCError::protocol)?; let dec_blob_data = Crypto::decrypt_aead(blob_data, &d.nonce, &dh_secret, None) - .map_err(RPCError::map_internal("encryption failed"))?; + .map_err(RPCError::map_internal( + "decryption of safety route hop failed", + ))?; let dec_blob_reader = capnp::message::Reader::new( RPCMessageData { contents: dec_blob_data, @@ -129,7 +296,7 @@ impl RPCProcessor { ); // Decode the blob appropriately - if blob_tag == 0 { + if blob_tag == 1 { // PrivateRoute let private_route = { let pr_reader = dec_blob_reader @@ -138,44 +305,21 @@ impl RPCProcessor { decode_private_route(&pr_reader)? }; - // Make sure hop count makes sense - if route.safety_route.hop_count as usize != 0 { - return Err(RPCError::protocol( - "Safety hop count should be zero if switched to private route", - )); - } - // Get the next hop node ref if private_route.first_hop.is_some() { - // Make sure hop count makes sense - if private_route.hop_count as usize - > self.unlocked_inner.max_route_hop_count - { - return Err(RPCError::protocol( - "Private route hop count too high to process", - )); - } - if private_route.hop_count == 0 { - return Err(RPCError::protocol( - "Private route hop count should not be zero if there are more hops", - )); - } - // Switching to private route from safety route - self.process_route_safety_route_private_route_hop(route, &private_route) + self.process_route_private_route_hop(route, private_route) .await?; } else { - // Make sure hop count makes sense - if private_route.hop_count != 0 { - return Err(RPCError::protocol( - "Private route hop count should be zero if we are at the end", - )); - } - - // Private route was a stub, process routed operation - self.process_routed_operation(route, &private_route).await?; + // Private route is empty, process routed operation + self.process_routed_operation( + envelope.get_sender_id(), + route, + &private_route, + ) + .await?; } - } else if blob_tag == 1 { + } else if blob_tag == 0 { // RouteHop let route_hop = { let rh_reader = dec_blob_reader @@ -184,54 +328,66 @@ impl RPCProcessor { decode_route_hop(&rh_reader)? }; - // Make sure hop count makes sense - if route.safety_route.hop_count as usize - > self.unlocked_inner.max_route_hop_count - { - return Err(RPCError::protocol( - "Safety route hop count too high to process", - )); - } - if route.safety_route.hop_count == 0 { - return Err(RPCError::protocol( - "Safety route hop count should not be zero if there are more hops", - )); - } - self.process_route_safety_route_hop(route, route_hop) .await?; } else { return Err(RPCError::protocol("invalid blob tag")); } } - // Safety route has ended, now do private route - SafetyRouteHops::Private(private_route) => { - if private_route.first_hop.is_some() { - // Make sure hop count makes sense - if private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { - return Err(RPCError::protocol( - "Private route hop count too high to process", - )); - } - if private_route.hop_count == 0 { - return Err(RPCError::protocol( - "Private route hop count should not be zero if there are more hops", - )); - } + // No safety route left, now doing private route + SafetyRouteHops::Private(ref private_route) => { + if let Some(first_hop) = &private_route.first_hop { + // See if we have a next hop to send to + let opt_next_first_hop = if let Some(next_hop) = &first_hop.next_hop { + // Decrypt the blob with DEC(nonce, DH(the PR's public key, this hop's secret) + let node_id_secret = self.routing_table.node_id_secret(); + let dh_secret = self + .crypto + .cached_dh(&private_route.public_key, &node_id_secret) + .map_err(RPCError::protocol)?; + let dec_blob_data = + Crypto::decrypt_aead(&next_hop.blob, &next_hop.nonce, &dh_secret, None) + .map_err(RPCError::map_internal( + "decryption of private route hop failed", + ))?; + let dec_blob_reader = capnp::message::Reader::new( + RPCMessageData { + contents: dec_blob_data, + }, + Default::default(), + ); - // There are some hops left - self.process_route_safety_route_private_route_hop(route, private_route) + // Decode next RouteHop + let route_hop = { + let rh_reader = dec_blob_reader + .get_root::() + .map_err(RPCError::protocol)?; + decode_route_hop(&rh_reader)? + }; + Some(route_hop) + } else { + // If the first hop has no RouteHopData, then this is a stub private route + // and we should just pass the operation to its final destination with + // an empty safety and private route + None + }; + + // Make next PrivateRoute and pass it on + let private_route = PrivateRoute { + public_key: private_route.public_key, + hop_count: private_route.hop_count - 1, + first_hop: opt_next_first_hop, + }; + self.process_route_private_route_hop(route, private_route) .await?; } else { - // Make sure hop count makes sense - if private_route.hop_count != 0 { - return Err(RPCError::protocol( - "Private route hop count should be zero if we are at the end", - )); - } - // No hops left, time to process the routed operation - self.process_routed_operation(route, private_route).await?; + self.process_routed_operation( + msg.header.envelope.get_sender_id(), + route, + private_route, + ) + .await?; } } } diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index a167339a..a3c04603 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -30,7 +30,7 @@ impl fmt::Display for RouteNode { #[derive(Clone, Debug)] pub struct RouteHop { pub node: RouteNode, - pub next_hop: RouteHopData, + pub next_hop: Option, } #[derive(Clone, Debug)] @@ -41,13 +41,29 @@ pub struct PrivateRoute { } impl PrivateRoute { - pub fn new_stub(public_key: DHTKey) -> Self { + /// Empty private route is the form used when receiving the last hop + pub fn new_empty(public_key: DHTKey) -> Self { Self { public_key, hop_count: 0, first_hop: None, } } + /// Stub route is the form used when no privacy is required, but you need to specify the destination for a safety route + pub fn new_stub(public_key: DHTKey, node: RouteNode) -> Self { + Self { + public_key, + hop_count: 1, + first_hop: Some(RouteHop { + node, + next_hop: None, + }), + } + } + /// Switch from full node info to simple node id + /// Only simplified single hop, primarily useful for stubs + /// Published routes with >= 1 hops should be in NodeId form already, as they have + /// already been connectivity-verified by the time the route is published pub fn simplify(self) -> Self { Self { public_key: self.public_key, From 50718b707425946218ec8eae9060b92bf5b83aff Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 30 Oct 2022 19:29:31 -0400 Subject: [PATCH 24/67] checkpoint --- doc/config/sample.config | 2 +- veilid-core/proto/veilid.capnp | 7 +- veilid-core/src/attachment_manager.rs | 2 +- veilid-core/src/core_context.rs | 10 +- veilid-core/src/{dht => crypto}/envelope.rs | 17 +- veilid-core/src/{dht => crypto}/key.rs | 0 .../src/{dht/crypto.rs => crypto/mod.rs} | 16 +- veilid-core/src/{dht => crypto}/receipt.rs | 7 +- veilid-core/src/{dht => crypto}/tests/mod.rs | 0 .../src/{dht => crypto}/tests/test_crypto.rs | 0 .../src/{dht => crypto}/tests/test_dht_key.rs | 0 .../tests/test_envelope_receipt.rs | 0 veilid-core/src/{dht => crypto}/value.rs | 0 veilid-core/src/dht/mod.rs | 13 -- veilid-core/src/lib.rs | 2 +- veilid-core/src/network_manager/mod.rs | 14 +- veilid-core/src/network_manager/tasks.rs | 10 +- veilid-core/src/receipt_manager.rs | 2 +- veilid-core/src/routing_table/debug.rs | 4 +- veilid-core/src/routing_table/mod.rs | 2 +- veilid-core/src/routing_table/node_ref.rs | 2 +- .../src/routing_table/route_spec_store.rs | 127 ++++++++++++--- .../src/routing_table/routing_domains.rs | 4 +- .../src/rpc_processor/coders/block_id.rs | 2 +- .../coders/operations/operation.rs | 22 +-- .../operations/operation_node_info_update.rs | 8 +- .../coders/operations/operation_route.rs | 7 +- .../coders/operations/statement.rs | 8 +- .../src/rpc_processor/coders/public_key.rs | 2 +- veilid-core/src/rpc_processor/mod.rs | 150 +++++++++--------- veilid-core/src/rpc_processor/rpc_route.rs | 104 +++++++----- veilid-core/src/tests/common/mod.rs | 2 +- veilid-core/src/tests/native/mod.rs | 2 +- veilid-core/src/veilid_api/mod.rs | 4 +- veilid-core/src/veilid_config.rs | 2 +- 35 files changed, 334 insertions(+), 220 deletions(-) rename veilid-core/src/{dht => crypto}/envelope.rs (96%) rename veilid-core/src/{dht => crypto}/key.rs (100%) rename veilid-core/src/{dht/crypto.rs => crypto/mod.rs} (97%) rename veilid-core/src/{dht => crypto}/receipt.rs (97%) rename veilid-core/src/{dht => crypto}/tests/mod.rs (100%) rename veilid-core/src/{dht => crypto}/tests/test_crypto.rs (100%) rename veilid-core/src/{dht => crypto}/tests/test_dht_key.rs (100%) rename veilid-core/src/{dht => crypto}/tests/test_envelope_receipt.rs (100%) rename veilid-core/src/{dht => crypto}/value.rs (100%) delete mode 100644 veilid-core/src/dht/mod.rs diff --git a/doc/config/sample.config b/doc/config/sample.config index f4693374..88665c76 100644 --- a/doc/config/sample.config +++ b/doc/config/sample.config @@ -63,7 +63,7 @@ core: max_timestamp_behind_ms: 10000 max_timestamp_ahead_ms: 10000 timeout_ms: 10000 - max_route_hop_count: 5 + max_route_hop_count: 4 default_route_hop_count: 2 dht: diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 0d1ec7e3..97904483 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -258,9 +258,10 @@ struct PeerInfo { } struct RoutedOperation { - signatures @0 :List(Signature); # signatures from nodes that have handled the private route - nonce @1 :Nonce; # nonce Xmsg - data @2 :Data; # Operation encrypted with ENC(Xmsg,DH(PKapr,SKbsr)) + version @0 :UInt8; # crypto version in use for the data + signatures @1 :List(Signature); # signatures from nodes that have handled the private route + nonce @2 :Nonce; # nonce Xmsg + data @3 :Data; # operation encrypted with ENC(Xmsg,DH(PKapr,SKbsr)) } struct OperationStatusQ { diff --git a/veilid-core/src/attachment_manager.rs b/veilid-core/src/attachment_manager.rs index 3d283140..0d834561 100644 --- a/veilid-core/src/attachment_manager.rs +++ b/veilid-core/src/attachment_manager.rs @@ -1,5 +1,5 @@ use crate::callback_state_machine::*; -use crate::dht::Crypto; +use crate::crypto::Crypto; use crate::network_manager::*; use crate::routing_table::*; use crate::xx::*; diff --git a/veilid-core/src/core_context.rs b/veilid-core/src/core_context.rs index 2640f455..698a6a84 100644 --- a/veilid-core/src/core_context.rs +++ b/veilid-core/src/core_context.rs @@ -1,6 +1,6 @@ use crate::api_tracing_layer::*; use crate::attachment_manager::*; -use crate::dht::Crypto; +use crate::crypto::Crypto; use crate::veilid_api::*; use crate::veilid_config::*; use crate::xx::*; @@ -103,7 +103,13 @@ impl ServicesContext { // Set up attachment manager trace!("init attachment manager"); let update_callback = self.update_callback.clone(); - let attachment_manager = AttachmentManager::new(self.config.clone(), protected_store, table_store, block_store, crypto); + let attachment_manager = AttachmentManager::new( + self.config.clone(), + protected_store, + table_store, + block_store, + crypto, + ); if let Err(e) = attachment_manager.init(update_callback).await { self.shutdown().await; return Err(e); diff --git a/veilid-core/src/dht/envelope.rs b/veilid-core/src/crypto/envelope.rs similarity index 96% rename from veilid-core/src/dht/envelope.rs rename to veilid-core/src/crypto/envelope.rs index d94fbf06..3a3c75e4 100644 --- a/veilid-core/src/dht/envelope.rs +++ b/veilid-core/src/crypto/envelope.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] #![allow(clippy::absurd_extreme_comparisons)] -use super::crypto::*; -use super::key::*; +use super::*; use crate::xx::*; use crate::*; use core::convert::TryInto; @@ -38,8 +37,6 @@ use core::convert::TryInto; pub const MAX_ENVELOPE_SIZE: usize = 65507; pub const MIN_ENVELOPE_SIZE: usize = 0x6A + 0x40; // Header + Signature pub const ENVELOPE_MAGIC: &[u8; 4] = b"VLID"; -pub const MIN_VERSION: u8 = 0u8; -pub const MAX_VERSION: u8 = 0u8; pub type EnvelopeNonce = [u8; 24]; #[derive(Debug, Clone, PartialEq, Eq, Default)] @@ -64,12 +61,12 @@ impl Envelope { assert!(sender_id.valid); assert!(recipient_id.valid); - assert!(version >= MIN_VERSION); - assert!(version <= MAX_VERSION); + assert!(version >= MIN_CRYPTO_VERSION); + assert!(version <= MAX_CRYPTO_VERSION); Self { version, - min_version: MIN_VERSION, - max_version: MAX_VERSION, + min_version: MIN_CRYPTO_VERSION, + max_version: MAX_CRYPTO_VERSION, timestamp, nonce, sender_id, @@ -94,9 +91,9 @@ impl Envelope { // Check version let version = data[0x04]; - if version > MAX_VERSION || version < MIN_VERSION { + if version > MAX_CRYPTO_VERSION || version < MIN_CRYPTO_VERSION { return Err(VeilidAPIError::parse_error( - "unsupported protocol version", + "unsupported cryptography version", version, )); } diff --git a/veilid-core/src/dht/key.rs b/veilid-core/src/crypto/key.rs similarity index 100% rename from veilid-core/src/dht/key.rs rename to veilid-core/src/crypto/key.rs diff --git a/veilid-core/src/dht/crypto.rs b/veilid-core/src/crypto/mod.rs similarity index 97% rename from veilid-core/src/dht/crypto.rs rename to veilid-core/src/crypto/mod.rs index 7680a47b..c7836445 100644 --- a/veilid-core/src/dht/crypto.rs +++ b/veilid-core/src/crypto/mod.rs @@ -1,4 +1,18 @@ -use super::key::*; +mod envelope; +mod key; +mod receipt; +mod value; + +pub mod tests; + +pub use envelope::*; +pub use key::*; +pub use receipt::*; +pub use value::*; + +pub const MIN_CRYPTO_VERSION: u8 = 0u8; +pub const MAX_CRYPTO_VERSION: u8 = 0u8; + use crate::xx::*; use crate::*; use chacha20::cipher::{KeyIvInit, StreamCipher}; diff --git a/veilid-core/src/dht/receipt.rs b/veilid-core/src/crypto/receipt.rs similarity index 97% rename from veilid-core/src/dht/receipt.rs rename to veilid-core/src/crypto/receipt.rs index 74d806cb..b4990b4e 100644 --- a/veilid-core/src/dht/receipt.rs +++ b/veilid-core/src/crypto/receipt.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] #![allow(clippy::absurd_extreme_comparisons)] -use super::envelope::{MAX_VERSION, MIN_VERSION}; -use super::key::*; +use super::*; use crate::xx::*; use crate::*; use core::convert::TryInto; @@ -90,9 +89,9 @@ impl Receipt { // Check version let version = data[0x04]; - if version > MAX_VERSION || version < MIN_VERSION { + if version > MAX_CRYPTO_VERSION || version < MIN_CRYPTO_VERSION { return Err(VeilidAPIError::parse_error( - "unsupported protocol version", + "unsupported cryptography version", version, )); } diff --git a/veilid-core/src/dht/tests/mod.rs b/veilid-core/src/crypto/tests/mod.rs similarity index 100% rename from veilid-core/src/dht/tests/mod.rs rename to veilid-core/src/crypto/tests/mod.rs diff --git a/veilid-core/src/dht/tests/test_crypto.rs b/veilid-core/src/crypto/tests/test_crypto.rs similarity index 100% rename from veilid-core/src/dht/tests/test_crypto.rs rename to veilid-core/src/crypto/tests/test_crypto.rs diff --git a/veilid-core/src/dht/tests/test_dht_key.rs b/veilid-core/src/crypto/tests/test_dht_key.rs similarity index 100% rename from veilid-core/src/dht/tests/test_dht_key.rs rename to veilid-core/src/crypto/tests/test_dht_key.rs diff --git a/veilid-core/src/dht/tests/test_envelope_receipt.rs b/veilid-core/src/crypto/tests/test_envelope_receipt.rs similarity index 100% rename from veilid-core/src/dht/tests/test_envelope_receipt.rs rename to veilid-core/src/crypto/tests/test_envelope_receipt.rs diff --git a/veilid-core/src/dht/value.rs b/veilid-core/src/crypto/value.rs similarity index 100% rename from veilid-core/src/dht/value.rs rename to veilid-core/src/crypto/value.rs diff --git a/veilid-core/src/dht/mod.rs b/veilid-core/src/dht/mod.rs deleted file mode 100644 index 60edad22..00000000 --- a/veilid-core/src/dht/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod crypto; -mod envelope; -mod key; -mod receipt; -mod value; - -pub mod tests; - -pub use crypto::*; -pub use envelope::*; -pub use key::*; -pub use receipt::*; -pub use value::*; diff --git a/veilid-core/src/lib.rs b/veilid-core/src/lib.rs index 222e47b6..1acea791 100644 --- a/veilid-core/src/lib.rs +++ b/veilid-core/src/lib.rs @@ -22,7 +22,7 @@ mod api_tracing_layer; mod attachment_manager; mod callback_state_machine; mod core_context; -mod dht; +mod crypto; mod intf; mod network_manager; mod receipt_manager; diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 75969de0..a139334b 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -22,7 +22,7 @@ pub use network_connection::*; //////////////////////////////////////////////////////////////////////////////////////// use connection_handle::*; use connection_limits::*; -use dht::*; +use crypto::*; use futures_util::stream::{FuturesOrdered, FuturesUnordered, StreamExt}; use hashlink::LruCache; use intf::*; @@ -783,11 +783,7 @@ impl NetworkManager { // Process a received signal #[instrument(level = "trace", skip(self), err)] - pub async fn handle_signal( - &self, - _sender_id: DHTKey, - signal_info: SignalInfo, - ) -> EyreResult> { + pub async fn handle_signal(&self, signal_info: SignalInfo) -> EyreResult> { match signal_info { SignalInfo::ReverseConnect { receipt, peer_info } => { let routing_table = self.routing_table(); @@ -923,7 +919,7 @@ impl NetworkManager { // and if so, get the max version we can use let version = if let Some((node_min, node_max)) = node_ref.min_max_version() { #[allow(clippy::absurd_extreme_comparisons)] - if node_min > MAX_VERSION || node_max < MIN_VERSION { + if node_min > MAX_CRYPTO_VERSION || node_max < MIN_CRYPTO_VERSION { bail!( "can't talk to this node {} because version is unsupported: ({},{})", via_node_id, @@ -931,9 +927,9 @@ impl NetworkManager { node_max ); } - cmp::min(node_max, MAX_VERSION) + cmp::min(node_max, MAX_CRYPTO_VERSION) } else { - MAX_VERSION + MAX_CRYPTO_VERSION }; // Build the envelope to send diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index c34b9cb7..5de6568c 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -1,6 +1,6 @@ use super::*; -use crate::dht::*; +use crate::crypto::*; use crate::xx::*; use futures_util::FutureExt; use stop_token::future::FutureExt as StopFutureExt; @@ -265,8 +265,8 @@ impl NetworkManager { bsmap .entry(node_id) .or_insert_with(|| BootstrapRecord { - min_version: MIN_VERSION, - max_version: MAX_VERSION, + min_version: MIN_CRYPTO_VERSION, + max_version: MAX_CRYPTO_VERSION, dial_info_details: Vec::new(), }) .dial_info_details @@ -299,8 +299,8 @@ impl NetworkManager { network_class: NetworkClass::InboundCapable, // Bootstraps are always inbound capable outbound_protocols: ProtocolTypeSet::only(ProtocolType::UDP), // Bootstraps do not participate in relaying and will not make outbound requests, but will have UDP enabled address_types: AddressTypeSet::all(), // Bootstraps are always IPV4 and IPV6 capable - min_version: v.min_version, // Minimum protocol version specified in txt record - max_version: v.max_version, // Maximum protocol version specified in txt record + min_version: v.min_version, // Minimum crypto version specified in txt record + max_version: v.max_version, // Maximum crypto version specified in txt record dial_info_detail_list: v.dial_info_details, // Dial info is as specified in the bootstrap list relay_peer_info: None, // Bootstraps never require a relay themselves }), diff --git a/veilid-core/src/receipt_manager.rs b/veilid-core/src/receipt_manager.rs index 02bf8a75..06101a3c 100644 --- a/veilid-core/src/receipt_manager.rs +++ b/veilid-core/src/receipt_manager.rs @@ -1,6 +1,6 @@ use crate::*; use core::fmt; -use dht::*; +use crypto::*; use futures_util::stream::{FuturesUnordered, StreamExt}; use network_manager::*; use routing_table::*; diff --git a/veilid-core/src/routing_table/debug.rs b/veilid-core/src/routing_table/debug.rs index 3fa6fbc7..43af8b78 100644 --- a/veilid-core/src/routing_table/debug.rs +++ b/veilid-core/src/routing_table/debug.rs @@ -58,8 +58,8 @@ impl RoutingTable { out += &format!( "{},{},{},{},{}", BOOTSTRAP_TXT_VERSION, - MIN_VERSION, - MAX_VERSION, + MIN_CRYPTO_VERSION, + MAX_CRYPTO_VERSION, self.node_id().encode(), some_hostname.unwrap() ); diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 3c1ed52c..d11880c5 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -9,7 +9,7 @@ mod routing_table_inner; mod stats_accounting; mod tasks; -use crate::dht::*; +use crate::crypto::*; use crate::network_manager::*; use crate::rpc_processor::*; use crate::xx::*; diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 1e4385d8..c555c172 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -1,5 +1,5 @@ use super::*; -use crate::dht::*; +use crate::crypto::*; use alloc::fmt; // Connectionless protocols like UDP are dependent on a NAT translation timeout diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 369c3be6..82a1c6a3 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -49,9 +49,9 @@ struct RouteSpecDetail { /// Directions this route is guaranteed to work in directions: DirectionSet, /// Stability preference (prefer reliable nodes over faster) - stability: Stability, + pub stability: Stability, /// Sequencing preference (connection oriented protocols vs datagram) - sequencing: Sequencing, + pub sequencing: Sequencing, } /// The core representation of the RouteSpecStore that can be serialized @@ -616,11 +616,11 @@ impl RouteSpecStore { routing_table: RoutingTable, safety_selection: SafetySelection, private_route: PrivateRoute, - ) -> Result, RPCError> { + ) -> EyreResult> { let pr_hopcount = private_route.hop_count as usize; let max_route_hop_count = self.max_route_hop_count; if pr_hopcount > max_route_hop_count { - return Err(RPCError::internal("private route hop count too long")); + bail!("private route hop count too long"); } // See if we are using a safety route, if not, short circuit this operation @@ -628,7 +628,7 @@ impl RouteSpecStore { SafetySelection::Unsafe(sequencing) => { // Safety route stub with the node's public key as the safety route key since it's the 0th hop if private_route.first_hop.is_none() { - return Err(RPCError::internal("can't compile zero length route")); + bail!("can't compile zero length route"); } let first_hop = private_route.first_hop.as_ref().unwrap(); let opt_first_hop = match &first_hop.node { @@ -707,10 +707,10 @@ impl RouteSpecStore { // Ensure the total hop count isn't too long for our config let sr_hopcount = safety_spec.hop_count; if sr_hopcount == 0 { - return Err(RPCError::internal("safety route hop count is zero")); + bail!("safety route hop count is zero"); } if sr_hopcount > max_route_hop_count { - return Err(RPCError::internal("safety route hop count too long")); + bail!("safety route hop count too long"); } // See if we can optimize this compilation yet @@ -746,10 +746,10 @@ impl RouteSpecStore { // Encrypt the previous blob ENC(nonce, DH(PKhop,SKsr)) let dh_secret = crypto .cached_dh(&safety_rsd.hops[h], &safety_rsd.secret_key) - .map_err(RPCError::map_internal("dh failed"))?; + .wrap_err("dh failed")?; let enc_msg_data = Crypto::encrypt_aead(blob_data.as_slice(), &nonce, &dh_secret, None) - .map_err(RPCError::map_internal("encryption failed"))?; + .wrap_err("encryption failed")?; // Make route hop data let route_hop_data = RouteHopData { @@ -759,26 +759,23 @@ impl RouteSpecStore { // Make route hop let route_hop = RouteHop { - node: match optimize { + node: if optimize { // Optimized, no peer info, just the dht key - true => RouteNode::NodeId(NodeId::new(safety_rsd.hops[h])), + RouteNode::NodeId(NodeId::new(safety_rsd.hops[h])) + } else { // Full peer info, required until we are sure the route has been fully established - false => { - let node_id = safety_rsd.hops[h]; - let pi = rti - .with_node_entry(node_id, |entry| { - entry.with(rti, |_rti, e| { - e.make_peer_info(node_id, RoutingDomain::PublicInternet) - }) + let node_id = safety_rsd.hops[h]; + let pi = rti + .with_node_entry(node_id, |entry| { + entry.with(rti, |_rti, e| { + e.make_peer_info(node_id, RoutingDomain::PublicInternet) }) - .flatten(); - if pi.is_none() { - return Err(RPCError::internal( - "peer info should exist for route but doesn't", - )); - } - RouteNode::PeerInfo(pi.unwrap()) + }) + .flatten(); + if pi.is_none() { + bail!("peer info should exist for route but doesn't"); } + RouteNode::PeerInfo(pi.unwrap()) }, next_hop: Some(route_hop_data), }; @@ -838,6 +835,86 @@ impl RouteSpecStore { Ok(Some(compiled_route)) } + /// Assemble private route for publication + pub fn assemble_private_route( + &mut self, + rti: &RoutingTableInner, + routing_table: RoutingTable, + key: &DHTKey, + ) -> EyreResult { + let rsd = self + .detail(&key) + .ok_or_else(|| eyre!("route does not exist"))?; + + // See if we can optimize this compilation yet + // We don't want to include full nodeinfo if we don't have to + let optimize = rsd.reachable; + + // Make innermost route hop to our own node + let mut route_hop = RouteHop { + node: if optimize { + RouteNode::NodeId(NodeId::new(routing_table.node_id())) + } else { + RouteNode::PeerInfo(rti.get_own_peer_info(RoutingDomain::PublicInternet)) + }, + next_hop: None, + }; + + let crypto = routing_table.network_manager().crypto(); + // Loop for each hop + let hop_count = rsd.hops.len(); + for h in (0..hop_count).rev() { + let nonce = Crypto::get_random_nonce(); + + let blob_data = { + let mut rh_message = ::capnp::message::Builder::new_default(); + let mut rh_builder = rh_message.init_root::(); + encode_route_hop(&route_hop, &mut rh_builder)?; + builder_to_vec(rh_message)? + }; + + // Encrypt the previous blob ENC(nonce, DH(PKhop,SKpr)) + let dh_secret = crypto + .cached_dh(&rsd.hops[h], &rsd.secret_key) + .wrap_err("dh failed")?; + let enc_msg_data = Crypto::encrypt_aead(blob_data.as_slice(), &nonce, &dh_secret, None) + .wrap_err("encryption failed")?; + let route_hop_data = RouteHopData { + nonce, + blob: enc_msg_data, + }; + + route_hop = RouteHop { + node: if optimize { + // Optimized, no peer info, just the dht key + RouteNode::NodeId(NodeId::new(rsd.hops[h])) + } else { + // Full peer info, required until we are sure the route has been fully established + let node_id = rsd.hops[h]; + let pi = rti + .with_node_entry(node_id, |entry| { + entry.with(rti, |_rti, e| { + e.make_peer_info(node_id, RoutingDomain::PublicInternet) + }) + }) + .flatten(); + if pi.is_none() { + bail!("peer info should exist for route but doesn't",); + } + RouteNode::PeerInfo(pi.unwrap()) + }, + next_hop: Some(route_hop_data), + } + } + + let private_route = PrivateRoute { + public_key: key.clone(), + hop_count: hop_count.try_into().unwrap(), + first_hop: Some(route_hop), + }; + Ok(private_route) + } + /// Mark route as published /// When first deserialized, routes must be re-published in order to ensure they remain /// in the RouteSpecStore. diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 7a7cc0e8..b63462a8 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -126,8 +126,8 @@ impl RoutingDomainDetailCommon { network_class: self.network_class.unwrap_or(NetworkClass::Invalid), outbound_protocols: self.outbound_protocols, address_types: self.address_types, - min_version: MIN_VERSION, - max_version: MAX_VERSION, + min_version: MIN_CRYPTO_VERSION, + max_version: MAX_CRYPTO_VERSION, dial_info_detail_list: self.dial_info_details.clone(), relay_peer_info: self .relay_node diff --git a/veilid-core/src/rpc_processor/coders/block_id.rs b/veilid-core/src/rpc_processor/coders/block_id.rs index d071197f..1d23f19b 100644 --- a/veilid-core/src/rpc_processor/coders/block_id.rs +++ b/veilid-core/src/rpc_processor/coders/block_id.rs @@ -1,4 +1,4 @@ -use crate::dht::*; +use crate::crypto::*; use crate::*; use core::convert::TryInto; use rpc_processor::*; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation.rs b/veilid-core/src/rpc_processor/coders/operations/operation.rs index fc61cf32..9d36c192 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation.rs @@ -19,7 +19,7 @@ impl RPCOperationKind { pub fn decode( kind_reader: &veilid_capnp::operation::kind::Reader, - sender_node_id: &DHTKey, + opt_sender_node_id: Option<&DHTKey>, ) -> Result { let which_reader = kind_reader.which().map_err(RPCError::protocol)?; let out = match which_reader { @@ -30,7 +30,7 @@ impl RPCOperationKind { } veilid_capnp::operation::kind::Which::Statement(r) => { let q_reader = r.map_err(RPCError::protocol)?; - let out = RPCStatement::decode(&q_reader, sender_node_id)?; + let out = RPCStatement::decode(&q_reader, opt_sender_node_id)?; RPCOperationKind::Statement(out) } veilid_capnp::operation::kind::Which::Answer(r) => { @@ -111,22 +111,26 @@ impl RPCOperation { pub fn decode( operation_reader: &veilid_capnp::operation::Reader, - sender_node_id: &DHTKey, + opt_sender_node_id: Option<&DHTKey>, ) -> Result { let op_id = operation_reader.get_op_id(); let sender_node_info = if operation_reader.has_sender_node_info() { - let sni_reader = operation_reader - .get_sender_node_info() - .map_err(RPCError::protocol)?; - let sni = decode_signed_node_info(&sni_reader, sender_node_id, true)?; - Some(sni) + if let Some(sender_node_id) = opt_sender_node_id { + let sni_reader = operation_reader + .get_sender_node_info() + .map_err(RPCError::protocol)?; + let sni = decode_signed_node_info(&sni_reader, sender_node_id, true)?; + Some(sni) + } else { + None + } } else { None }; let kind_reader = operation_reader.get_kind(); - let kind = RPCOperationKind::decode(&kind_reader, sender_node_id)?; + let kind = RPCOperationKind::decode(&kind_reader, opt_sender_node_id)?; Ok(RPCOperation { op_id, diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_node_info_update.rs b/veilid-core/src/rpc_processor/coders/operations/operation_node_info_update.rs index 00c5943a..a785e6b5 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_node_info_update.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_node_info_update.rs @@ -9,8 +9,14 @@ pub struct RPCOperationNodeInfoUpdate { impl RPCOperationNodeInfoUpdate { pub fn decode( reader: &veilid_capnp::operation_node_info_update::Reader, - sender_node_id: &DHTKey, + opt_sender_node_id: Option<&DHTKey>, ) -> Result { + if opt_sender_node_id.is_none() { + return Err(RPCError::protocol( + "can't decode node info update without sender node id", + )); + } + let sender_node_id = opt_sender_node_id.unwrap(); let sni_reader = reader.get_signed_node_info().map_err(RPCError::protocol)?; let signed_node_info = decode_signed_node_info(&sni_reader, sender_node_id, true)?; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_route.rs b/veilid-core/src/rpc_processor/coders/operations/operation_route.rs index 7ded216d..68c97191 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_route.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_route.rs @@ -3,14 +3,16 @@ use rpc_processor::*; #[derive(Debug, Clone)] pub struct RoutedOperation { + pub version: u8, pub signatures: Vec, pub nonce: Nonce, pub data: Vec, } impl RoutedOperation { - pub fn new(nonce: Nonce, data: Vec) -> Self { + pub fn new(version: u8, nonce: Nonce, data: Vec) -> Self { Self { + version, signatures: Vec::new(), nonce, data, @@ -32,11 +34,13 @@ impl RoutedOperation { signatures.push(sig); } + let version = reader.get_version(); let n_reader = reader.get_nonce().map_err(RPCError::protocol)?; let nonce = decode_nonce(&n_reader); let data = reader.get_data().map_err(RPCError::protocol)?.to_vec(); Ok(RoutedOperation { + version, signatures, nonce, data, @@ -47,6 +51,7 @@ impl RoutedOperation { &self, builder: &mut veilid_capnp::routed_operation::Builder, ) -> Result<(), RPCError> { + builder.reborrow().set_version(self.version); let mut sigs_builder = builder.reborrow().init_signatures( self.signatures .len() diff --git a/veilid-core/src/rpc_processor/coders/operations/statement.rs b/veilid-core/src/rpc_processor/coders/operations/statement.rs index bbac6455..96c71a4c 100644 --- a/veilid-core/src/rpc_processor/coders/operations/statement.rs +++ b/veilid-core/src/rpc_processor/coders/operations/statement.rs @@ -22,10 +22,10 @@ impl RPCStatement { } pub fn decode( reader: &veilid_capnp::statement::Reader, - sender_node_id: &DHTKey, + opt_sender_node_id: Option<&DHTKey>, ) -> Result { let d_reader = reader.get_detail(); - let detail = RPCStatementDetail::decode(&d_reader, sender_node_id)?; + let detail = RPCStatementDetail::decode(&d_reader, opt_sender_node_id)?; Ok(RPCStatement { detail }) } pub fn encode(&self, builder: &mut veilid_capnp::statement::Builder) -> Result<(), RPCError> { @@ -59,7 +59,7 @@ impl RPCStatementDetail { } pub fn decode( reader: &veilid_capnp::statement::detail::Reader, - sender_node_id: &DHTKey, + opt_sender_node_id: Option<&DHTKey>, ) -> Result { let which_reader = reader.which().map_err(RPCError::protocol)?; let out = match which_reader { @@ -75,7 +75,7 @@ impl RPCStatementDetail { } veilid_capnp::statement::detail::NodeInfoUpdate(r) => { let op_reader = r.map_err(RPCError::protocol)?; - let out = RPCOperationNodeInfoUpdate::decode(&op_reader, sender_node_id)?; + let out = RPCOperationNodeInfoUpdate::decode(&op_reader, opt_sender_node_id)?; RPCStatementDetail::NodeInfoUpdate(out) } veilid_capnp::statement::detail::ValueChanged(r) => { diff --git a/veilid-core/src/rpc_processor/coders/public_key.rs b/veilid-core/src/rpc_processor/coders/public_key.rs index d80d9401..f7eb530a 100644 --- a/veilid-core/src/rpc_processor/coders/public_key.rs +++ b/veilid-core/src/rpc_processor/coders/public_key.rs @@ -1,4 +1,4 @@ -use crate::dht::*; +use crate::crypto::*; use crate::*; use core::convert::TryInto; use rpc_processor::*; diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index c426e822..e1742359 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -27,7 +27,7 @@ pub use operation_waiter::*; pub use rpc_error::*; use super::*; -use crate::dht::*; +use crate::crypto::*; use crate::xx::*; use capnp::message::ReaderSegments; use futures_util::StreamExt; @@ -54,7 +54,6 @@ struct RPCMessageHeaderDetailDirect { #[derive(Debug, Clone)] struct RPCMessageHeaderDetailPrivateRoute { - /// The private route we received the rpc over private_route: DHTKey, // The safety selection for replying to this private routed rpc @@ -70,12 +69,6 @@ enum RPCMessageHeaderDetail { /// The decoded header of an RPC message #[derive(Debug, Clone)] struct RPCMessageHeader { - - version: u8, - min_version: u8, - max_version: u8, - timestamp: u64,???? do we need a header for private routed messages? write process_rpc_message - /// Time the message was received, not sent timestamp: u64, /// The length in bytes of the rpc message body @@ -84,6 +77,8 @@ struct RPCMessageHeader { detail: RPCMessageHeaderDetail, } +impl RPCMessageHeader {} + #[derive(Debug)] struct RPCMessageData { contents: Vec, // rpc messages must be a canonicalized single segment @@ -437,6 +432,7 @@ impl RPCProcessor { match self.routing_table().with_route_spec_store_mut(|rss, rti| { // Compile the safety route with the private route rss.compile_safety_route(rti, routing_table, safety_selection, private_route) + .map_err(RPCError::internal) })? { Some(cr) => cr, None => { @@ -448,6 +444,7 @@ impl RPCProcessor { // Encrypt routed operation // Xmsg + ENC(Xmsg, DH(PKapr, SKbsr)) + // xxx use factory method, get version from somewhere... let nonce = Crypto::get_random_nonce(); let dh_secret = self .crypto @@ -457,7 +454,8 @@ impl RPCProcessor { .map_err(RPCError::map_internal("encryption failed"))?; // Make the routed operation - let operation = RoutedOperation::new(nonce, enc_msg_data); + // xxx: replace MAX_CRYPTO_VERSION with the version from the factory + let operation = RoutedOperation::new(MAX_CRYPTO_VERSION, nonce, enc_msg_data); // Prepare route operation let sr_hop_count = compiled_route.safety_route.hop_count; @@ -864,59 +862,79 @@ impl RPCProcessor { ////////////////////////////////////////////////////////////////////// #[instrument(level = "trace", skip(self, encoded_msg), err)] - async fn process_rpc_message_version_0( - &self, - encoded_msg: RPCMessageEncoded, - ) -> Result<(), RPCError> { - // Get the routing domain this message came over - let routing_domain = encoded_msg.header.routing_domain; + async fn process_rpc_message(&self, encoded_msg: RPCMessageEncoded) -> Result<(), RPCError> { + // Decode operation appropriately based on header detail + let msg = match &encoded_msg.header.detail { + RPCMessageHeaderDetail::Direct(detail) => { + // Get the routing domain this message came over + let routing_domain = detail.routing_domain; - // Decode the operation - let sender_node_id = encoded_msg.header.envelope.get_sender_id(); + // Decode the operation + let sender_node_id = detail.envelope.get_sender_id(); - // Decode the RPC message - let operation = { - let reader = capnp::message::Reader::new(encoded_msg.data, Default::default()); - let op_reader = reader - .get_root::() - .map_err(RPCError::protocol) - .map_err(logthru_rpc!())?; - RPCOperation::decode(&op_reader, &sender_node_id)? - }; + // Decode the RPC message + let operation = { + let reader = capnp::message::Reader::new(encoded_msg.data, Default::default()); + let op_reader = reader + .get_root::() + .map_err(RPCError::protocol) + .map_err(logthru_rpc!())?; + RPCOperation::decode(&op_reader, Some(&sender_node_id))? + }; - // Get the sender noderef, incorporating and 'sender node info' - let mut opt_sender_nr: Option = None; - if let Some(sender_node_info) = operation.sender_node_info() { - // Sender NodeInfo was specified, update our routing table with it - if !self.filter_node_info(routing_domain, &sender_node_info.node_info) { - return Err(RPCError::invalid_format( - "sender signednodeinfo has invalid peer scope", - )); + // Get the sender noderef, incorporating and 'sender node info' + let mut opt_sender_nr: Option = None; + if let Some(sender_node_info) = operation.sender_node_info() { + // Sender NodeInfo was specified, update our routing table with it + if !self.filter_node_info(routing_domain, &sender_node_info.node_info) { + return Err(RPCError::invalid_format( + "sender signednodeinfo has invalid peer scope", + )); + } + opt_sender_nr = self.routing_table().register_node_with_signed_node_info( + routing_domain, + sender_node_id, + sender_node_info.clone(), + false, + ); + } + + // look up sender node, in case it's different than our peer due to relaying + if opt_sender_nr.is_none() { + opt_sender_nr = self.routing_table().lookup_node_ref(sender_node_id) + } + + // Mark this sender as having seen our node info over this routing domain + // because it managed to reach us over that routing domain + if let Some(sender_nr) = &opt_sender_nr { + sender_nr.set_seen_our_node_info(routing_domain); + } + + // Make the RPC message + RPCMessage { + header: encoded_msg.header, + operation, + opt_sender_nr, + } } - opt_sender_nr = self.routing_table().register_node_with_signed_node_info( - routing_domain, - sender_node_id, - sender_node_info.clone(), - false, - ); - } + RPCMessageHeaderDetail::PrivateRoute(detail) => { + // Decode the RPC message + let operation = { + let reader = capnp::message::Reader::new(encoded_msg.data, Default::default()); + let op_reader = reader + .get_root::() + .map_err(RPCError::protocol) + .map_err(logthru_rpc!())?; + RPCOperation::decode(&op_reader, None)? + }; - // look up sender node, in case it's different than our peer due to relaying - if opt_sender_nr.is_none() { - opt_sender_nr = self.routing_table().lookup_node_ref(sender_node_id) - } - - // Mark this sender as having seen our node info over this routing domain - // because it managed to reach us over that routing domain - if let Some(sender_nr) = &opt_sender_nr { - sender_nr.set_seen_our_node_info(routing_domain); - } - - // Make the RPC message - let msg = RPCMessage { - header: encoded_msg.header, - operation, - opt_sender_nr, + // Make the RPC message + RPCMessage { + header: encoded_msg.header, + operation, + opt_sender_nr: None, + } + } }; // Process stats @@ -940,7 +958,7 @@ impl RPCProcessor { }; // Log rpc receive - debug!(target: "rpc_message", dir = "recv", kind, op_id = msg.operation.op_id(), desc = msg.operation.kind().desc(), sender_id = ?sender_node_id); + debug!(target: "rpc_message", dir = "recv", kind, op_id = msg.operation.op_id(), desc = msg.operation.kind().desc(), header = ?msg.header); // Process specific message kind match msg.operation.kind() { @@ -977,18 +995,6 @@ impl RPCProcessor { } } - #[instrument(level = "trace", skip(self, msg), err)] - async fn process_rpc_message(&self, msg: RPCMessageEncoded) -> Result<(), RPCError> { - if msg.header.envelope.get_version() == 0 { - self.process_rpc_message_version_0(msg).await - } else { - Err(RPCError::Internal(format!( - "unsupported envelope version: {}, newest supported is version 0", - msg.header.envelope.get_version() - ))) - } - } - async fn rpc_worker( self, stop_token: StopToken, @@ -1044,10 +1050,6 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, body), err)] pub fn enqueue_private_route_message( &self, - - xx need rpc version somewhere! the rpc version to decode should be packaged in with the body, and the allocator should ensure the version is compatible at the end node. can append to body, and then pop the last u8. - xx try to write the whole process_rpc_message pipeline first - private_route: DHTKey, safety_selection: SafetySelection, body: Vec, diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 001bf2bf..91d5aebe 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -72,8 +72,8 @@ impl RPCProcessor { #[instrument(level = "trace", skip_all, err)] async fn process_route_private_route_hop( &self, - route: RPCOperationRoute, - private_route: PrivateRoute, + mut route: RPCOperationRoute, + next_private_route: PrivateRoute, ) -> Result<(), RPCError> { // Make sure hop count makes sense if route.safety_route.hop_count != 0 { @@ -81,19 +81,14 @@ impl RPCProcessor { "Safety hop count should be zero if switched to private route", )); } - if private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { + if next_private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { return Err(RPCError::protocol( "Private route hop count too high to process", )); } - if private_route.hop_count == 0 { - return Err(RPCError::protocol( - "Private route hop count should not be zero if there are more hops", - )); - } // Get private route first hop (this is validated to not be None before calling this function) - let first_hop = private_route.first_hop.as_ref().unwrap(); + let first_hop = next_private_route.first_hop.as_ref().unwrap(); // Get next hop node ref let next_hop_nr = match &first_hop.node { @@ -121,12 +116,21 @@ impl RPCProcessor { } }?; + // Sign the operation if this is not our last hop + // as the last hop is already signed by the envelope + if next_private_route.hop_count != 0 { + let node_id = self.routing_table.node_id(); + let node_id_secret = self.routing_table.node_id_secret(); + let sig = sign(&node_id, &node_id_secret, &route.operation.data).map_err(RPCError::internal)?; + route.operation.signatures.push(sig); + } + // Pass along the route let next_hop_route = RPCOperationRoute { safety_route: SafetyRoute { public_key: route.safety_route.public_key, hop_count: 0, - hops: SafetyRouteHops::Private(private_route), + hops: SafetyRouteHops::Private(next_private_route), }, operation: route.operation, }; @@ -146,12 +150,13 @@ impl RPCProcessor { #[instrument(level = "trace", skip_all, err)] async fn process_routed_operation( &self, - sender_id: DHTKey, - route: RPCOperationRoute, + detail: RPCMessageHeaderDetailDirect, + routed_operation: RoutedOperation, + safety_route: &SafetyRoute, private_route: &PrivateRoute, ) -> Result<(), RPCError> { // Make sure hop count makes sense - if route.safety_route.hop_count != 0 { + if safety_route.hop_count != 0 { return Err(RPCError::protocol( "Safety hop count should be zero if switched to private route", )); @@ -162,21 +167,29 @@ impl RPCProcessor { )); } - let routed_operation = &route.operation; - - // Get sequencing preference - if route. - // If the private route public key is our node id, then this was sent via safety route to our node directly // so there will be no signatures to validate let opt_pr_info = if private_route.public_key == self.routing_table.node_id() { // the private route was a stub to our own node's secret - // return our secret key + // return our secret key and an appropriate safety selection + // Get sequencing preference + let sequencing = if detail + .connection_descriptor + .protocol_type() + .is_connection_oriented() + { + Sequencing::EnsureOrdered + } else { + Sequencing::NoPreference + }; Some(( - self.routing_table.node_id_secret(), // Figure out how we'd reply to this if it were a question + self.routing_table.node_id_secret(), SafetySelection::Unsafe(sequencing), )) } else { + // Get sender id + let sender_id = detail.envelope.get_sender_id(); + // Look up the private route and ensure it's one in our spec store let opt_signatures_valid = self.routing_table.with_route_spec_store(|rss, rti| { rss.with_route_spec_detail(&private_route.public_key, |rsd| { @@ -207,10 +220,15 @@ impl RPCProcessor { } } } - // Correct signatures + // We got the correct signatures, return a key ans Some(( rsd.secret_key, - SafetySelection::Safe(SafetySpec { preferred_route: todo!(), hop_count: todo!(), stability: todo!(), sequencing: todo!() }) + SafetySelection::Safe(SafetySpec { + preferred_route: Some(private_route.public_key), + hop_count: rsd.hops.len(), + stability: rsd.stability, + sequencing: rsd.sequencing, + }) )) }) }); @@ -229,7 +247,7 @@ impl RPCProcessor { // xxx: punish nodes that send messages that fail to decrypt eventually let dh_secret = self .crypto - .cached_dh(&route.safety_route.public_key, &secret_key) + .cached_dh(&safety_route.public_key, &secret_key) .map_err(RPCError::protocol)?; let body = Crypto::decrypt_aead( &routed_operation.data, @@ -250,12 +268,14 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), err)] pub(crate) async fn process_route(&self, msg: RPCMessage) -> Result<(), RPCError> { - // xxx do not process latency for routed messages - // Get header detail, must be direct and not inside a route itself - let (envelope, peer_noderef, connection_descriptor, routing_domain) = match msg.header.detail { - RPCMessageHeaderDetail::Direct { envelope, peer_noderef, connection_descriptor, routing_domain } => (envelope, peer_noderef, connection_descriptor, routing_domain), - RPCMessageHeaderDetail::PrivateRoute { private_route, safety_selection } => { return Err(RPCError::protocol("route operation can not be inside route")) }, + let detail = match msg.header.detail { + RPCMessageHeaderDetail::Direct(detail) => detail, + RPCMessageHeaderDetail::PrivateRoute(_) => { + return Err(RPCError::protocol( + "route operation can not be inside route", + )) + } }; // Get the statement @@ -267,6 +287,14 @@ impl RPCProcessor { _ => panic!("not a statement"), }; + // Process routed operation version + // xxx switch this to a Crypto trait factory method per issue#140 + if route.operation.version != MAX_CRYPTO_VERSION { + return Err(RPCError::protocol( + "routes operation crypto is not valid version", + )); + } + // See what kind of safety route we have going on here match route.safety_route.hops { // There is a safety route hop @@ -312,12 +340,8 @@ impl RPCProcessor { .await?; } else { // Private route is empty, process routed operation - self.process_routed_operation( - envelope.get_sender_id(), - route, - &private_route, - ) - .await?; + self.process_routed_operation(detail, route.operation, &route.safety_route, &private_route) + .await?; } } else if blob_tag == 0 { // RouteHop @@ -373,21 +397,17 @@ impl RPCProcessor { }; // Make next PrivateRoute and pass it on - let private_route = PrivateRoute { + let next_private_route = PrivateRoute { public_key: private_route.public_key, hop_count: private_route.hop_count - 1, first_hop: opt_next_first_hop, }; - self.process_route_private_route_hop(route, private_route) + self.process_route_private_route_hop(route, next_private_route) .await?; } else { // No hops left, time to process the routed operation - self.process_routed_operation( - msg.header.envelope.get_sender_id(), - route, - private_route, - ) - .await?; + self.process_routed_operation(detail, route.operation, &route.safety_route, private_route) + .await?; } } } diff --git a/veilid-core/src/tests/common/mod.rs b/veilid-core/src/tests/common/mod.rs index 537a41a5..ab47cdd5 100644 --- a/veilid-core/src/tests/common/mod.rs +++ b/veilid-core/src/tests/common/mod.rs @@ -7,5 +7,5 @@ pub mod test_veilid_core; use super::*; -pub use dht::tests::*; +pub use crypto::tests::*; pub use network_manager::tests::*; diff --git a/veilid-core/src/tests/native/mod.rs b/veilid-core/src/tests/native/mod.rs index da86b323..d693eaa5 100644 --- a/veilid-core/src/tests/native/mod.rs +++ b/veilid-core/src/tests/native/mod.rs @@ -3,7 +3,7 @@ mod test_async_peek_stream; -use crate::dht::tests::*; +use crate::crypto::tests::*; use crate::network_manager::tests::*; use crate::tests::common::*; use crate::xx::*; diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 9d16b15b..9f14bd68 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -19,8 +19,8 @@ pub use crate::xx::{ pub use alloc::string::ToString; pub use attachment_manager::AttachmentManager; pub use core::str::FromStr; -pub use dht::Crypto; -pub use dht::{generate_secret, sign, verify, DHTKey, DHTKeySecret, DHTSignature, Nonce}; +pub use crypto::Crypto; +pub use crypto::{generate_secret, sign, verify, DHTKey, DHTKeySecret, DHTSignature, Nonce}; pub use intf::BlockStore; pub use intf::ProtectedStore; pub use intf::TableStore; diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index 9fdb88a8..d1287864 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -708,7 +708,7 @@ impl VeilidConfig { // If we have a node id from storage, check it if node_id.valid && node_id_secret.valid { // Validate node id - if !dht::validate_key(&node_id, &node_id_secret) { + if !crypto::validate_key(&node_id, &node_id_secret) { apibail_generic!("node id secret and node id key don't match"); } } From 68d55a5e776c5802c26a55abffcd028f5963995d Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 30 Oct 2022 23:23:12 -0400 Subject: [PATCH 25/67] more route work --- veilid-core/src/network_manager/mod.rs | 32 ++- veilid-core/src/receipt_manager.rs | 30 ++- veilid-core/src/routing_table/bucket.rs | 10 +- veilid-core/src/routing_table/bucket_entry.rs | 18 ++ veilid-core/src/routing_table/mod.rs | 14 +- .../src/routing_table/route_spec_store.rs | 201 +++++++++++------- .../src/routing_table/routing_domains.rs | 9 +- .../src/routing_table/routing_table_inner.rs | 26 +-- .../coders/private_safety_route.rs | 4 +- veilid-core/src/rpc_processor/destination.rs | 6 +- veilid-core/src/rpc_processor/mod.rs | 29 +-- .../src/rpc_processor/rpc_node_info_update.rs | 10 +- .../src/rpc_processor/rpc_return_receipt.rs | 22 +- veilid-core/src/rpc_processor/rpc_route.rs | 84 ++++---- veilid-core/src/rpc_processor/rpc_signal.rs | 2 +- veilid-core/src/rpc_processor/rpc_status.rs | 11 +- .../rpc_processor/rpc_validate_dial_info.rs | 15 +- 17 files changed, 312 insertions(+), 211 deletions(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index a139334b..5a771d95 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -757,7 +757,9 @@ impl NetworkManager { Ok(v) => v, }; - receipt_manager.handle_receipt(receipt, None).await + receipt_manager + .handle_receipt(receipt, ReceiptReturned::OutOfBand) + .await } /// Process a received in-band receipt @@ -765,7 +767,7 @@ impl NetworkManager { pub async fn handle_in_band_receipt>( &self, receipt_data: R, - inbound_nr: NodeRef, + inbound_noderef: NodeRef, ) -> NetworkResult<()> { let receipt_manager = self.receipt_manager(); @@ -777,7 +779,27 @@ impl NetworkManager { }; receipt_manager - .handle_receipt(receipt, Some(inbound_nr)) + .handle_receipt(receipt, ReceiptReturned::InBand { inbound_noderef }) + .await + } + + /// Process a received private receipt + #[instrument(level = "trace", skip(self, receipt_data), ret)] + pub async fn handle_private_receipt>( + &self, + receipt_data: R, + ) -> NetworkResult<()> { + let receipt_manager = self.receipt_manager(); + + let receipt = match Receipt::from_signed_data(receipt_data.as_ref()) { + Err(e) => { + return NetworkResult::invalid_message(e.to_string()); + } + Ok(v) => v, + }; + + receipt_manager + .handle_receipt(receipt, ReceiptReturned::Private) .await } @@ -1001,7 +1023,7 @@ impl NetworkManager { // Wait for the return receipt let inbound_nr = match eventual_value.await.take_value().unwrap() { - ReceiptEvent::ReturnedOutOfBand => { + ReceiptEvent::ReturnedPrivate | ReceiptEvent::ReturnedOutOfBand => { return Ok(NetworkResult::invalid_message( "reverse connect receipt should be returned in-band", )); @@ -1102,7 +1124,7 @@ impl NetworkManager { // Wait for the return receipt let inbound_nr = match eventual_value.await.take_value().unwrap() { - ReceiptEvent::ReturnedOutOfBand => { + ReceiptEvent::ReturnedPrivate | ReceiptEvent::ReturnedOutOfBand => { return Ok(NetworkResult::invalid_message( "hole punch receipt should be returned in-band", )); diff --git a/veilid-core/src/receipt_manager.rs b/veilid-core/src/receipt_manager.rs index 06101a3c..28ba23d2 100644 --- a/veilid-core/src/receipt_manager.rs +++ b/veilid-core/src/receipt_manager.rs @@ -11,10 +11,18 @@ use xx::*; pub enum ReceiptEvent { ReturnedOutOfBand, ReturnedInBand { inbound_noderef: NodeRef }, + ReturnedPrivate, Expired, Cancelled, } +#[derive(Clone, Debug)] +pub enum ReceiptReturned { + OutOfBand, + InBand { inbound_noderef: NodeRef }, + Private, +} + pub trait ReceiptCallback: Send + 'static { fn call( &self, @@ -394,17 +402,17 @@ impl ReceiptManager { pub async fn handle_receipt( &self, receipt: Receipt, - inbound_noderef: Option, + receipt_returned: ReceiptReturned, ) -> NetworkResult<()> { let receipt_nonce = receipt.get_nonce(); let extra_data = receipt.get_extra_data(); log_rpc!(debug "<<== RECEIPT {} <- {}{}", receipt_nonce.encode(), - if let Some(nr) = &inbound_noderef { - nr.to_string() - } else { - "DIRECT".to_owned() + match receipt_returned { + ReceiptReturned::OutOfBand => "OutOfBand".to_owned(), + ReceiptReturned::InBand { ref inbound_noderef } => format!("InBand({})", inbound_noderef), + ReceiptReturned::Private => "Private".to_owned(), }, if extra_data.is_empty() { "".to_owned() @@ -435,10 +443,14 @@ impl ReceiptManager { record_mut.returns_so_far += 1; // Get the receipt event to return - let receipt_event = if let Some(inbound_noderef) = inbound_noderef { - ReceiptEvent::ReturnedInBand { inbound_noderef } - } else { - ReceiptEvent::ReturnedOutOfBand + let receipt_event = match receipt_returned { + ReceiptReturned::OutOfBand => ReceiptEvent::ReturnedOutOfBand, + ReceiptReturned::InBand { + ref inbound_noderef, + } => ReceiptEvent::ReturnedInBand { + inbound_noderef: inbound_noderef.clone(), + }, + ReceiptReturned::Private => ReceiptEvent::ReturnedPrivate, }; let callback_future = Self::perform_callback(receipt_event, &mut record_mut); diff --git a/veilid-core/src/routing_table/bucket.rs b/veilid-core/src/routing_table/bucket.rs index ba48563c..48199bfc 100644 --- a/veilid-core/src/routing_table/bucket.rs +++ b/veilid-core/src/routing_table/bucket.rs @@ -56,11 +56,7 @@ impl Bucket { self.entries.iter() } - pub(super) fn kick( - &mut self, - inner: &mut RoutingTableInner, - bucket_depth: usize, - ) -> Option> { + pub(super) fn kick(&mut self, bucket_depth: usize) -> Option> { // Get number of entries to attempt to purge from bucket let bucket_len = self.entries.len(); @@ -84,8 +80,8 @@ impl Bucket { if a.0 == b.0 { return core::cmp::Ordering::Equal; } - a.1.with(inner, |rti, ea| { - b.1.with(rti, |_rti, eb| { + a.1.with_inner(|ea| { + b.1.with_inner(|eb| { let astate = state_ordering(ea.state(cur_ts)); let bstate = state_ordering(eb.state(cur_ts)); // first kick dead nodes, then unreliable nodes diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 94428da9..f858fbd7 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -675,6 +675,24 @@ impl BucketEntry { let mut inner = self.inner.write(); f(rti, &mut *inner) } + + // Internal inner access for RoutingTableInner only + pub(super) fn with_inner(&self, f: F) -> R + where + F: FnOnce(&BucketEntryInner) -> R, + { + let inner = self.inner.read(); + f(&*inner) + } + + // Internal inner access for RoutingTableInner only + pub(super) fn with_mut_inner(&self, f: F) -> R + where + F: FnOnce(&mut BucketEntryInner) -> R, + { + let mut inner = self.inner.write(); + f(&mut *inner) + } } impl Drop for BucketEntry { diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index d11880c5..6214d335 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -191,18 +191,8 @@ impl RoutingTable { self.inner.read().routing_domain_for_address(address) } - pub fn with_route_spec_store_mut(&self, f: F) -> R - where - F: FnOnce(&mut RouteSpecStore, &mut RoutingTableInner) -> R, - { - self.inner.write().with_route_spec_store_mut(f) - } - - pub fn with_route_spec_store(&self, f: F) -> R - where - F: FnOnce(&RouteSpecStore, &RoutingTableInner) -> R, - { - self.inner.read().with_route_spec_store(f) + pub fn route_spec_store(&self) -> RouteSpecStore { + self.inner.read().route_spec_store.clone() } pub fn relay_node(&self, domain: RoutingDomain) -> Option { diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 82a1c6a3..f51f0589 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -72,18 +72,27 @@ pub struct RouteSpecStoreCache { hop_cache: HashSet>, } -/// The routing table's storage for private/safety routes #[derive(Debug)] -pub struct RouteSpecStore { - /// Maximum number of hops in a route - max_route_hop_count: usize, - /// Default number of hops in a route - default_route_hop_count: usize, +pub struct RouteSpecStoreInner { /// Serialize RouteSpecStore content content: RouteSpecStoreContent, /// RouteSpecStore cache cache: RouteSpecStoreCache, } +#[derive(Debug)] +pub struct RouteSpecStoreUnlockedInner { + /// Maximum number of hops in a route + max_route_hop_count: usize, + /// Default number of hops in a route + default_route_hop_count: usize, +} + +/// The routing table's storage for private/safety routes +#[derive(Clone, Debug)] +pub struct RouteSpecStore { + inner: Arc>, + unlocked_inner: Arc, +} fn route_hops_to_hop_cache(hops: &[DHTKey]) -> Vec { let mut cache: Vec = Vec::with_capacity(hops.len() * DHT_KEY_LENGTH); @@ -167,17 +176,24 @@ where heaps_permutation(&mut permutation, hop_count - 1, f) } +xxx get routing table handle into routespecstore +xxx first figure out when new/load get called, does routing table need 'init' or can we just pick the right time to load the cache? what about flushing the cache ? we don't 'save' it yet, that should probably get flushed at the same time as the DH cache. + impl RouteSpecStore { pub fn new(config: VeilidConfig) -> Self { let c = config.get(); Self { - max_route_hop_count: c.network.rpc.max_route_hop_count.into(), - default_route_hop_count: c.network.rpc.default_route_hop_count.into(), - content: RouteSpecStoreContent { - details: HashMap::new(), - }, - cache: Default::default(), + unlocked_inner: Arc::new(RouteSpecStoreUnlockedInner { + max_route_hop_count: c.network.rpc.max_route_hop_count.into(), + default_route_hop_count: c.network.rpc.default_route_hop_count.into(), + }), + inner: Arc::new(Mutex::new(RouteSpecStoreInner { + content: RouteSpecStoreContent { + details: HashMap::new(), + }, + cache: Default::default(), + })), } } @@ -187,18 +203,13 @@ impl RouteSpecStore { // Get cbor blob from table store let table_store = routing_table.network_manager().table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; - let content = rsstdb.load_cbor(0, b"content").await?.unwrap_or_default(); - let mut rss = RouteSpecStore { - max_route_hop_count: c.network.rpc.max_route_hop_count.into(), - default_route_hop_count: c.network.rpc.default_route_hop_count.into(), - content, - cache: Default::default(), - }; + let mut content: RouteSpecStoreContent = + rsstdb.load_cbor(0, b"content").await?.unwrap_or_default(); // Load secrets from pstore let pstore = routing_table.network_manager().protected_store(); let mut dead_keys = Vec::new(); - for (k, v) in &mut rss.content.details { + for (k, v) in &mut content.details { if let Some(secret_key) = pstore .load_user_secret(&format!("RouteSpecStore_{}", k.encode())) .await? @@ -217,23 +228,39 @@ impl RouteSpecStore { } for k in dead_keys { log_rtab!(debug "killing off private route: {}", k.encode()); - rss.content.details.remove(&k); + content.details.remove(&k); } + let mut inner = RouteSpecStoreInner { + content, + cache: Default::default(), + }; + // Rebuild the routespecstore cache - rss.rebuild_cache(); + Self::rebuild_cache(&mut inner); + + let rss = RouteSpecStore { + unlocked_inner: Arc::new(RouteSpecStoreUnlockedInner { + max_route_hop_count: c.network.rpc.max_route_hop_count.into(), + default_route_hop_count: c.network.rpc.default_route_hop_count.into(), + }), + inner: Arc::new(Mutex::new(inner)), + }; + Ok(rss) } pub async fn save(&self, routing_table: RoutingTable) -> EyreResult<()> { + let inner = self.inner.lock(); + // Save all the fields we care about to the cbor blob in table storage let table_store = routing_table.network_manager().table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; - rsstdb.store_cbor(0, b"content", &self.content).await?; + rsstdb.store_cbor(0, b"content", &inner.content).await?; // Keep secrets in protected store as well let pstore = routing_table.network_manager().protected_store(); - for (k, v) in &self.content.details { + for (k, v) in &inner.content.details { if pstore .save_user_secret( &format!("RouteSpecStore_{}", k.encode()), @@ -266,18 +293,24 @@ impl RouteSpecStore { .or_insert(1); } - fn rebuild_cache(&mut self) { - for v in self.content.details.values() { + fn rebuild_cache(inner: &mut RouteSpecStoreInner) { + for v in inner.content.details.values() { let cache_key = route_hops_to_hop_cache(&v.hops); - Self::add_to_cache(&mut self.cache, cache_key, &v); + Self::add_to_cache(&mut inner.cache, cache_key, &v); } } - fn detail(&self, public_key: &DHTKey) -> Option<&RouteSpecDetail> { - self.content.details.get(&public_key) + fn detail<'a>( + inner: &'a RouteSpecStoreInner, + public_key: &DHTKey, + ) -> Option<&'a RouteSpecDetail> { + inner.content.details.get(public_key) } - fn detail_mut(&mut self, public_key: &DHTKey) -> Option<&mut RouteSpecDetail> { - self.content.details.get_mut(&public_key) + fn detail_mut<'a>( + inner: &'a mut RouteSpecStoreInner, + public_key: &DHTKey, + ) -> Option<&'a mut RouteSpecDetail> { + inner.content.details.get_mut(public_key) } /// Create a new route @@ -285,7 +318,7 @@ impl RouteSpecStore { /// The route is not yet tested for its reachability /// Returns None if no route could be allocated at this time pub fn allocate_route( - &mut self, + &self, rti: &RoutingTableInner, routing_table: RoutingTable, stability: Stability, @@ -294,12 +327,13 @@ impl RouteSpecStore { directions: DirectionSet, ) -> EyreResult> { use core::cmp::Ordering; + let mut inner = self.inner.lock(); if hop_count < 1 { bail!("Not allocating route less than one hop in length"); } - if hop_count > self.max_route_hop_count { + if hop_count > self.unlocked_inner.max_route_hop_count { bail!("Not allocating route longer than max route hop count"); } @@ -342,13 +376,13 @@ impl RouteSpecStore { v2: &(DHTKey, Option>)| -> Ordering { // deprioritize nodes that we have already used as end points - let e1_used_end = self + let e1_used_end = inner .cache .used_end_nodes .get(&v1.0) .cloned() .unwrap_or_default(); - let e2_used_end = self + let e2_used_end = inner .cache .used_end_nodes .get(&v2.0) @@ -360,13 +394,13 @@ impl RouteSpecStore { } // deprioritize nodes we have used already anywhere - let e1_used = self + let e1_used = inner .cache .used_nodes .get(&v1.0) .cloned() .unwrap_or_default(); - let e2_used = self + let e2_used = inner .cache .used_nodes .get(&v2.0) @@ -428,7 +462,7 @@ impl RouteSpecStore { cache_key = route_permutation_to_hop_cache(&nodes, permutation); // Skip routes we have already seen - if self.cache.hop_cache.contains(&cache_key) { + if inner.cache.hop_cache.contains(&cache_key) { return false; } @@ -529,10 +563,10 @@ impl RouteSpecStore { }; // Add to cache - Self::add_to_cache(&mut self.cache, cache_key, &rsd); + Self::add_to_cache(&mut inner.cache, cache_key, &rsd); // Keep route in spec store - self.content.details.insert(public_key, rsd); + inner.content.details.insert(public_key, rsd); Ok(Some(public_key)) } @@ -541,19 +575,21 @@ impl RouteSpecStore { where F: FnOnce(&RouteSpecDetail) -> R, { - self.detail(&public_key).map(|rsd| f(rsd)) + let inner = self.inner.lock(); + Self::detail(&*inner, &public_key).map(f) } - pub fn release_route(&mut self, public_key: DHTKey) { - if let Some(detail) = self.content.details.remove(&public_key) { + pub fn release_route(&self, public_key: DHTKey) { + let mut inner = self.inner.lock(); + if let Some(detail) = inner.content.details.remove(&public_key) { // Remove from hop cache let cache_key = route_hops_to_hop_cache(&detail.hops); - if !self.cache.hop_cache.remove(&cache_key) { + if !inner.cache.hop_cache.remove(&cache_key) { panic!("hop cache should have contained cache key"); } // Remove from used nodes cache for h in &detail.hops { - match self.cache.used_nodes.entry(*h) { + match inner.cache.used_nodes.entry(*h) { std::collections::hash_map::Entry::Occupied(mut o) => { *o.get_mut() -= 1; if *o.get() == 0 { @@ -566,7 +602,7 @@ impl RouteSpecStore { } } // Remove from end nodes cache - match self.cache.used_nodes.entry(*detail.hops.last().unwrap()) { + match inner.cache.used_nodes.entry(*detail.hops.last().unwrap()) { std::collections::hash_map::Entry::Occupied(mut o) => { *o.get_mut() -= 1; if *o.get() == 0 { @@ -584,14 +620,16 @@ impl RouteSpecStore { /// Find first matching unpublished route that fits into the selection criteria pub fn first_unpublished_route( - &mut self, + &self, min_hop_count: usize, max_hop_count: usize, stability: Stability, sequencing: Sequencing, directions: DirectionSet, ) -> Option { - for detail in &self.content.details { + let inner = self.inner.lock(); + + for detail in &inner.content.details { if detail.1.stability >= stability && detail.1.sequencing >= sequencing && detail.1.hops.len() >= min_hop_count @@ -611,14 +649,16 @@ impl RouteSpecStore { /// Returns an Err() if the parameters are wrong /// Returns Ok(None) if no allocation could happen at this time (not an error) pub fn compile_safety_route( - &mut self, + &self, rti: &mut RoutingTableInner, routing_table: RoutingTable, safety_selection: SafetySelection, private_route: PrivateRoute, ) -> EyreResult> { + let inner = &mut *self.inner.lock(); + let pr_hopcount = private_route.hop_count as usize; - let max_route_hop_count = self.max_route_hop_count; + let max_route_hop_count = self.unlocked_inner.max_route_hop_count; if pr_hopcount > max_route_hop_count { bail!("private route hop count too long"); } @@ -664,8 +704,7 @@ impl RouteSpecStore { // See if the preferred route is here let opt_safety_rsd: Option<(&mut RouteSpecDetail, DHTKey)> = if let Some(preferred_route) = safety_spec.preferred_route { - self.detail_mut(&preferred_route) - .map(|rsd| (rsd, preferred_route)) + Self::detail_mut(inner, &preferred_route).map(|rsd| (rsd, preferred_route)) } else { // Preferred safety route was not requested None @@ -683,7 +722,7 @@ impl RouteSpecStore { Direction::Outbound.into(), ) { // Found a route to use - (self.detail_mut(&sr_pubkey).unwrap(), sr_pubkey) + (Self::detail_mut(inner, &sr_pubkey).unwrap(), sr_pubkey) } else { // No route found, gotta allocate one let sr_pubkey = match self @@ -700,7 +739,7 @@ impl RouteSpecStore { Some(pk) => pk, None => return Ok(None), }; - (self.detail_mut(&sr_pubkey).unwrap(), sr_pubkey) + (Self::detail_mut(inner, &sr_pubkey).unwrap(), sr_pubkey) } }; @@ -837,14 +876,14 @@ impl RouteSpecStore { /// Assemble private route for publication pub fn assemble_private_route( - &mut self, + &self, rti: &RoutingTableInner, routing_table: RoutingTable, key: &DHTKey, ) -> EyreResult { - let rsd = self - .detail(&key) - .ok_or_else(|| eyre!("route does not exist"))?; + let inner = &*self.inner.lock(); + + let rsd = Self::detail(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; // See if we can optimize this compilation yet // We don't want to include full nodeinfo if we don't have to @@ -919,7 +958,8 @@ impl RouteSpecStore { /// When first deserialized, routes must be re-published in order to ensure they remain /// in the RouteSpecStore. pub fn mark_route_published(&mut self, key: &DHTKey) -> EyreResult<()> { - self.detail_mut(&key) + let inner = &mut *self.inner.lock(); + Self::detail_mut(inner, &key) .ok_or_else(|| eyre!("route does not exist"))? .published = true; Ok(()) @@ -929,7 +969,8 @@ impl RouteSpecStore { /// When first deserialized, routes must be re-tested for reachability /// This can be used to determine if routes need to be sent with full peerinfo or can just use a node id pub fn mark_route_reachable(&mut self, key: &DHTKey) -> EyreResult<()> { - self.detail_mut(&key) + let inner = &mut *self.inner.lock(); + Self::detail_mut(inner, &key) .ok_or_else(|| eyre!("route does not exist"))? .published = true; Ok(()) @@ -937,7 +978,8 @@ impl RouteSpecStore { /// Mark route as checked pub fn touch_route_checked(&mut self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { - self.detail_mut(&key) + let inner = &mut *self.inner.lock(); + Self::detail_mut(inner, &key) .ok_or_else(|| eyre!("route does not exist"))? .last_checked_ts = Some(cur_ts); Ok(()) @@ -945,7 +987,8 @@ impl RouteSpecStore { /// Mark route as used pub fn touch_route_used(&mut self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { - self.detail_mut(&key) + let inner = &mut *self.inner.lock(); + Self::detail_mut(inner, &key) .ok_or_else(|| eyre!("route does not exist"))? .last_used_ts = Some(cur_ts); Ok(()) @@ -953,20 +996,17 @@ impl RouteSpecStore { /// Record latency on the route pub fn record_latency(&mut self, key: &DHTKey, latency: u64) -> EyreResult<()> { - let lsa = &mut self - .detail_mut(&key) - .ok_or_else(|| eyre!("route does not exist"))? - .latency_stats_accounting; - self.detail_mut(&key) - .ok_or_else(|| eyre!("route does not exist"))? - .latency_stats = lsa.record_latency(latency); + let inner = &mut *self.inner.lock(); + + let rsd = Self::detail_mut(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; + rsd.latency_stats = rsd.latency_stats_accounting.record_latency(latency); Ok(()) } /// Get the calculated latency stats pub fn latency_stats(&mut self, key: &DHTKey) -> EyreResult { - Ok(self - .detail_mut(&key) + let inner = &mut *self.inner.lock(); + Ok(Self::detail_mut(inner, &key) .ok_or_else(|| eyre!("route does not exist"))? .latency_stats .clone()) @@ -974,27 +1014,24 @@ impl RouteSpecStore { /// Add download transfers to route pub fn add_down(&mut self, key: &DHTKey, bytes: u64) -> EyreResult<()> { - let tsa = &mut self - .detail_mut(&key) - .ok_or_else(|| eyre!("route does not exist"))? - .transfer_stats_accounting; - tsa.add_down(bytes); + let inner = &mut *self.inner.lock(); + let rsd = Self::detail_mut(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; + rsd.transfer_stats_accounting.add_down(bytes); Ok(()) } /// Add upload transfers to route pub fn add_up(&mut self, key: &DHTKey, bytes: u64) -> EyreResult<()> { - let tsa = &mut self - .detail_mut(&key) - .ok_or_else(|| eyre!("route does not exist"))? - .transfer_stats_accounting; - tsa.add_up(bytes); + let inner = &mut *self.inner.lock(); + let rsd = Self::detail_mut(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; + rsd.transfer_stats_accounting.add_up(bytes); Ok(()) } /// Process transfer statistics to get averages pub fn roll_transfers(&mut self, last_ts: u64, cur_ts: u64) { - for rsd in self.content.details.values_mut() { + let inner = &mut *self.inner.lock(); + for rsd in inner.content.details.values_mut() { rsd.transfer_stats_accounting.roll_transfers( last_ts, cur_ts, diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index b63462a8..6729667a 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -116,7 +116,7 @@ impl RoutingDomainDetailCommon { where F: FnOnce(&PeerInfo) -> R, { - let cpi = self.cached_peer_info.lock(); + let mut cpi = self.cached_peer_info.lock(); if cpi.is_none() { // Regenerate peer info let pi = PeerInfo::new( @@ -131,6 +131,7 @@ impl RoutingDomainDetailCommon { dial_info_detail_list: self.dial_info_details.clone(), relay_peer_info: self .relay_node + .as_ref() .and_then(|rn| rn.make_peer_info(self.routing_domain).map(Box::new)), }, NodeId::new(self.node_id), @@ -268,7 +269,7 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { } // Get the target's inbound relay, it must have one or it is not reachable - if let Some(inbound_relay) = node_b.relay_peer_info { + if let Some(inbound_relay) = &node_b.relay_peer_info { // Note that relay_peer_info could be node_a, in which case a connection already exists // and we shouldn't have even gotten here if inbound_relay.node_id.key == *node_a_id { @@ -348,7 +349,7 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { } } // If the node B has no direct dial info, it needs to have an inbound relay - else if let Some(inbound_relay) = node_b.relay_peer_info { + else if let Some(inbound_relay) = &node_b.relay_peer_info { // Can we reach the full relay? if first_filtered_dial_info_detail( node_a, @@ -363,7 +364,7 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { } // If node A can't reach the node by other means, it may need to use its own relay - if let Some(outbound_relay) = node_a.relay_peer_info { + if let Some(outbound_relay) = &node_a.relay_peer_info { return ContactMethod::OutboundRelay(outbound_relay.node_id.key); } diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index 10918799..4c90a59b 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -35,6 +35,7 @@ pub struct RoutingTableInner { impl RoutingTableInner { pub fn new(unlocked_inner: Arc) -> RoutingTableInner { + let config = unlocked_inner.config.clone(); RoutingTableInner { unlocked_inner, buckets: Vec::new(), @@ -45,7 +46,7 @@ impl RoutingTableInner { self_transfer_stats_accounting: TransferStatsAccounting::new(), self_transfer_stats: TransferStatsDownUp::default(), recent_peers: LruCache::new(RECENT_PEERS_TABLE_SIZE), - route_spec_store: RouteSpecStore::new(unlocked_inner.config.clone()), + route_spec_store: RouteSpecStore::new(config), } } @@ -105,20 +106,6 @@ impl RoutingTableInner { } } - pub fn with_route_spec_store_mut(&mut self, f: F) -> R - where - F: FnOnce(&mut RouteSpecStore, &mut RoutingTableInner) -> R, - { - f(&mut self.route_spec_store, self) - } - - pub fn with_route_spec_store(&self, f: F) -> R - where - F: FnOnce(&RouteSpecStore, &RoutingTableInner) -> R, - { - f(&self.route_spec_store, self) - } - pub fn relay_node(&self, domain: RoutingDomain) -> Option { self.with_routing_domain(domain, |rd| rd.common().relay_node()) } @@ -382,8 +369,8 @@ impl RoutingTableInner { "Starting routing table buckets purge. Table currently has {} nodes", self.bucket_entry_count ); - for bucket in &self.buckets { - bucket.kick(self, 0); + for bucket in &mut self.buckets { + bucket.kick(0); } log_rtab!(debug "Routing table buckets purge complete. Routing table now has {} nodes", @@ -399,11 +386,12 @@ impl RoutingTableInner { ); for bucket in &self.buckets { for entry in bucket.entries() { - entry.1.with_mut(self, |_rti, e| { + entry.1.with_mut_inner(|e| { e.clear_last_connections(); }); } } + log_rtab!(debug "Routing table last_connections purge complete. Routing table now has {} nodes", self.bucket_entry_count @@ -416,7 +404,7 @@ impl RoutingTableInner { let bucket = &mut self.buckets[idx]; let bucket_depth = Self::bucket_depth(idx); - if let Some(dead_node_ids) = bucket.kick(self, bucket_depth) { + if let Some(dead_node_ids) = bucket.kick(bucket_depth) { // Remove counts self.bucket_entry_count -= dead_node_ids.len(); log_rtab!(debug "Routing table now has {} nodes", self.bucket_entry_count); diff --git a/veilid-core/src/rpc_processor/coders/private_safety_route.rs b/veilid-core/src/rpc_processor/coders/private_safety_route.rs index 50604970..4ac99409 100644 --- a/veilid-core/src/rpc_processor/coders/private_safety_route.rs +++ b/veilid-core/src/rpc_processor/coders/private_safety_route.rs @@ -52,11 +52,11 @@ pub fn encode_route_hop( let node_builder = builder.reborrow().init_node(); match &route_hop.node { RouteNode::NodeId(ni) => { - let ni_builder = node_builder.init_node_id(); + let mut ni_builder = node_builder.init_node_id(); encode_public_key(&ni.key, &mut ni_builder)?; } RouteNode::PeerInfo(pi) => { - let pi_builder = node_builder.init_peer_info(); + let mut pi_builder = node_builder.init_peer_info(); encode_peer_info(&pi, &mut pi_builder)?; } } diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index d68ba5f4..033b08f7 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -30,16 +30,18 @@ pub enum Destination { impl Destination { pub fn direct(target: NodeRef) -> Self { + let sequencing = target.sequencing(); Self::Direct { target, - safety_selection: SafetySelection::Unsafe(target.sequencing()), + safety_selection: SafetySelection::Unsafe(sequencing), } } pub fn relay(relay: NodeRef, target: DHTKey) -> Self { + let sequencing = relay.sequencing(); Self::Relay { relay, target, - safety_selection: SafetySelection::Unsafe(relay.sequencing()), + safety_selection: SafetySelection::Unsafe(sequencing), } } pub fn private_route(private_route: PrivateRoute, safety_selection: SafetySelection) -> Self { diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index e1742359..fe177606 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -425,22 +425,23 @@ impl RPCProcessor { message_data: Vec, ) -> Result, RPCError> { let routing_table = self.routing_table(); + let rss = routing_table.route_spec_store(); + let pr_hop_count = private_route.hop_count; let pr_pubkey = private_route.public_key; - let compiled_route: CompiledRoute = - match self.routing_table().with_route_spec_store_mut(|rss, rti| { - // Compile the safety route with the private route - rss.compile_safety_route(rti, routing_table, safety_selection, private_route) - .map_err(RPCError::internal) - })? { - Some(cr) => cr, - None => { - return Ok(NetworkResult::no_connection_other( - "private route could not be compiled at this time", - )) - } - }; + // Compile the safety route with the private route + let compiled_route: CompiledRoute = match rss + .compile_safety_route(rti, routing_table, safety_selection, private_route) + .map_err(RPCError::internal)? + { + Some(cr) => cr, + None => { + return Ok(NetworkResult::no_connection_other( + "private route could not be compiled at this time", + )) + } + }; // Encrypt routed operation // Xmsg + ENC(Xmsg, DH(PKapr, SKbsr)) @@ -917,7 +918,7 @@ impl RPCProcessor { opt_sender_nr, } } - RPCMessageHeaderDetail::PrivateRoute(detail) => { + RPCMessageHeaderDetail::PrivateRoute(_) => { // Decode the RPC message let operation = { let reader = capnp::message::Reader::new(encoded_msg.data, Default::default()); diff --git a/veilid-core/src/rpc_processor/rpc_node_info_update.rs b/veilid-core/src/rpc_processor/rpc_node_info_update.rs index 6e61c754..0e317d9c 100644 --- a/veilid-core/src/rpc_processor/rpc_node_info_update.rs +++ b/veilid-core/src/rpc_processor/rpc_node_info_update.rs @@ -31,8 +31,14 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] pub(crate) async fn process_node_info_update(&self, msg: RPCMessage) -> Result<(), RPCError> { - let sender_node_id = msg.header.envelope.get_sender_id(); - let routing_domain = msg.header.routing_domain; + let detail = match msg.header.detail { + RPCMessageHeaderDetail::Direct(detail) => detail, + RPCMessageHeaderDetail::PrivateRoute(_) => { + return Err(RPCError::protocol("node_info_update must be direct")); + } + }; + let sender_node_id = detail.envelope.get_sender_id(); + let routing_domain = detail.routing_domain; // Get the statement let node_info_update = match msg.operation.into_kind() { diff --git a/veilid-core/src/rpc_processor/rpc_return_receipt.rs b/veilid-core/src/rpc_processor/rpc_return_receipt.rs index 31c19023..667ca81e 100644 --- a/veilid-core/src/rpc_processor/rpc_return_receipt.rs +++ b/veilid-core/src/rpc_processor/rpc_return_receipt.rs @@ -33,11 +33,23 @@ impl RPCProcessor { // Handle it let network_manager = self.network_manager(); - network_result_value_or_log!(debug - network_manager - .handle_in_band_receipt(receipt, msg.header.peer_noderef) - .await => {} - ); + + match msg.header.detail { + RPCMessageHeaderDetail::Direct(detail) => { + network_result_value_or_log!(debug + network_manager + .handle_in_band_receipt(receipt, detail.peer_noderef) + .await => {} + ); + } + RPCMessageHeaderDetail::PrivateRoute(detail) => { + network_result_value_or_log!(debug + network_manager + .handle_private_receipt(receipt) + .await => {} + ); + } + } Ok(()) } diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 91d5aebe..b12fa50b 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -170,8 +170,13 @@ impl RPCProcessor { // If the private route public key is our node id, then this was sent via safety route to our node directly // so there will be no signatures to validate let opt_pr_info = if private_route.public_key == self.routing_table.node_id() { - // the private route was a stub to our own node's secret - // return our secret key and an appropriate safety selection + // The private route was a stub + // Return our secret key and an appropriate safety selection + // + // Note: it is important that we never respond with a safety route to questions that come + // in without a private route. Giving away a safety route when the node id is known is + // a privacy violation! + // Get sequencing preference let sequencing = if detail .connection_descriptor @@ -191,46 +196,45 @@ impl RPCProcessor { let sender_id = detail.envelope.get_sender_id(); // Look up the private route and ensure it's one in our spec store - let opt_signatures_valid = self.routing_table.with_route_spec_store(|rss, rti| { - rss.with_route_spec_detail(&private_route.public_key, |rsd| { - // Ensure we have the right number of signatures - if routed_operation.signatures.len() != rsd.hops.len() - 1 { - // Wrong number of signatures - log_rpc!(debug "wrong number of signatures ({} should be {}) for routed operation on private route {}", routed_operation.signatures.len(), rsd.hops.len() - 1, private_route.public_key); - return None; - } - // Validate signatures to ensure the route was handled by the nodes and not messed with - for (hop_n, hop_public_key) in rsd.hops.iter().enumerate() { - // The last hop is not signed, as the whole packet is signed - if hop_n == routed_operation.signatures.len() { - // Verify the node we received the routed operation from is the last hop in our route - if *hop_public_key != sender_id { - log_rpc!(debug "received routed operation from the wrong hop ({} should be {}) on private route {}", hop_public_key.encode(), sender_id.encode(), private_route.public_key); - return None; - } - } else { - // Verify a signature for a hop node along the route - if let Err(e) = verify( - hop_public_key, - &routed_operation.data, - &routed_operation.signatures[hop_n], - ) { - log_rpc!(debug "failed to verify signature for hop {} at {} on private route {}", hop_n, hop_public_key, private_route.public_key); - return None; - } + let rss= self.routing_table.route_spec_store(); + let opt_signatures_valid = rss.with_route_spec_detail(&private_route.public_key, |rsd| { + // Ensure we have the right number of signatures + if routed_operation.signatures.len() != rsd.hops.len() - 1 { + // Wrong number of signatures + log_rpc!(debug "wrong number of signatures ({} should be {}) for routed operation on private route {}", routed_operation.signatures.len(), rsd.hops.len() - 1, private_route.public_key); + return None; + } + // Validate signatures to ensure the route was handled by the nodes and not messed with + for (hop_n, hop_public_key) in rsd.hops.iter().enumerate() { + // The last hop is not signed, as the whole packet is signed + if hop_n == routed_operation.signatures.len() { + // Verify the node we received the routed operation from is the last hop in our route + if *hop_public_key != sender_id { + log_rpc!(debug "received routed operation from the wrong hop ({} should be {}) on private route {}", hop_public_key.encode(), sender_id.encode(), private_route.public_key); + return None; + } + } else { + // Verify a signature for a hop node along the route + if let Err(e) = verify( + hop_public_key, + &routed_operation.data, + &routed_operation.signatures[hop_n], + ) { + log_rpc!(debug "failed to verify signature for hop {} at {} on private route {}", hop_n, hop_public_key, private_route.public_key); + return None; } } - // We got the correct signatures, return a key ans - Some(( - rsd.secret_key, - SafetySelection::Safe(SafetySpec { - preferred_route: Some(private_route.public_key), - hop_count: rsd.hops.len(), - stability: rsd.stability, - sequencing: rsd.sequencing, - }) - )) - }) + } + // We got the correct signatures, return a key ans + Some(( + rsd.secret_key, + SafetySelection::Safe(SafetySpec { + preferred_route: Some(private_route.public_key), + hop_count: rsd.hops.len(), + stability: rsd.stability, + sequencing: rsd.sequencing, + }) + )) }); opt_signatures_valid.ok_or_else(|| { RPCError::protocol("routed operation received on unallocated private route") diff --git a/veilid-core/src/rpc_processor/rpc_signal.rs b/veilid-core/src/rpc_processor/rpc_signal.rs index 08cb3713..63fbcb79 100644 --- a/veilid-core/src/rpc_processor/rpc_signal.rs +++ b/veilid-core/src/rpc_processor/rpc_signal.rs @@ -32,7 +32,7 @@ impl RPCProcessor { // Handle it let network_manager = self.network_manager(); network_result_value_or_log!(debug network_manager - .handle_signal(msg.header.envelope.get_sender_id(), signal.signal_info) + .handle_signal(signal.signal_info) .await .map_err(RPCError::network)? => { return Ok(()); diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 079665d4..feebaa5c 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -103,8 +103,15 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] pub(crate) async fn process_status_q(&self, msg: RPCMessage) -> Result<(), RPCError> { - let connection_descriptor = msg.header.connection_descriptor; - let routing_domain = msg.header.routing_domain; + let detail = match &msg.header.detail { + RPCMessageHeaderDetail::Direct(detail) => detail, + RPCMessageHeaderDetail::PrivateRoute(_) => { + return Err(RPCError::protocol("status_q must be direct")); + } + }; + + let connection_descriptor = detail.connection_descriptor; + let routing_domain = detail.routing_domain; // Get the question let status_q = match msg.operation.kind() { diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 9c73baf2..454acb9c 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -34,7 +34,7 @@ impl RPCProcessor { // Wait for receipt match eventual_value.await.take_value().unwrap() { - ReceiptEvent::ReturnedInBand { inbound_noderef: _ } => { + ReceiptEvent::ReturnedPrivate | ReceiptEvent::ReturnedInBand { inbound_noderef: _ } => { log_net!(debug "validate_dial_info receipt should be returned out-of-band".green()); Ok(false) } @@ -54,6 +54,13 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] pub(crate) async fn process_validate_dial_info(&self, msg: RPCMessage) -> Result<(), RPCError> { + let detail = match msg.header.detail { + RPCMessageHeaderDetail::Direct(detail) => detail, + RPCMessageHeaderDetail::PrivateRoute(_) => { + return Err(RPCError::protocol("validate_dial_info must be direct")); + } + }; + // Get the statement let RPCOperationValidateDialInfo { dial_info, @@ -74,8 +81,8 @@ impl RPCProcessor { // Use the address type though, to ensure we reach an ipv6 capable node if this is // an ipv6 address let routing_table = self.routing_table(); - let sender_id = msg.header.envelope.get_sender_id(); - let routing_domain = msg.header.routing_domain; + let sender_id = detail.envelope.get_sender_id(); + let routing_domain = detail.routing_domain; let node_count = { let c = self.config.get(); c.network.dht.max_find_node_count as usize @@ -141,8 +148,6 @@ impl RPCProcessor { .await .map_err(RPCError::network)?; - // tracing::Span::current().record("res", &tracing::field::display(res)); - Ok(()) } } From 5ae0bd834c1abc77c22e8731fb88f008d2d886c7 Mon Sep 17 00:00:00 2001 From: John Smith Date: Mon, 31 Oct 2022 22:03:05 -0400 Subject: [PATCH 26/67] fuck --- veilid-core/src/routing_table/mod.rs | 63 +++++++++++----- .../src/routing_table/route_spec_store.rs | 73 ++++++++++++++----- .../src/routing_table/routing_table_inner.rs | 38 ++++++---- veilid-core/src/rpc_processor/mod.rs | 2 +- .../rpc_processor/rpc_validate_dial_info.rs | 6 +- 5 files changed, 124 insertions(+), 58 deletions(-) diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 6214d335..ef6837d1 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -150,8 +150,25 @@ impl RoutingTable { /// Called to initialize the routing table after it is created pub async fn init(&self) -> EyreResult<()> { + debug!("starting routing table init"); + + // Set up routespecstore + debug!("starting route spec store init"); + let route_spec_store = match RouteSpecStore::load(self.clone()).await { + Ok(v) => v, + Err(e) => { + log_rtab!(warn "Error loading route spec store: {}. Resetting.", e); + RouteSpecStore::new(self.clone()) + } + }; + debug!("finished route spec store init"); + let mut inner = self.inner.write(); inner.init(self.clone()); + + inner.route_spec_store = Some(route_spec_store); + + debug!("finished routing table init"); Ok(()) } @@ -169,6 +186,16 @@ impl RoutingTable { error!("kick_buckets_task not stopped: {}", e); } + debug!("saving route spec store"); + let rss = { + let mut inner = self.inner.write(); + inner.route_spec_store.take() + }; + if let Some(rss) = rss { + rss.save().await; + } + debug!("shutting down routing table"); + let mut inner = self.inner.write(); inner.terminate(); *inner = RoutingTableInner::new(self.unlocked_inner.clone()); @@ -192,7 +219,7 @@ impl RoutingTable { } pub fn route_spec_store(&self) -> RouteSpecStore { - self.inner.read().route_spec_store.clone() + self.inner.read().route_spec_store.as_ref().unwrap().clone() } pub fn relay_node(&self, domain: RoutingDomain) -> Option { @@ -521,9 +548,9 @@ impl RoutingTable { pub fn make_inbound_dial_info_entry_filter( routing_domain: RoutingDomain, dial_info_filter: DialInfoFilter, - ) -> impl FnMut(&RoutingTableInner, &BucketEntryInner) -> bool { + ) -> Box bool> { // does it have matching public dial info? - move |_rti, e| { + Box::new(move |_rti, e| { if let Some(ni) = e.node_info(routing_domain) { if ni .first_filtered_dial_info_detail(DialInfoDetail::NO_SORT, |did| { @@ -535,16 +562,16 @@ impl RoutingTable { } } false - } + }) } /// Makes a filter that finds nodes capable of dialing a particular outbound dialinfo - pub fn make_outbound_dial_info_entry_filter<'s>( + pub fn make_outbound_dial_info_entry_filter( routing_domain: RoutingDomain, dial_info: DialInfo, - ) -> impl FnMut(&RoutingTableInner, &'s BucketEntryInner) -> bool { + ) -> Box bool> { // does the node's outbound capabilities match the dialinfo? - move |_rti, e| { + Box::new(move |_rti, e| { if let Some(ni) = e.node_info(routing_domain) { let dif = DialInfoFilter::all() .with_protocol_type_set(ni.outbound_protocols) @@ -554,19 +581,15 @@ impl RoutingTable { } } false - } + }) } /// Make a filter that wraps another filter - pub fn combine_entry_filters<'a, 'b, F, G>( - mut f1: F, - mut f2: G, - ) -> impl FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool - where - F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, - G: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, - { - move |rti, e| { + pub fn combine_entry_filters( + mut f1: Box bool>, + mut f2: Box bool>, + ) -> Box bool> { + Box::new(move |rti, e| { if !f1(rti, e) { return false; } @@ -574,16 +597,16 @@ impl RoutingTable { return false; } true - } + }) } - pub fn find_fast_public_nodes_filtered<'a, 'b, F>( + pub fn find_fast_public_nodes_filtered( &self, node_count: usize, mut entry_filter: F, ) -> Vec where - F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, + F: FnMut(&RoutingTableInner, &BucketEntryInner) -> bool, { self.inner .read() diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index f51f0589..9d49d7fa 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -55,7 +55,7 @@ struct RouteSpecDetail { } /// The core representation of the RouteSpecStore that can be serialized -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct RouteSpecStoreContent { /// All of the routes we have allocated so far details: HashMap, @@ -79,14 +79,25 @@ pub struct RouteSpecStoreInner { /// RouteSpecStore cache cache: RouteSpecStoreCache, } -#[derive(Debug)] + pub struct RouteSpecStoreUnlockedInner { + /// Handle to routing table + routing_table: RoutingTable, /// Maximum number of hops in a route max_route_hop_count: usize, /// Default number of hops in a route default_route_hop_count: usize, } +impl fmt::Debug for RouteSpecStoreUnlockedInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RouteSpecStoreUnlockedInner") + .field("max_route_hop_count", &self.max_route_hop_count) + .field("default_route_hop_count", &self.default_route_hop_count) + .finish() + } +} + /// The routing table's storage for private/safety routes #[derive(Clone, Debug)] pub struct RouteSpecStore { @@ -176,17 +187,16 @@ where heaps_permutation(&mut permutation, hop_count - 1, f) } -xxx get routing table handle into routespecstore -xxx first figure out when new/load get called, does routing table need 'init' or can we just pick the right time to load the cache? what about flushing the cache ? we don't 'save' it yet, that should probably get flushed at the same time as the DH cache. - impl RouteSpecStore { - pub fn new(config: VeilidConfig) -> Self { + pub fn new(routing_table: RoutingTable) -> Self { + let config = routing_table.network_manager().config(); let c = config.get(); Self { unlocked_inner: Arc::new(RouteSpecStoreUnlockedInner { max_route_hop_count: c.network.rpc.max_route_hop_count.into(), default_route_hop_count: c.network.rpc.default_route_hop_count.into(), + routing_table, }), inner: Arc::new(Mutex::new(RouteSpecStoreInner { content: RouteSpecStoreContent { @@ -243,6 +253,7 @@ impl RouteSpecStore { unlocked_inner: Arc::new(RouteSpecStoreUnlockedInner { max_route_hop_count: c.network.rpc.max_route_hop_count.into(), default_route_hop_count: c.network.rpc.default_route_hop_count.into(), + routing_table, }), inner: Arc::new(Mutex::new(inner)), }; @@ -250,17 +261,28 @@ impl RouteSpecStore { Ok(rss) } - pub async fn save(&self, routing_table: RoutingTable) -> EyreResult<()> { - let inner = self.inner.lock(); + pub async fn save(&self) -> EyreResult<()> { + let content = { + let inner = self.inner.lock(); + inner.content.clone() + }; // Save all the fields we care about to the cbor blob in table storage - let table_store = routing_table.network_manager().table_store(); + let table_store = self + .unlocked_inner + .routing_table + .network_manager() + .table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; - rsstdb.store_cbor(0, b"content", &inner.content).await?; + rsstdb.store_cbor(0, b"content", &content).await?; // Keep secrets in protected store as well - let pstore = routing_table.network_manager().protected_store(); - for (k, v) in &inner.content.details { + let pstore = self + .unlocked_inner + .routing_table + .network_manager() + .protected_store(); + for (k, v) in &content.details { if pstore .save_user_secret( &format!("RouteSpecStore_{}", k.encode()), @@ -319,15 +341,28 @@ impl RouteSpecStore { /// Returns None if no route could be allocated at this time pub fn allocate_route( &self, + stability: Stability, + sequencing: Sequencing, + hop_count: usize, + directions: DirectionSet, + ) -> EyreResult> { + let inner = &mut *self.inner.lock(); + let routing_table = self.unlocked_inner.routing_table.clone(); + let rti = &mut *routing_table.inner.write(); + + self.allocate_route_inner(inner, rti, stability, sequencing, hop_count, directions) + } + + fn allocate_route_inner( + &self, + inner: &mut RouteSpecStoreInner, rti: &RoutingTableInner, - routing_table: RoutingTable, stability: Stability, sequencing: Sequencing, hop_count: usize, directions: DirectionSet, ) -> EyreResult> { use core::cmp::Ordering; - let mut inner = self.inner.lock(); if hop_count < 1 { bail!("Not allocating route less than one hop in length"); @@ -537,7 +572,7 @@ impl RouteSpecStore { let hop_node_refs = route_nodes .iter() .map(|v| { - rti.lookup_node_ref(routing_table.clone(), nodes[*v].0) + rti.lookup_node_ref(self.unlocked_inner.routing_table.clone(), nodes[*v].0) .unwrap() }) .collect(); @@ -650,12 +685,12 @@ impl RouteSpecStore { /// Returns Ok(None) if no allocation could happen at this time (not an error) pub fn compile_safety_route( &self, - rti: &mut RoutingTableInner, - routing_table: RoutingTable, safety_selection: SafetySelection, private_route: PrivateRoute, ) -> EyreResult> { let inner = &mut *self.inner.lock(); + let routing_table = self.unlocked_inner.routing_table.clone(); + let rti = &mut *routing_table.inner.write(); let pr_hopcount = private_route.hop_count as usize; let max_route_hop_count = self.unlocked_inner.max_route_hop_count; @@ -726,9 +761,9 @@ impl RouteSpecStore { } else { // No route found, gotta allocate one let sr_pubkey = match self - .allocate_route( + .allocate_route_inner( + inner, rti, - routing_table.clone(), safety_spec.stability, safety_spec.sequencing, safety_spec.hop_count, diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index 4c90a59b..579c5c55 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -30,12 +30,11 @@ pub struct RoutingTableInner { /// Peers we have recently communicated with pub(super) recent_peers: LruCache, /// Storage for private/safety RouteSpecs - pub(super) route_spec_store: RouteSpecStore, + pub(super) route_spec_store: Option, } impl RoutingTableInner { pub fn new(unlocked_inner: Arc) -> RoutingTableInner { - let config = unlocked_inner.config.clone(); RoutingTableInner { unlocked_inner, buckets: Vec::new(), @@ -46,7 +45,7 @@ impl RoutingTableInner { self_transfer_stats_accounting: TransferStatsAccounting::new(), self_transfer_stats: TransferStatsDownUp::default(), recent_peers: LruCache::new(RECENT_PEERS_TABLE_SIZE), - route_spec_store: RouteSpecStore::new(config), + route_spec_store: None, } } @@ -331,12 +330,11 @@ impl RoutingTableInner { let bucket = Bucket::new(routing_table.clone()); self.buckets.push(bucket); } + Ok(()) } - pub fn terminate(&mut self) { - // - } + pub fn terminate(&mut self) {} pub fn configure_local_network_routing_domain( &mut self, @@ -448,15 +446,20 @@ impl RoutingTableInner { min_state: BucketEntryState, mut f: F, ) -> Option { + let mut entryvec = Vec::with_capacity(self.bucket_entry_count); for bucket in &self.buckets { for entry in bucket.entries() { if entry.1.with(self, |_rti, e| e.state(cur_ts) >= min_state) { - if let Some(out) = f(self, *entry.0, entry.1.clone()) { - return Some(out); - } + entryvec.push((*entry.0, entry.1.clone())); } } } + for entry in entryvec { + if let Some(out) = f(self, entry.0, entry.1) { + return Some(out); + } + } + None } @@ -469,15 +472,20 @@ impl RoutingTableInner { min_state: BucketEntryState, mut f: F, ) -> Option { + let mut entryvec = Vec::with_capacity(self.bucket_entry_count); for bucket in &self.buckets { for entry in bucket.entries() { if entry.1.with(self, |_rti, e| e.state(cur_ts) >= min_state) { - if let Some(out) = f(self, *entry.0, entry.1.clone()) { - return Some(out); - } + entryvec.push((*entry.0, entry.1.clone())); } } } + for entry in entryvec { + if let Some(out) = f(self, entry.0, entry.1) { + return Some(out); + } + } + None } @@ -777,7 +785,7 @@ impl RoutingTableInner { out } - pub fn touch_recent_peer(&self, node_id: DHTKey, last_connection: ConnectionDescriptor) { + pub fn touch_recent_peer(&mut self, node_id: DHTKey, last_connection: ConnectionDescriptor) { self.recent_peers .insert(node_id, RecentPeersEntry { last_connection }); } @@ -786,14 +794,14 @@ impl RoutingTableInner { // Find Nodes // Retrieve the fastest nodes in the routing table matching an entry filter - pub fn find_fast_public_nodes_filtered<'a, 'b, F>( + pub fn find_fast_public_nodes_filtered( &self, outer_self: RoutingTable, node_count: usize, mut entry_filter: F, ) -> Vec where - F: FnMut(&'a RoutingTableInner, &'b BucketEntryInner) -> bool, + F: FnMut(&RoutingTableInner, &BucketEntryInner) -> bool, { self.find_fastest_nodes( // count diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index fe177606..b68dac80 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -432,7 +432,7 @@ impl RPCProcessor { // Compile the safety route with the private route let compiled_route: CompiledRoute = match rss - .compile_safety_route(rti, routing_table, safety_selection, private_route) + .compile_safety_route(safety_selection, private_route) .map_err(RPCError::internal)? { Some(cr) => cr, diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 454acb9c..72b5f2f3 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -94,16 +94,16 @@ impl RPCProcessor { routing_domain, dial_info.clone(), ); - let will_validate_dial_info_filter = |_rti, e: &BucketEntryInner| { + let will_validate_dial_info_filter = Box::new(move |_rti, e: &BucketEntryInner| { if let Some(status) = &e.node_status(routing_domain) { status.will_validate_dial_info() } else { true } - }; + }); let filter = RoutingTable::combine_entry_filters( outbound_dial_info_entry_filter, - will_validate_dial_info_filter, + will_validate_dial_info_filter, fuck this shit. do it tomorrow. ); // Find nodes matching filter to redirect this to From d96b83fb4e5264a18e8479b6b9490cba0053ade0 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 1 Nov 2022 21:05:48 -0400 Subject: [PATCH 27/67] private route work --- veilid-core/src/attachment_manager.rs | 2 +- veilid-core/src/network_manager/mod.rs | 9 +- .../native/network_class_discovery.rs | 26 +- veilid-core/src/network_manager/tasks.rs | 2 +- veilid-core/src/receipt_manager.rs | 10 +- veilid-core/src/routing_table/mod.rs | 147 +++++----- .../src/routing_table/route_spec_store.rs | 122 ++++++--- .../src/routing_table/routing_domains.rs | 2 +- .../src/routing_table/routing_table_inner.rs | 251 +++++++++--------- veilid-core/src/rpc_processor/mod.rs | 6 +- .../src/rpc_processor/rpc_find_node.rs | 14 +- .../src/rpc_processor/rpc_return_receipt.rs | 2 +- veilid-core/src/rpc_processor/rpc_route.rs | 93 +++---- .../rpc_processor/rpc_validate_dial_info.rs | 31 ++- veilid-core/src/veilid_api/mod.rs | 2 +- 15 files changed, 377 insertions(+), 342 deletions(-) diff --git a/veilid-core/src/attachment_manager.rs b/veilid-core/src/attachment_manager.rs index 0d834561..8c304e44 100644 --- a/veilid-core/src/attachment_manager.rs +++ b/veilid-core/src/attachment_manager.rs @@ -342,7 +342,7 @@ impl AttachmentManager { #[instrument(level = "trace", skip(self))] fn attach(&self) { // Create long-running connection maintenance routine - let inner = self.inner.lock(); + let mut inner = self.inner.lock(); if inner.attachment_maintainer_jh.is_some() { return; } diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 5a771d95..8ae5256f 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -788,6 +788,7 @@ impl NetworkManager { pub async fn handle_private_receipt>( &self, receipt_data: R, + private_route: DHTKey, ) -> NetworkResult<()> { let receipt_manager = self.receipt_manager(); @@ -799,7 +800,7 @@ impl NetworkManager { }; receipt_manager - .handle_receipt(receipt, ReceiptReturned::Private) + .handle_receipt(receipt, ReceiptReturned::Private { private_route }) .await } @@ -1023,7 +1024,8 @@ impl NetworkManager { // Wait for the return receipt let inbound_nr = match eventual_value.await.take_value().unwrap() { - ReceiptEvent::ReturnedPrivate | ReceiptEvent::ReturnedOutOfBand => { + ReceiptEvent::ReturnedPrivate { private_route: _ } + | ReceiptEvent::ReturnedOutOfBand => { return Ok(NetworkResult::invalid_message( "reverse connect receipt should be returned in-band", )); @@ -1124,7 +1126,8 @@ impl NetworkManager { // Wait for the return receipt let inbound_nr = match eventual_value.await.take_value().unwrap() { - ReceiptEvent::ReturnedPrivate | ReceiptEvent::ReturnedOutOfBand => { + ReceiptEvent::ReturnedPrivate { private_route: _ } + | ReceiptEvent::ReturnedOutOfBand => { return Ok(NetworkResult::invalid_message( "hole punch receipt should be returned in-band", )); diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/network_class_discovery.rs index aff9bd28..cd4989ce 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/network_class_discovery.rs @@ -125,22 +125,24 @@ impl DiscoveryContext { RoutingDomain::PublicInternet, dial_info_filter.clone(), ); - let disallow_relays_filter = move |_rti, e: &BucketEntryInner| { - if let Some(n) = e.node_info(RoutingDomain::PublicInternet) { - n.relay_peer_info.is_none() - } else { - false - } - }; - let filter = RoutingTable::combine_entry_filters( - inbound_dial_info_entry_filter, - disallow_relays_filter, - ); + let disallow_relays_filter = Box::new( + move |rti: &RoutingTableInner, _k: DHTKey, v: Option>| { + let v = v.unwrap(); + v.with(rti, |_rti, e| { + if let Some(n) = e.node_info(RoutingDomain::PublicInternet) { + n.relay_peer_info.is_none() + } else { + false + } + }) + }, + ) as RoutingTableEntryFilter; + let filters = VecDeque::from([inbound_dial_info_entry_filter, disallow_relays_filter]); // Find public nodes matching this filter let peers = self .routing_table - .find_fast_public_nodes_filtered(node_count, filter); + .find_fast_public_nodes_filtered(node_count, filters); if peers.is_empty() { log_net!( "no external address detection peers of type {:?}:{:?}", diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index 5de6568c..ddcc065d 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -495,7 +495,7 @@ impl NetworkManager { // even the unreliable ones, and ask them to find nodes close to our node too let noderefs = routing_table.find_fastest_nodes( min_peer_count, - |_rti, _k, _v| true, + VecDeque::new(), |_rti, k: DHTKey, v: Option>| { NodeRef::new(routing_table.clone(), k, v.unwrap().clone(), None) }, diff --git a/veilid-core/src/receipt_manager.rs b/veilid-core/src/receipt_manager.rs index 28ba23d2..4ea815e8 100644 --- a/veilid-core/src/receipt_manager.rs +++ b/veilid-core/src/receipt_manager.rs @@ -11,7 +11,7 @@ use xx::*; pub enum ReceiptEvent { ReturnedOutOfBand, ReturnedInBand { inbound_noderef: NodeRef }, - ReturnedPrivate, + ReturnedPrivate { private_route: DHTKey }, Expired, Cancelled, } @@ -20,7 +20,7 @@ pub enum ReceiptEvent { pub enum ReceiptReturned { OutOfBand, InBand { inbound_noderef: NodeRef }, - Private, + Private { private_route: DHTKey }, } pub trait ReceiptCallback: Send + 'static { @@ -412,7 +412,7 @@ impl ReceiptManager { match receipt_returned { ReceiptReturned::OutOfBand => "OutOfBand".to_owned(), ReceiptReturned::InBand { ref inbound_noderef } => format!("InBand({})", inbound_noderef), - ReceiptReturned::Private => "Private".to_owned(), + ReceiptReturned::Private { ref private_route } => format!("Private({})", private_route), }, if extra_data.is_empty() { "".to_owned() @@ -450,7 +450,9 @@ impl ReceiptManager { } => ReceiptEvent::ReturnedInBand { inbound_noderef: inbound_noderef.clone(), }, - ReceiptReturned::Private => ReceiptEvent::ReturnedPrivate, + ReceiptReturned::Private { ref private_route } => ReceiptEvent::ReturnedPrivate { + private_route: private_route.clone(), + }, }; let callback_future = Self::perform_callback(receipt_event, &mut record_mut); diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index ef6837d1..96fb913b 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -25,8 +25,6 @@ pub use routing_domains::*; pub use routing_table_inner::*; pub use stats_accounting::*; -const RECENT_PEERS_TABLE_SIZE: usize = 64; - ////////////////////////////////////////////////////////////////////////// pub type LowLevelProtocolPorts = BTreeSet<(LowLevelProtocolType, AddressType, u16)>; @@ -36,6 +34,8 @@ pub struct LowLevelPortInfo { pub low_level_protocol_ports: LowLevelProtocolPorts, pub protocol_to_port: ProtocolToPortMapping, } +pub type RoutingTableEntryFilter = + Box FnMut(&'r RoutingTableInner, DHTKey, Option>) -> bool + Send>; #[derive(Clone, Debug, Default)] pub struct RoutingTableHealth { @@ -47,7 +47,7 @@ pub struct RoutingTableHealth { pub dead_entry_count: usize, } -struct RoutingTableUnlockedInner { +pub(super) struct RoutingTableUnlockedInner { // Accessors config: VeilidConfig, network_manager: NetworkManager, @@ -164,7 +164,7 @@ impl RoutingTable { debug!("finished route spec store init"); let mut inner = self.inner.write(); - inner.init(self.clone()); + inner.init(self.clone())?; inner.route_spec_store = Some(route_spec_store); @@ -192,7 +192,9 @@ impl RoutingTable { inner.route_spec_store.take() }; if let Some(rss) = rss { - rss.save().await; + if let Err(e) = rss.save().await { + error!("couldn't save route spec store: {}", e); + } } debug!("shutting down routing table"); @@ -545,23 +547,30 @@ impl RoutingTable { } /// Makes a filter that finds nodes with a matching inbound dialinfo - pub fn make_inbound_dial_info_entry_filter( + pub fn make_inbound_dial_info_entry_filter<'a>( routing_domain: RoutingDomain, dial_info_filter: DialInfoFilter, - ) -> Box bool> { + ) -> RoutingTableEntryFilter { // does it have matching public dial info? - Box::new(move |_rti, e| { - if let Some(ni) = e.node_info(routing_domain) { - if ni - .first_filtered_dial_info_detail(DialInfoDetail::NO_SORT, |did| { - did.matches_filter(&dial_info_filter) - }) + Box::new(move |rti, _k, e| { + if let Some(e) = e { + e.with(rti, |_rti, e| { + if let Some(ni) = e.node_info(routing_domain) { + if ni + .first_filtered_dial_info_detail(DialInfoDetail::NO_SORT, |did| { + did.matches_filter(&dial_info_filter) + }) + .is_some() + { + return true; + } + } + false + }) + } else { + rti.first_filtered_dial_info_detail(routing_domain.into(), &dial_info_filter) .is_some() - { - return true; - } } - false }) } @@ -569,48 +578,36 @@ impl RoutingTable { pub fn make_outbound_dial_info_entry_filter( routing_domain: RoutingDomain, dial_info: DialInfo, - ) -> Box bool> { + ) -> RoutingTableEntryFilter { // does the node's outbound capabilities match the dialinfo? - Box::new(move |_rti, e| { - if let Some(ni) = e.node_info(routing_domain) { - let dif = DialInfoFilter::all() - .with_protocol_type_set(ni.outbound_protocols) - .with_address_type_set(ni.address_types); - if dial_info.matches_filter(&dif) { - return true; - } + Box::new(move |rti, _k, e| { + if let Some(e) = e { + e.with(rti, |_rti, e| { + if let Some(ni) = e.node_info(routing_domain) { + let dif = DialInfoFilter::all() + .with_protocol_type_set(ni.outbound_protocols) + .with_address_type_set(ni.address_types); + if dial_info.matches_filter(&dif) { + return true; + } + } + false + }) + } else { + let dif = rti.get_outbound_dial_info_filter(routing_domain); + dial_info.matches_filter(&dif) } - false }) } - /// Make a filter that wraps another filter - pub fn combine_entry_filters( - mut f1: Box bool>, - mut f2: Box bool>, - ) -> Box bool> { - Box::new(move |rti, e| { - if !f1(rti, e) { - return false; - } - if !f2(rti, e) { - return false; - } - true - }) - } - - pub fn find_fast_public_nodes_filtered( + pub fn find_fast_public_nodes_filtered( &self, node_count: usize, - mut entry_filter: F, - ) -> Vec - where - F: FnMut(&RoutingTableInner, &BucketEntryInner) -> bool, - { + filters: VecDeque, + ) -> Vec { self.inner .read() - .find_fast_public_nodes_filtered(self.clone(), node_count, entry_filter) + .find_fast_public_nodes_filtered(self.clone(), node_count, filters) } /// Retrieve up to N of each type of protocol capable nodes @@ -621,14 +618,12 @@ impl RoutingTable { ProtocolType::WS, ProtocolType::WSS, ]; + let protocol_types_len = protocol_types.len(); let mut nodes_proto_v4 = vec![0usize, 0usize, 0usize, 0usize]; let mut nodes_proto_v6 = vec![0usize, 0usize, 0usize, 0usize]; - self.find_fastest_nodes( - // count - protocol_types.len() * 2 * max_per_type, - // filter - move |rti, _k: DHTKey, v: Option>| { + let filter = Box::new( + move |rti: &RoutingTableInner, _k: DHTKey, v: Option>| { let entry = v.unwrap(); entry.with(rti, |_rti, e| { // skip nodes on our local network here @@ -668,64 +663,68 @@ impl RoutingTable { .unwrap_or(false) }) }, - // transform + ) as RoutingTableEntryFilter; + + let filters = VecDeque::from([filter]); + + self.find_fastest_nodes( + protocol_types_len * 2 * max_per_type, + filters, |_rti, k: DHTKey, v: Option>| { NodeRef::new(self.clone(), k, v.unwrap().clone(), None) }, ) } - pub fn find_peers_with_sort_and_filter<'a, 'b, F, C, T, O>( + pub fn find_peers_with_sort_and_filter( &self, node_count: usize, cur_ts: u64, - mut filter: F, + filters: VecDeque, compare: C, - mut transform: T, + transform: T, ) -> Vec where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - C: FnMut( + C: for<'a, 'b> FnMut( &'a RoutingTableInner, &'b (DHTKey, Option>), &'b (DHTKey, Option>), ) -> core::cmp::Ordering, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + T: for<'r> FnMut(&'r RoutingTableInner, DHTKey, Option>) -> O + Send, { self.inner .read() - .find_peers_with_sort_and_filter(node_count, cur_ts, filter, compare, transform) + .find_peers_with_sort_and_filter(node_count, cur_ts, filters, compare, transform) } - pub fn find_fastest_nodes<'a, T, F, O>( + pub fn find_fastest_nodes<'a, T, O>( &self, node_count: usize, - mut filter: F, + filters: VecDeque, transform: T, ) -> Vec where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + T: for<'r> FnMut(&'r RoutingTableInner, DHTKey, Option>) -> O + Send, { self.inner .read() - .find_fastest_nodes(node_count, filter, transform) + .find_fastest_nodes(node_count, filters, transform) } - pub fn find_closest_nodes<'a, F, T, O>( + pub fn find_closest_nodes<'a, T, O>( &self, node_id: DHTKey, - filter: F, - mut transform: T, + filters: VecDeque, + transform: T, ) -> Vec where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + T: for<'r> FnMut(&'r RoutingTableInner, DHTKey, Option>) -> O + Send, { self.inner .read() - .find_closest_nodes(node_id, filter, transform) + .find_closest_nodes(node_id, filters, transform) } + #[instrument(level = "trace", skip(self), ret)] pub fn register_find_node_answer(&self, peers: Vec) -> Vec { let node_id = self.node_id(); diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 9d49d7fa..12dddeed 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -124,7 +124,7 @@ fn route_permutation_to_hop_cache(nodes: &[(DHTKey, NodeInfo)], perm: &[usize]) /// number of route permutations is the number of unique orderings /// for a set of nodes, given that the first node is fixed -fn get_route_permutation_count(hop_count: usize) -> usize { +fn _get_route_permutation_count(hop_count: usize) -> usize { if hop_count == 0 { unreachable!(); } @@ -374,39 +374,43 @@ impl RouteSpecStore { // Get list of all nodes, and sort them for selection let cur_ts = intf::get_timestamp(); - let filter = |rti, _k: DHTKey, v: Option>| -> bool { - // Exclude our own node from routes - if v.is_none() { - return false; - } - let v = v.unwrap(); + let filter = Box::new( + move |rti: &RoutingTableInner, _k: DHTKey, v: Option>| -> bool { + // Exclude our own node from routes + if v.is_none() { + return false; + } + let v = v.unwrap(); - // Exclude nodes on our local network - let on_local_network = v.with(rti, |_rti, e| { - e.node_info(RoutingDomain::LocalNetwork).is_some() - }); - if on_local_network { - return false; - } + // Exclude nodes on our local network + let on_local_network = v.with(rti, |_rti, e| { + e.node_info(RoutingDomain::LocalNetwork).is_some() + }); + if on_local_network { + return false; + } - // Exclude nodes with no publicinternet nodeinfo, or incompatible nodeinfo or node status won't route - v.with(rti, |_rti, e| { - let node_info_ok = if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) { - ni.has_sequencing_matched_dial_info(sequencing) - } else { - false - }; - let node_status_ok = if let Some(ns) = e.node_status(RoutingDomain::PublicInternet) - { - ns.will_route() - } else { - false - }; + // Exclude nodes with no publicinternet nodeinfo, or incompatible nodeinfo or node status won't route + v.with(rti, move |_rti, e| { + let node_info_ok = if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) + { + ni.has_sequencing_matched_dial_info(sequencing) + } else { + false + }; + let node_status_ok = + if let Some(ns) = e.node_status(RoutingDomain::PublicInternet) { + ns.will_route() + } else { + false + }; - node_info_ok && node_status_ok - }) - }; - let compare = |rti, + node_info_ok && node_status_ok + }) + }, + ) as RoutingTableEntryFilter; + let filters = VecDeque::from([filter]); + let compare = |rti: &RoutingTableInner, v1: &(DHTKey, Option>), v2: &(DHTKey, Option>)| -> Ordering { @@ -461,7 +465,10 @@ impl RouteSpecStore { }); cmpout }; - let transform = |rti, k: DHTKey, v: Option>| -> (DHTKey, NodeInfo) { + let transform = |rti: &RoutingTableInner, + k: DHTKey, + v: Option>| + -> (DHTKey, NodeInfo) { // Return the key and the nodeinfo for that key ( k, @@ -479,7 +486,7 @@ impl RouteSpecStore { BucketEntryState::Unreliable, ); let nodes = - rti.find_peers_with_sort_and_filter(node_count, cur_ts, filter, compare, transform); + rti.find_peers_with_sort_and_filter(node_count, cur_ts, filters, compare, transform); // If we couldn't find enough nodes, wait until we have more nodes in the routing table if nodes.len() < hop_count { @@ -606,12 +613,49 @@ impl RouteSpecStore { Ok(Some(public_key)) } - pub fn with_route_spec_detail(&self, public_key: &DHTKey, f: F) -> Option - where - F: FnOnce(&RouteSpecDetail) -> R, - { - let inner = self.inner.lock(); - Self::detail(&*inner, &public_key).map(f) + pub fn validate_signatures( + &self, + public_key: &DHTKey, + signatures: &[DHTSignature], + data: &[u8], + last_hop_id: DHTKey, + ) -> EyreResult> { + let inner = &*self.inner.lock(); + let rsd = Self::detail(inner, &public_key).ok_or_else(|| eyre!("route does not exist"))?; + + // Ensure we have the right number of signatures + if signatures.len() != rsd.hops.len() - 1 { + // Wrong number of signatures + log_rpc!(debug "wrong number of signatures ({} should be {}) for routed operation on private route {}", signatures.len(), rsd.hops.len() - 1, public_key); + return Ok(None); + } + // Validate signatures to ensure the route was handled by the nodes and not messed with + for (hop_n, hop_public_key) in rsd.hops.iter().enumerate() { + // The last hop is not signed, as the whole packet is signed + if hop_n == signatures.len() { + // Verify the node we received the routed operation from is the last hop in our route + if *hop_public_key != last_hop_id { + log_rpc!(debug "received routed operation from the wrong hop ({} should be {}) on private route {}", hop_public_key.encode(), last_hop_id.encode(), public_key); + return Ok(None); + } + } else { + // Verify a signature for a hop node along the route + if let Err(e) = verify(hop_public_key, data, &signatures[hop_n]) { + log_rpc!(debug "failed to verify signature for hop {} at {} on private route {}: {}", hop_n, hop_public_key, public_key, e); + return Ok(None); + } + } + } + // We got the correct signatures, return a key ans + Ok(Some(( + rsd.secret_key, + SafetySelection::Safe(SafetySpec { + preferred_route: Some(*public_key), + hop_count: rsd.hops.len(), + stability: rsd.stability, + sequencing: rsd.sequencing, + }), + ))) } pub fn release_route(&self, public_key: DHTKey) { diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 6729667a..25efeddd 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -2,7 +2,7 @@ use super::*; /// Mechanism required to contact another node #[derive(Clone, Debug)] -pub(crate) enum ContactMethod { +pub enum ContactMethod { /// Node is not reachable by any means Unreachable, /// Connection should have already existed diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index 579c5c55..2e7c9989 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -34,7 +34,7 @@ pub struct RoutingTableInner { } impl RoutingTableInner { - pub fn new(unlocked_inner: Arc) -> RoutingTableInner { + pub(super) fn new(unlocked_inner: Arc) -> RoutingTableInner { RoutingTableInner { unlocked_inner, buckets: Vec::new(), @@ -794,22 +794,16 @@ impl RoutingTableInner { // Find Nodes // Retrieve the fastest nodes in the routing table matching an entry filter - pub fn find_fast_public_nodes_filtered( + pub fn find_fast_public_nodes_filtered( &self, outer_self: RoutingTable, node_count: usize, - mut entry_filter: F, - ) -> Vec - where - F: FnMut(&RoutingTableInner, &BucketEntryInner) -> bool, - { - self.find_fastest_nodes( - // count - node_count, - // filter - |rti, _k: DHTKey, v: Option>| { + mut filters: VecDeque, + ) -> Vec { + let public_node_filter = Box::new( + |rti: &RoutingTableInner, _k: DHTKey, v: Option>| { let entry = v.unwrap(); - entry.with(rti, |rti, e| { + entry.with(rti, |_rti, e| { // skip nodes on local network if e.node_info(RoutingDomain::LocalNetwork).is_some() { return false; @@ -818,12 +812,16 @@ impl RoutingTableInner { if e.node_info(RoutingDomain::PublicInternet).is_none() { return false; } - // skip nodes that dont match entry filter - entry_filter(rti, e) + true }) }, - // transform - |_rti, k: DHTKey, v: Option>| { + ) as RoutingTableEntryFilter; + filters.push_front(public_node_filter); + + self.find_fastest_nodes( + node_count, + filters, + |_rti: &RoutingTableInner, k: DHTKey, v: Option>| { NodeRef::new(outer_self.clone(), k, v.unwrap().clone(), None) }, ) @@ -858,37 +856,42 @@ impl RoutingTableInner { } } - pub fn find_peers_with_sort_and_filter<'a, 'b, F, C, T, O>( + pub fn find_peers_with_sort_and_filter( &self, node_count: usize, cur_ts: u64, - mut filter: F, - compare: C, + mut filters: VecDeque, + mut compare: C, mut transform: T, ) -> Vec where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - C: FnMut( + C: for<'a, 'b> FnMut( &'a RoutingTableInner, &'b (DHTKey, Option>), &'b (DHTKey, Option>), ) -> core::cmp::Ordering, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + T: for<'r> FnMut(&'r RoutingTableInner, DHTKey, Option>) -> O, { // collect all the nodes for sorting let mut nodes = Vec::<(DHTKey, Option>)>::with_capacity(self.bucket_entry_count + 1); // add our own node (only one of there with the None entry) - if filter(self, self.unlocked_inner.node_id, None) { - nodes.push((self.unlocked_inner.node_id, None)); + for filter in &mut filters { + if filter(self, self.unlocked_inner.node_id, None) { + nodes.push((self.unlocked_inner.node_id, None)); + break; + } } // add all nodes from buckets self.with_entries(cur_ts, BucketEntryState::Unreliable, |rti, k, v| { // Apply filter - if filter(rti, k, Some(v.clone())) { - nodes.push((k, Some(v.clone()))); + for filter in &mut filters { + if filter(rti, k, Some(v.clone())) { + nodes.push((k, Some(v.clone()))); + break; + } } Option::<()>::None }); @@ -907,97 +910,99 @@ impl RoutingTableInner { out } - pub fn find_fastest_nodes<'a, T, F, O>( + pub fn find_fastest_nodes( &self, node_count: usize, - mut filter: F, + mut filters: VecDeque, transform: T, ) -> Vec where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + T: for<'r> FnMut(&'r RoutingTableInner, DHTKey, Option>) -> O, { let cur_ts = intf::get_timestamp(); - let out = self.find_peers_with_sort_and_filter( - node_count, - cur_ts, - // filter - |rti, k, v| { + + // Add filter to remove dead nodes always + let filter_dead = Box::new( + move |rti: &RoutingTableInner, _k: DHTKey, v: Option>| { if let Some(entry) = &v { // always filter out dead nodes if entry.with(rti, |_rti, e| e.state(cur_ts) == BucketEntryState::Dead) { false } else { - filter(rti, k, v) + true } } else { // always filter out self peer, as it is irrelevant to the 'fastest nodes' search false } }, - // sort - |rti, (a_key, a_entry), (b_key, b_entry)| { - // same nodes are always the same - if a_key == b_key { - return core::cmp::Ordering::Equal; - } - // our own node always comes last (should not happen, here for completeness) - if a_entry.is_none() { - return core::cmp::Ordering::Greater; - } - if b_entry.is_none() { - return core::cmp::Ordering::Less; - } - // reliable nodes come first - let ae = a_entry.as_ref().unwrap(); - let be = b_entry.as_ref().unwrap(); - ae.with(rti, |rti, ae| { - be.with(rti, |_rti, be| { - let ra = ae.check_reliable(cur_ts); - let rb = be.check_reliable(cur_ts); - if ra != rb { - if ra { - return core::cmp::Ordering::Less; - } else { - return core::cmp::Ordering::Greater; - } - } + ) as RoutingTableEntryFilter; + filters.push_front(filter_dead); - // latency is the next metric, closer nodes first - let a_latency = match ae.peer_stats().latency.as_ref() { - None => { - // treat unknown latency as slow - return core::cmp::Ordering::Greater; - } - Some(l) => l, - }; - let b_latency = match be.peer_stats().latency.as_ref() { - None => { - // treat unknown latency as slow - return core::cmp::Ordering::Less; - } - Some(l) => l, - }; - // Sort by average latency - a_latency.average.cmp(&b_latency.average) - }) + // Fastest sort + let sort = |rti: &RoutingTableInner, + (a_key, a_entry): &(DHTKey, Option>), + (b_key, b_entry): &(DHTKey, Option>)| { + // same nodes are always the same + if a_key == b_key { + return core::cmp::Ordering::Equal; + } + // our own node always comes last (should not happen, here for completeness) + if a_entry.is_none() { + return core::cmp::Ordering::Greater; + } + if b_entry.is_none() { + return core::cmp::Ordering::Less; + } + // reliable nodes come first + let ae = a_entry.as_ref().unwrap(); + let be = b_entry.as_ref().unwrap(); + ae.with(rti, |rti, ae| { + be.with(rti, |_rti, be| { + let ra = ae.check_reliable(cur_ts); + let rb = be.check_reliable(cur_ts); + if ra != rb { + if ra { + return core::cmp::Ordering::Less; + } else { + return core::cmp::Ordering::Greater; + } + } + + // latency is the next metric, closer nodes first + let a_latency = match ae.peer_stats().latency.as_ref() { + None => { + // treat unknown latency as slow + return core::cmp::Ordering::Greater; + } + Some(l) => l, + }; + let b_latency = match be.peer_stats().latency.as_ref() { + None => { + // treat unknown latency as slow + return core::cmp::Ordering::Less; + } + Some(l) => l, + }; + // Sort by average latency + a_latency.average.cmp(&b_latency.average) }) - }, - // transform, - transform, - ); + }) + }; + + let out = + self.find_peers_with_sort_and_filter(node_count, cur_ts, filters, sort, transform); out } - pub fn find_closest_nodes<'a, F, T, O>( + pub fn find_closest_nodes( &self, node_id: DHTKey, - filter: F, - mut transform: T, + filters: VecDeque, + transform: T, ) -> Vec where - F: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> bool, - T: FnMut(&'a RoutingTableInner, DHTKey, Option>) -> O, + T: for<'r> FnMut(&'r RoutingTableInner, DHTKey, Option>) -> O, { let cur_ts = intf::get_timestamp(); let node_count = { @@ -1005,41 +1010,39 @@ impl RoutingTableInner { let c = config.get(); c.network.dht.max_find_node_count as usize }; - let out = self.find_peers_with_sort_and_filter( - node_count, - cur_ts, - // filter - filter, - // sort - |rti, (a_key, a_entry), (b_key, b_entry)| { - // same nodes are always the same - if a_key == b_key { - return core::cmp::Ordering::Equal; - } - // reliable nodes come first, pessimistically treating our own node as unreliable - let ra = a_entry - .as_ref() - .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); - let rb = b_entry - .as_ref() - .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); - if ra != rb { - if ra { - return core::cmp::Ordering::Less; - } else { - return core::cmp::Ordering::Greater; - } - } + // closest sort + let sort = |rti: &RoutingTableInner, + (a_key, a_entry): &(DHTKey, Option>), + (b_key, b_entry): &(DHTKey, Option>)| { + // same nodes are always the same + if a_key == b_key { + return core::cmp::Ordering::Equal; + } - // distance is the next metric, closer nodes first - let da = distance(a_key, &node_id); - let db = distance(b_key, &node_id); - da.cmp(&db) - }, - // transform, - &mut transform, - ); + // reliable nodes come first, pessimistically treating our own node as unreliable + let ra = a_entry + .as_ref() + .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); + let rb = b_entry + .as_ref() + .map_or(false, |x| x.with(rti, |_rti, x| x.check_reliable(cur_ts))); + if ra != rb { + if ra { + return core::cmp::Ordering::Less; + } else { + return core::cmp::Ordering::Greater; + } + } + + // distance is the next metric, closer nodes first + let da = distance(a_key, &node_id); + let db = distance(b_key, &node_id); + da.cmp(&db) + }; + + let out = + self.find_peers_with_sort_and_filter(node_count, cur_ts, filters, sort, transform); log_rtab!(">> find_closest_nodes: node count = {}", out.len()); out } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index b68dac80..e020aa45 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -171,7 +171,6 @@ pub struct RPCProcessorUnlockedInner { queue_size: u32, concurrency: u32, max_route_hop_count: usize, - default_route_hop_count: usize, validate_dial_info_receipt_time_ms: u32, update_callback: UpdateCallback, waiting_rpc_table: OperationWaiter, @@ -208,7 +207,6 @@ impl RPCProcessor { let queue_size = c.network.rpc.queue_size; let timeout = ms_to_us(c.network.rpc.timeout_ms); let max_route_hop_count = c.network.rpc.max_route_hop_count as usize; - let default_route_hop_count = c.network.rpc.default_route_hop_count as usize; if concurrency == 0 { concurrency = intf::get_concurrency() / 2; if concurrency == 0 { @@ -222,7 +220,6 @@ impl RPCProcessor { queue_size, concurrency, max_route_hop_count, - default_route_hop_count, validate_dial_info_receipt_time_ms, update_callback, waiting_rpc_table: OperationWaiter::new(), @@ -418,7 +415,7 @@ impl RPCProcessor { } // Wrap an operation with a private route inside a safety route - pub(super) fn wrap_with_route( + fn wrap_with_route( &self, safety_selection: SafetySelection, private_route: PrivateRoute, @@ -540,6 +537,7 @@ impl RPCProcessor { match safety_selection { SafetySelection::Unsafe(sequencing) => { // Apply safety selection sequencing requirement if it is more strict than the node_ref's sequencing requirement + let mut node_ref = node_ref.clone(); if sequencing > node_ref.sequencing() { node_ref.set_sequencing(sequencing) } diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index b5674b70..65dde614 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -62,22 +62,26 @@ impl RPCProcessor { // add node information for the requesting node to our routing table let routing_table = self.routing_table(); - let network_manager = self.network_manager(); let has_valid_own_node_info = routing_table.has_valid_own_node_info(RoutingDomain::PublicInternet); let own_peer_info = routing_table.get_own_peer_info(RoutingDomain::PublicInternet); // find N nodes closest to the target node in our routing table - let closest_nodes = routing_table.find_closest_nodes( - find_node_q.node_id, - // filter - |rti, _k, v| { + + let filter = Box::new( + move |rti: &RoutingTableInner, _k: DHTKey, v: Option>| { rti.filter_has_valid_signed_node_info( RoutingDomain::PublicInternet, has_valid_own_node_info, v, ) }, + ) as RoutingTableEntryFilter; + let filters = VecDeque::from([filter]); + + let closest_nodes = routing_table.find_closest_nodes( + find_node_q.node_id, + filters, // transform |rti, k, v| { rti.transform_to_peer_info( diff --git a/veilid-core/src/rpc_processor/rpc_return_receipt.rs b/veilid-core/src/rpc_processor/rpc_return_receipt.rs index 667ca81e..3087dd0f 100644 --- a/veilid-core/src/rpc_processor/rpc_return_receipt.rs +++ b/veilid-core/src/rpc_processor/rpc_return_receipt.rs @@ -45,7 +45,7 @@ impl RPCProcessor { RPCMessageHeaderDetail::PrivateRoute(detail) => { network_result_value_or_log!(debug network_manager - .handle_private_receipt(receipt) + .handle_private_receipt(receipt, detail.private_route) .await => {} ); } diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index b12fa50b..6c3f4384 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -121,7 +121,8 @@ impl RPCProcessor { if next_private_route.hop_count != 0 { let node_id = self.routing_table.node_id(); let node_id_secret = self.routing_table.node_id_secret(); - let sig = sign(&node_id, &node_id_secret, &route.operation.data).map_err(RPCError::internal)?; + let sig = sign(&node_id, &node_id_secret, &route.operation.data) + .map_err(RPCError::internal)?; route.operation.signatures.push(sig); } @@ -169,14 +170,16 @@ impl RPCProcessor { // If the private route public key is our node id, then this was sent via safety route to our node directly // so there will be no signatures to validate - let opt_pr_info = if private_route.public_key == self.routing_table.node_id() { + let (secret_key, safety_selection) = if private_route.public_key + == self.routing_table.node_id() + { // The private route was a stub // Return our secret key and an appropriate safety selection // // Note: it is important that we never respond with a safety route to questions that come - // in without a private route. Giving away a safety route when the node id is known is + // in without a private route. Giving away a safety route when the node id is known is // a privacy violation! - + // Get sequencing preference let sequencing = if detail .connection_descriptor @@ -187,65 +190,25 @@ impl RPCProcessor { } else { Sequencing::NoPreference }; - Some(( - self.routing_table.node_id_secret(), + ( + self.routing_table.node_id_secret(), SafetySelection::Unsafe(sequencing), - )) + ) } else { // Get sender id let sender_id = detail.envelope.get_sender_id(); // Look up the private route and ensure it's one in our spec store - let rss= self.routing_table.route_spec_store(); - let opt_signatures_valid = rss.with_route_spec_detail(&private_route.public_key, |rsd| { - // Ensure we have the right number of signatures - if routed_operation.signatures.len() != rsd.hops.len() - 1 { - // Wrong number of signatures - log_rpc!(debug "wrong number of signatures ({} should be {}) for routed operation on private route {}", routed_operation.signatures.len(), rsd.hops.len() - 1, private_route.public_key); - return None; - } - // Validate signatures to ensure the route was handled by the nodes and not messed with - for (hop_n, hop_public_key) in rsd.hops.iter().enumerate() { - // The last hop is not signed, as the whole packet is signed - if hop_n == routed_operation.signatures.len() { - // Verify the node we received the routed operation from is the last hop in our route - if *hop_public_key != sender_id { - log_rpc!(debug "received routed operation from the wrong hop ({} should be {}) on private route {}", hop_public_key.encode(), sender_id.encode(), private_route.public_key); - return None; - } - } else { - // Verify a signature for a hop node along the route - if let Err(e) = verify( - hop_public_key, - &routed_operation.data, - &routed_operation.signatures[hop_n], - ) { - log_rpc!(debug "failed to verify signature for hop {} at {} on private route {}", hop_n, hop_public_key, private_route.public_key); - return None; - } - } - } - // We got the correct signatures, return a key ans - Some(( - rsd.secret_key, - SafetySelection::Safe(SafetySpec { - preferred_route: Some(private_route.public_key), - hop_count: rsd.hops.len(), - stability: rsd.stability, - sequencing: rsd.sequencing, - }) - )) - }); - opt_signatures_valid.ok_or_else(|| { - RPCError::protocol("routed operation received on unallocated private route") - })? + let rss = self.routing_table.route_spec_store(); + rss.validate_signatures( + &private_route.public_key, + &routed_operation.signatures, + &routed_operation.data, + sender_id, + ) + .map_err(RPCError::protocol)? + .ok_or_else(|| RPCError::protocol("signatures did not validate for private route"))? }; - if opt_pr_info.is_none() { - return Err(RPCError::protocol( - "signatures did not validate for private route", - )); - } - let (secret_key, safety_selection) = opt_pr_info.unwrap(); // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret) // xxx: punish nodes that send messages that fail to decrypt eventually @@ -344,8 +307,13 @@ impl RPCProcessor { .await?; } else { // Private route is empty, process routed operation - self.process_routed_operation(detail, route.operation, &route.safety_route, &private_route) - .await?; + self.process_routed_operation( + detail, + route.operation, + &route.safety_route, + &private_route, + ) + .await?; } } else if blob_tag == 0 { // RouteHop @@ -410,8 +378,13 @@ impl RPCProcessor { .await?; } else { // No hops left, time to process the routed operation - self.process_routed_operation(detail, route.operation, &route.safety_route, private_route) - .await?; + self.process_routed_operation( + detail, + route.operation, + &route.safety_route, + private_route, + ) + .await?; } } } diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 72b5f2f3..167f9527 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -34,7 +34,8 @@ impl RPCProcessor { // Wait for receipt match eventual_value.await.take_value().unwrap() { - ReceiptEvent::ReturnedPrivate | ReceiptEvent::ReturnedInBand { inbound_noderef: _ } => { + ReceiptEvent::ReturnedPrivate { private_route: _ } + | ReceiptEvent::ReturnedInBand { inbound_noderef: _ } => { log_net!(debug "validate_dial_info receipt should be returned out-of-band".green()); Ok(false) } @@ -94,20 +95,26 @@ impl RPCProcessor { routing_domain, dial_info.clone(), ); - let will_validate_dial_info_filter = Box::new(move |_rti, e: &BucketEntryInner| { - if let Some(status) = &e.node_status(routing_domain) { - status.will_validate_dial_info() - } else { - true - } - }); - let filter = RoutingTable::combine_entry_filters( + let will_validate_dial_info_filter = Box::new( + move |rti: &RoutingTableInner, _k: DHTKey, v: Option>| { + let entry = v.unwrap(); + entry.with(rti, move |_rti, e| { + if let Some(status) = &e.node_status(routing_domain) { + status.will_validate_dial_info() + } else { + true + } + }) + }, + ) as RoutingTableEntryFilter; + + let filters = VecDeque::from([ outbound_dial_info_entry_filter, - will_validate_dial_info_filter, fuck this shit. do it tomorrow. - ); + will_validate_dial_info_filter, + ]); // Find nodes matching filter to redirect this to - let peers = routing_table.find_fast_public_nodes_filtered(node_count, filter); + let peers = routing_table.find_fast_public_nodes_filtered(node_count, filters); if peers.is_empty() { return Err(RPCError::internal(format!( "no peers able to reach dialinfo '{:?}'", diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 9f14bd68..16a7ef4f 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -647,7 +647,7 @@ impl NodeInfo { .as_ref() .map(|rpi| { let relay_ni = &rpi.signed_node_info.node_info; - for did in relay_ni.dial_info_detail_list { + for did in &relay_ni.dial_info_detail_list { match sequencing { Sequencing::NoPreference | Sequencing::PreferOrdered => return true, Sequencing::EnsureOrdered => { From ec58574a5e2478061abbdc7b74a53b59a830e844 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 1 Nov 2022 22:42:34 -0400 Subject: [PATCH 28/67] checkpoint --- .../src/routing_table/route_spec_store.rs | 15 +++- veilid-core/src/rpc_processor/mod.rs | 90 +++++++++++++++---- .../src/rpc_processor/rpc_find_node.rs | 38 +++++++- .../src/rpc_processor/rpc_node_info_update.rs | 2 +- .../src/rpc_processor/rpc_return_receipt.rs | 2 +- veilid-core/src/rpc_processor/rpc_route.rs | 4 +- veilid-core/src/rpc_processor/rpc_status.rs | 2 +- .../rpc_processor/rpc_validate_dial_info.rs | 2 +- 8 files changed, 124 insertions(+), 31 deletions(-) diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 12dddeed..844e5a21 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -208,8 +208,15 @@ impl RouteSpecStore { } pub async fn load(routing_table: RoutingTable) -> EyreResult { - let config = routing_table.network_manager().config(); - let c = config.get(); + let (max_route_hop_count, default_route_hop_count) = { + let config = routing_table.network_manager().config(); + let c = config.get(); + ( + c.network.rpc.max_route_hop_count as usize, + c.network.rpc.default_route_hop_count as usize, + ) + }; + // Get cbor blob from table store let table_store = routing_table.network_manager().table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; @@ -251,8 +258,8 @@ impl RouteSpecStore { let rss = RouteSpecStore { unlocked_inner: Arc::new(RouteSpecStoreUnlockedInner { - max_route_hop_count: c.network.rpc.max_route_hop_count.into(), - default_route_hop_count: c.network.rpc.default_route_hop_count.into(), + max_route_hop_count, + default_route_hop_count, routing_table, }), inner: Arc::new(Mutex::new(inner)), diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index e020aa45..ac21e054 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -52,8 +52,16 @@ struct RPCMessageHeaderDetailDirect { routing_domain: RoutingDomain, } +/// Header details for rpc messages received over only a safety route but not a private route #[derive(Debug, Clone)] -struct RPCMessageHeaderDetailPrivateRoute { +struct RPCMessageHeaderDetailSafetyRouted { + /// The sequencing used for this route + sequencing: Sequencing, +} + +/// Header details for rpc messages received over a private route +#[derive(Debug, Clone)] +struct RPCMessageHeaderDetailPrivateRouted { /// The private route we received the rpc over private_route: DHTKey, // The safety selection for replying to this private routed rpc @@ -63,7 +71,8 @@ struct RPCMessageHeaderDetailPrivateRoute { #[derive(Debug, Clone)] enum RPCMessageHeaderDetail { Direct(RPCMessageHeaderDetailDirect), - PrivateRoute(RPCMessageHeaderDetailPrivateRoute), + SafetyRouted(RPCMessageHeaderDetailSafetyRouted), + PrivateRouted(RPCMessageHeaderDetailPrivateRouted), } /// The decoded header of an RPC message @@ -766,10 +775,11 @@ impl RPCProcessor { // Parse out the header detail from the question let detail = match &request.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, - RPCMessageHeaderDetail::PrivateRoute(_) => { + RPCMessageHeaderDetail::SafetyRouted(_) + | RPCMessageHeaderDetail::PrivateRouted(_) => { // If this was sent via a private route, we don't know what the sender was, so drop this return NetworkResult::invalid_message( - "not responding directly to question from private route", + "can't respond directly to non-direct question", ); } }; @@ -789,20 +799,29 @@ impl RPCProcessor { } } RespondTo::PrivateRoute(pr) => { - let detail = match &request.header.detail { + match &request.header.detail { RPCMessageHeaderDetail::Direct(_) => { - // If this was sent directly, don't respond to a private route as this could give away this node's safety routes + // If this was sent directly, we should only ever respond directly return NetworkResult::invalid_message( "not responding to private route from direct question", ); } - RPCMessageHeaderDetail::PrivateRoute(detail) => detail, - }; - - NetworkResult::value(Destination::private_route( - pr.clone(), - detail.safety_selection.clone(), - )) + RPCMessageHeaderDetail::SafetyRouted(detail) => { + // If this was sent via a safety route, but no received over our private route, don't respond with a safety route, + // it would give away which safety routes belong to this node + NetworkResult::value(Destination::private_route( + pr.clone(), + SafetySelection::Unsafe(detail.sequencing), + )) + } + RPCMessageHeaderDetail::PrivateRouted(detail) => { + // If this was received over our private route, it's okay to respond to a private route via our safety route + NetworkResult::value(Destination::private_route( + pr.clone(), + detail.safety_selection.clone(), + )) + } + } } } } @@ -916,7 +935,7 @@ impl RPCProcessor { opt_sender_nr, } } - RPCMessageHeaderDetail::PrivateRoute(_) => { + RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { // Decode the RPC message let operation = { let reader = capnp::message::Reader::new(encoded_msg.data, Default::default()); @@ -1047,7 +1066,38 @@ impl RPCProcessor { } #[instrument(level = "trace", skip(self, body), err)] - pub fn enqueue_private_route_message( + pub fn enqueue_safety_routed_message( + &self, xxx keep pushing this through + private_route: DHTKey, + safety_selection: SafetySelection, + body: Vec, + ) -> EyreResult<()> { + let msg = RPCMessageEncoded { + header: RPCMessageHeader { + detail: RPCMessageHeaderDetail::PrivateRouted( + RPCMessageHeaderDetailPrivateRouted { + private_route, + safety_selection, + }, + ), + timestamp: intf::get_timestamp(), + body_len: body.len() as u64, + }, + data: RPCMessageData { contents: body }, + }; + let send_channel = { + let inner = self.inner.lock(); + inner.send_channel.as_ref().unwrap().clone() + }; + let span_id = Span::current().id(); + send_channel + .try_send((span_id, msg)) + .wrap_err("failed to enqueue received RPC message")?; + Ok(()) + } + + #[instrument(level = "trace", skip(self, body), err)] + pub fn enqueue_private_routed_message( &self, private_route: DHTKey, safety_selection: SafetySelection, @@ -1055,10 +1105,12 @@ impl RPCProcessor { ) -> EyreResult<()> { let msg = RPCMessageEncoded { header: RPCMessageHeader { - detail: RPCMessageHeaderDetail::PrivateRoute(RPCMessageHeaderDetailPrivateRoute { - private_route, - safety_selection, - }), + detail: RPCMessageHeaderDetail::PrivateRouted( + RPCMessageHeaderDetailPrivateRouted { + private_route, + safety_selection, + }, + ), timestamp: intf::get_timestamp(), body_len: body.len() as u64, }, diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index 65dde614..493b4d5c 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -1,14 +1,31 @@ use super::*; impl RPCProcessor { - // Send FindNodeQ RPC request, receive FindNodeA answer - // Can be sent via all methods including relays and routes + /// Send FindNodeQ RPC request, receive FindNodeA answer + /// Can be sent via all methods including relays + /// Safety routes may be used, but never private routes. + /// Because this leaks information about the identity of the node itself, + /// replying to this request received over a private route will leak + /// the identity of the node and defeat the private route. #[instrument(level = "trace", skip(self), ret, err)] pub async fn rpc_call_find_node( self, dest: Destination, key: DHTKey, ) -> Result>>, RPCError> { + // Ensure destination never has a private route + if matches!( + dest, + Destination::PrivateRoute { + private_route: _, + safety_selection: _ + } + ) { + return Err(RPCError::internal( + "Never send find node requests over private routes", + )); + } + let find_node_q_detail = RPCQuestionDetail::FindNodeQ(RPCOperationFindNodeQ { node_id: key }); let find_node_q = RPCQuestion::new(RespondTo::Sender, find_node_q_detail); @@ -51,6 +68,23 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] pub(crate) async fn process_find_node_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + // Ensure this never came over a private route + match msg.header.detail { + RPCMessageHeaderDetail::Direct(_) => todo!(), + RPCMessageHeaderDetail::PrivateRouted(_) => todo!(), + } + if matches!( + dest, + Destination::PrivateRoute { + private_route: _, + safety_selection: _ + } + ) { + return Err(RPCError::internal( + "Never send find node requests over private routes", + )); + } + // Get the question let find_node_q = match msg.operation.kind() { RPCOperationKind::Question(q) => match q.detail() { diff --git a/veilid-core/src/rpc_processor/rpc_node_info_update.rs b/veilid-core/src/rpc_processor/rpc_node_info_update.rs index 0e317d9c..d14877b1 100644 --- a/veilid-core/src/rpc_processor/rpc_node_info_update.rs +++ b/veilid-core/src/rpc_processor/rpc_node_info_update.rs @@ -33,7 +33,7 @@ impl RPCProcessor { pub(crate) async fn process_node_info_update(&self, msg: RPCMessage) -> Result<(), RPCError> { let detail = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, - RPCMessageHeaderDetail::PrivateRoute(_) => { + RPCMessageHeaderDetail::PrivateRouted(_) => { return Err(RPCError::protocol("node_info_update must be direct")); } }; diff --git a/veilid-core/src/rpc_processor/rpc_return_receipt.rs b/veilid-core/src/rpc_processor/rpc_return_receipt.rs index 3087dd0f..09d8f309 100644 --- a/veilid-core/src/rpc_processor/rpc_return_receipt.rs +++ b/veilid-core/src/rpc_processor/rpc_return_receipt.rs @@ -42,7 +42,7 @@ impl RPCProcessor { .await => {} ); } - RPCMessageHeaderDetail::PrivateRoute(detail) => { + RPCMessageHeaderDetail::PrivateRouted(detail) => { network_result_value_or_log!(debug network_manager .handle_private_receipt(receipt, detail.private_route) diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 6c3f4384..63a56482 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -227,7 +227,7 @@ impl RPCProcessor { ))?; // Pass message to RPC system - self.enqueue_private_route_message(private_route.public_key, safety_selection, body) + self.enqueue_private_routed_message(private_route.public_key, safety_selection, body) .map_err(RPCError::internal)?; Ok(()) @@ -238,7 +238,7 @@ impl RPCProcessor { // Get header detail, must be direct and not inside a route itself let detail = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, - RPCMessageHeaderDetail::PrivateRoute(_) => { + RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { return Err(RPCError::protocol( "route operation can not be inside route", )) diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index feebaa5c..af02a6c3 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -105,7 +105,7 @@ impl RPCProcessor { pub(crate) async fn process_status_q(&self, msg: RPCMessage) -> Result<(), RPCError> { let detail = match &msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, - RPCMessageHeaderDetail::PrivateRoute(_) => { + RPCMessageHeaderDetail::PrivateRouted(_) => { return Err(RPCError::protocol("status_q must be direct")); } }; diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 167f9527..11de3e23 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -57,7 +57,7 @@ impl RPCProcessor { pub(crate) async fn process_validate_dial_info(&self, msg: RPCMessage) -> Result<(), RPCError> { let detail = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, - RPCMessageHeaderDetail::PrivateRoute(_) => { + RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { return Err(RPCError::protocol("validate_dial_info must be direct")); } }; From 92b22d5af5a8f383bd767b280f91acdf3d940945 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 2 Nov 2022 15:36:01 -0400 Subject: [PATCH 29/67] private routing --- veilid-core/proto/veilid.capnp | 8 +- veilid-core/src/network_manager/mod.rs | 26 +- .../native/network_class_discovery.rs | 4 +- veilid-core/src/network_manager/tasks.rs | 10 +- veilid-core/src/receipt_manager.rs | 4 + .../src/routing_table/route_spec_store.rs | 6 +- .../coders/operations/operation_status.rs | 51 +++- .../src/rpc_processor/coders/sender_info.rs | 29 +- veilid-core/src/rpc_processor/mod.rs | 27 +- .../src/rpc_processor/rpc_find_node.rs | 21 +- .../src/rpc_processor/rpc_node_info_update.rs | 2 +- .../src/rpc_processor/rpc_return_receipt.rs | 7 + veilid-core/src/rpc_processor/rpc_route.rs | 169 +++++++---- veilid-core/src/rpc_processor/rpc_signal.rs | 24 +- veilid-core/src/rpc_processor/rpc_status.rs | 270 ++++++++++++------ .../rpc_processor/rpc_validate_dial_info.rs | 3 +- veilid-core/src/veilid_api/debug.rs | 3 +- veilid-core/src/veilid_api/mod.rs | 7 +- 18 files changed, 426 insertions(+), 245 deletions(-) diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 97904483..b3c19e91 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -249,7 +249,7 @@ struct SignedNodeInfo { } struct SenderInfo { - socketAddress @0 :SocketAddress; # socket address was available for peer + socketAddress @0 :SocketAddress; # socket address that for the sending peer } struct PeerInfo { @@ -265,12 +265,12 @@ struct RoutedOperation { } struct OperationStatusQ { - nodeStatus @0 :NodeStatus; # node status update about the statusq sender + nodeStatus @0 :NodeStatus; # Optional: node status update about the statusq sender } struct OperationStatusA { - nodeStatus @0 :NodeStatus; # returned node status - senderInfo @1 :SenderInfo; # info about StatusQ sender from the perspective of the replier + nodeStatus @0 :NodeStatus; # Optional: returned node status + senderInfo @1 :SenderInfo; # Optional: info about StatusQ sender from the perspective of the replier } struct OperationValidateDialInfo { diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 8ae5256f..a7166f5e 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -783,6 +783,26 @@ impl NetworkManager { .await } + /// Process a received safety receipt + #[instrument(level = "trace", skip(self, receipt_data), ret)] + pub async fn handle_safety_receipt>( + &self, + receipt_data: R, + ) -> NetworkResult<()> { + let receipt_manager = self.receipt_manager(); + + let receipt = match Receipt::from_signed_data(receipt_data.as_ref()) { + Err(e) => { + return NetworkResult::invalid_message(e.to_string()); + } + Ok(v) => v, + }; + + receipt_manager + .handle_receipt(receipt, ReceiptReturned::Safety) + .await + } + /// Process a received private receipt #[instrument(level = "trace", skip(self, receipt_data), ret)] pub async fn handle_private_receipt>( @@ -1025,7 +1045,8 @@ impl NetworkManager { // Wait for the return receipt let inbound_nr = match eventual_value.await.take_value().unwrap() { ReceiptEvent::ReturnedPrivate { private_route: _ } - | ReceiptEvent::ReturnedOutOfBand => { + | ReceiptEvent::ReturnedOutOfBand + | ReceiptEvent::ReturnedSafety => { return Ok(NetworkResult::invalid_message( "reverse connect receipt should be returned in-band", )); @@ -1127,7 +1148,8 @@ impl NetworkManager { // Wait for the return receipt let inbound_nr = match eventual_value.await.take_value().unwrap() { ReceiptEvent::ReturnedPrivate { private_route: _ } - | ReceiptEvent::ReturnedOutOfBand => { + | ReceiptEvent::ReturnedOutOfBand + | ReceiptEvent::ReturnedSafety => { return Ok(NetworkResult::invalid_message( "hole punch receipt should be returned in-band", )); diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/network_class_discovery.rs index cd4989ce..c24ec733 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/network_class_discovery.rs @@ -79,7 +79,7 @@ impl DiscoveryContext { async fn request_public_address(&self, node_ref: NodeRef) -> Option { let rpc = self.routing_table.rpc_processor(); - let res = network_result_value_or_log!(debug match rpc.rpc_call_status(node_ref.clone()).await { + let res = network_result_value_or_log!(debug match rpc.rpc_call_status(Destination::direct(node_ref.clone())).await { Ok(v) => v, Err(e) => { log_net!(error @@ -98,7 +98,7 @@ impl DiscoveryContext { node_ref, res.answer ); - res.answer.socket_address + res.answer.map(|si| si.socket_address) } // find fast peers with a particular address type, and ask them to tell us what our external address is diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index ddcc065d..a12d6018 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -343,7 +343,7 @@ impl NetworkManager { &self, cur_ts: u64, unord: &mut FuturesUnordered< - SendPinBoxFuture>, RPCError>>, + SendPinBoxFuture>>, RPCError>>, >, ) -> EyreResult<()> { let rpc = self.rpc_processor(); @@ -394,7 +394,7 @@ impl NetworkManager { nr.filtered_clone(NodeRefFilter::new().with_dial_info_filter(dif)); log_net!("--> Keepalive ping to {:?}", nr_filtered); unord.push( - async move { rpc.rpc_call_status(nr_filtered).await } + async move { rpc.rpc_call_status(Destination::direct(nr_filtered)).await } .instrument(Span::current()) .boxed(), ); @@ -408,7 +408,7 @@ impl NetworkManager { if !did_pings { let rpc = rpc.clone(); unord.push( - async move { rpc.rpc_call_status(nr).await } + async move { rpc.rpc_call_status(Destination::direct(nr)).await } .instrument(Span::current()) .boxed(), ); @@ -425,7 +425,7 @@ impl NetworkManager { &self, cur_ts: u64, unord: &mut FuturesUnordered< - SendPinBoxFuture>, RPCError>>, + SendPinBoxFuture>>, RPCError>>, >, ) -> EyreResult<()> { let rpc = self.rpc_processor(); @@ -440,7 +440,7 @@ impl NetworkManager { // Just do a single ping with the best protocol for all the nodes unord.push( - async move { rpc.rpc_call_status(nr).await } + async move { rpc.rpc_call_status(Destination::direct(nr)).await } .instrument(Span::current()) .boxed(), ); diff --git a/veilid-core/src/receipt_manager.rs b/veilid-core/src/receipt_manager.rs index 4ea815e8..3e9971a3 100644 --- a/veilid-core/src/receipt_manager.rs +++ b/veilid-core/src/receipt_manager.rs @@ -11,6 +11,7 @@ use xx::*; pub enum ReceiptEvent { ReturnedOutOfBand, ReturnedInBand { inbound_noderef: NodeRef }, + ReturnedSafety, ReturnedPrivate { private_route: DHTKey }, Expired, Cancelled, @@ -20,6 +21,7 @@ pub enum ReceiptEvent { pub enum ReceiptReturned { OutOfBand, InBand { inbound_noderef: NodeRef }, + Safety, Private { private_route: DHTKey }, } @@ -412,6 +414,7 @@ impl ReceiptManager { match receipt_returned { ReceiptReturned::OutOfBand => "OutOfBand".to_owned(), ReceiptReturned::InBand { ref inbound_noderef } => format!("InBand({})", inbound_noderef), + ReceiptReturned::Safety => "Safety".to_owned(), ReceiptReturned::Private { ref private_route } => format!("Private({})", private_route), }, if extra_data.is_empty() { @@ -445,6 +448,7 @@ impl ReceiptManager { // Get the receipt event to return let receipt_event = match receipt_returned { ReceiptReturned::OutOfBand => ReceiptEvent::ReturnedOutOfBand, + ReceiptReturned::Safety => ReceiptEvent::ReturnedSafety, ReceiptReturned::InBand { ref inbound_noderef, } => ReceiptEvent::ReturnedInBand { diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 844e5a21..d99a5620 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -626,7 +626,7 @@ impl RouteSpecStore { signatures: &[DHTSignature], data: &[u8], last_hop_id: DHTKey, - ) -> EyreResult> { + ) -> EyreResult> { let inner = &*self.inner.lock(); let rsd = Self::detail(inner, &public_key).ok_or_else(|| eyre!("route does not exist"))?; @@ -656,12 +656,12 @@ impl RouteSpecStore { // We got the correct signatures, return a key ans Ok(Some(( rsd.secret_key, - SafetySelection::Safe(SafetySpec { + SafetySpec { preferred_route: Some(*public_key), hop_count: rsd.hops.len(), stability: rsd.stability, sequencing: rsd.sequencing, - }), + }, ))) } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_status.rs b/veilid-core/src/rpc_processor/coders/operations/operation_status.rs index 7f4d3da7..71ed2117 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_status.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_status.rs @@ -3,42 +3,59 @@ use rpc_processor::*; #[derive(Debug, Clone)] pub struct RPCOperationStatusQ { - pub node_status: NodeStatus, + pub node_status: Option, } impl RPCOperationStatusQ { pub fn decode( reader: &veilid_capnp::operation_status_q::Reader, ) -> Result { - let ns_reader = reader.get_node_status().map_err(RPCError::protocol)?; - let node_status = decode_node_status(&ns_reader)?; + let node_status = if reader.has_node_status() { + let ns_reader = reader.get_node_status().map_err(RPCError::protocol)?; + let node_status = decode_node_status(&ns_reader)?; + Some(node_status) + } else { + None + }; Ok(RPCOperationStatusQ { node_status }) } pub fn encode( &self, builder: &mut veilid_capnp::operation_status_q::Builder, ) -> Result<(), RPCError> { - let mut ns_builder = builder.reborrow().init_node_status(); - encode_node_status(&self.node_status, &mut ns_builder)?; + if let Some(ns) = &self.node_status { + let mut ns_builder = builder.reborrow().init_node_status(); + encode_node_status(&ns, &mut ns_builder)?; + } Ok(()) } } #[derive(Debug, Clone)] pub struct RPCOperationStatusA { - pub node_status: NodeStatus, - pub sender_info: SenderInfo, + pub node_status: Option, + pub sender_info: Option, } impl RPCOperationStatusA { pub fn decode( reader: &veilid_capnp::operation_status_a::Reader, ) -> Result { - let ns_reader = reader.get_node_status().map_err(RPCError::protocol)?; - let node_status = decode_node_status(&ns_reader)?; + let node_status = if reader.has_node_status() { + let ns_reader = reader.get_node_status().map_err(RPCError::protocol)?; + let node_status = decode_node_status(&ns_reader)?; + Some(node_status) + } else { + None + }; - let si_reader = reader.get_sender_info().map_err(RPCError::protocol)?; - let sender_info = decode_sender_info(&si_reader)?; + let sender_info = if reader.has_sender_info() { + let si_reader = reader.get_sender_info().map_err(RPCError::protocol)?; + let sender_info = decode_sender_info(&si_reader)?; + Some(sender_info) + } else { + None + }; Ok(RPCOperationStatusA { node_status, @@ -49,10 +66,14 @@ impl RPCOperationStatusA { &self, builder: &mut veilid_capnp::operation_status_a::Builder, ) -> Result<(), RPCError> { - let mut ns_builder = builder.reborrow().init_node_status(); - encode_node_status(&self.node_status, &mut ns_builder)?; - let mut si_builder = builder.reborrow().init_sender_info(); - encode_sender_info(&self.sender_info, &mut si_builder)?; + if let Some(ns) = &self.node_status { + let mut ns_builder = builder.reborrow().init_node_status(); + encode_node_status(&ns, &mut ns_builder)?; + } + if let Some(si) = &self.sender_info { + let mut si_builder = builder.reborrow().init_sender_info(); + encode_sender_info(&si, &mut si_builder)?; + } Ok(()) } } diff --git a/veilid-core/src/rpc_processor/coders/sender_info.rs b/veilid-core/src/rpc_processor/coders/sender_info.rs index ffea5be8..5fbea344 100644 --- a/veilid-core/src/rpc_processor/coders/sender_info.rs +++ b/veilid-core/src/rpc_processor/coders/sender_info.rs @@ -5,30 +5,21 @@ pub fn encode_sender_info( sender_info: &SenderInfo, builder: &mut veilid_capnp::sender_info::Builder, ) -> Result<(), RPCError> { - if let Some(socket_address) = &sender_info.socket_address { - let mut sab = builder.reborrow().init_socket_address(); - encode_socket_address(socket_address, &mut sab)?; - } + let mut sab = builder.reborrow().init_socket_address(); + encode_socket_address(&sender_info.socket_address, &mut sab)?; Ok(()) } pub fn decode_sender_info( reader: &veilid_capnp::sender_info::Reader, ) -> Result { - if !reader.has_socket_address() { - return Err(RPCError::internal("invalid socket address type")); - } - let socket_address = if reader.has_socket_address() { - Some(decode_socket_address( - &reader - .reborrow() - .get_socket_address() - .map_err(RPCError::map_internal( - "invalid socket address in sender_info", - ))?, - )?) - } else { - None - }; + let sa_reader = reader + .reborrow() + .get_socket_address() + .map_err(RPCError::map_internal( + "invalid socket address in sender_info", + ))?; + let socket_address = decode_socket_address(&sa_reader)?; + Ok(SenderInfo { socket_address }) } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index ac21e054..4d43c1cc 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -25,6 +25,7 @@ pub use coders::*; pub use destination::*; pub use operation_waiter::*; pub use rpc_error::*; +pub use rpc_status::*; use super::*; use crate::crypto::*; @@ -64,8 +65,8 @@ struct RPCMessageHeaderDetailSafetyRouted { struct RPCMessageHeaderDetailPrivateRouted { /// The private route we received the rpc over private_route: DHTKey, - // The safety selection for replying to this private routed rpc - safety_selection: SafetySelection, + // The safety spec for replying to this private routed rpc + safety_spec: SafetySpec, } #[derive(Debug, Clone)] @@ -807,7 +808,7 @@ impl RPCProcessor { ); } RPCMessageHeaderDetail::SafetyRouted(detail) => { - // If this was sent via a safety route, but no received over our private route, don't respond with a safety route, + // If this was sent via a safety route, but not received over our private route, don't respond with a safety route, // it would give away which safety routes belong to this node NetworkResult::value(Destination::private_route( pr.clone(), @@ -818,7 +819,7 @@ impl RPCProcessor { // If this was received over our private route, it's okay to respond to a private route via our safety route NetworkResult::value(Destination::private_route( pr.clone(), - detail.safety_selection.clone(), + SafetySelection::Safe(detail.safety_spec.clone()), )) } } @@ -1067,19 +1068,15 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, body), err)] pub fn enqueue_safety_routed_message( - &self, xxx keep pushing this through - private_route: DHTKey, - safety_selection: SafetySelection, + &self, + sequencing: Sequencing, body: Vec, ) -> EyreResult<()> { let msg = RPCMessageEncoded { header: RPCMessageHeader { - detail: RPCMessageHeaderDetail::PrivateRouted( - RPCMessageHeaderDetailPrivateRouted { - private_route, - safety_selection, - }, - ), + detail: RPCMessageHeaderDetail::SafetyRouted(RPCMessageHeaderDetailSafetyRouted { + sequencing, + }), timestamp: intf::get_timestamp(), body_len: body.len() as u64, }, @@ -1100,7 +1097,7 @@ impl RPCProcessor { pub fn enqueue_private_routed_message( &self, private_route: DHTKey, - safety_selection: SafetySelection, + safety_spec: SafetySpec, body: Vec, ) -> EyreResult<()> { let msg = RPCMessageEncoded { @@ -1108,7 +1105,7 @@ impl RPCProcessor { detail: RPCMessageHeaderDetail::PrivateRouted( RPCMessageHeaderDetailPrivateRouted { private_route, - safety_selection, + safety_spec, }, ), timestamp: intf::get_timestamp(), diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index 493b4d5c..42d7c4d1 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -68,21 +68,14 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] pub(crate) async fn process_find_node_q(&self, msg: RPCMessage) -> Result<(), RPCError> { - // Ensure this never came over a private route - match msg.header.detail { - RPCMessageHeaderDetail::Direct(_) => todo!(), - RPCMessageHeaderDetail::PrivateRouted(_) => todo!(), - } - if matches!( - dest, - Destination::PrivateRoute { - private_route: _, - safety_selection: _ + // Ensure this never came over a private route, safety route is okay though + match &msg.header.detail { + RPCMessageHeaderDetail::Direct(_) | RPCMessageHeaderDetail::SafetyRouted(_) => {} + RPCMessageHeaderDetail::PrivateRouted(_) => { + return Err(RPCError::protocol( + "not processing find node request over private route", + )) } - ) { - return Err(RPCError::internal( - "Never send find node requests over private routes", - )); } // Get the question diff --git a/veilid-core/src/rpc_processor/rpc_node_info_update.rs b/veilid-core/src/rpc_processor/rpc_node_info_update.rs index d14877b1..c5ec9f86 100644 --- a/veilid-core/src/rpc_processor/rpc_node_info_update.rs +++ b/veilid-core/src/rpc_processor/rpc_node_info_update.rs @@ -33,7 +33,7 @@ impl RPCProcessor { pub(crate) async fn process_node_info_update(&self, msg: RPCMessage) -> Result<(), RPCError> { let detail = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, - RPCMessageHeaderDetail::PrivateRouted(_) => { + RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { return Err(RPCError::protocol("node_info_update must be direct")); } }; diff --git a/veilid-core/src/rpc_processor/rpc_return_receipt.rs b/veilid-core/src/rpc_processor/rpc_return_receipt.rs index 09d8f309..c4ae3b4c 100644 --- a/veilid-core/src/rpc_processor/rpc_return_receipt.rs +++ b/veilid-core/src/rpc_processor/rpc_return_receipt.rs @@ -42,6 +42,13 @@ impl RPCProcessor { .await => {} ); } + RPCMessageHeaderDetail::SafetyRouted(_) => { + network_result_value_or_log!(debug + network_manager + .handle_safety_receipt(receipt) + .await => {} + ); + } RPCMessageHeaderDetail::PrivateRouted(detail) => { network_result_value_or_log!(debug network_manager diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 63a56482..257a2b71 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -148,8 +148,102 @@ impl RPCProcessor { Ok(()) } + /// Process a routed operation that came in over a safety route but no private route + /// + /// Note: it is important that we never respond with a safety route to questions that come + /// in without a private route. Giving away a safety route when the node id is known is + /// a privacy violation! #[instrument(level = "trace", skip_all, err)] - async fn process_routed_operation( + fn process_safety_routed_operation( + &self, + detail: RPCMessageHeaderDetailDirect, + routed_operation: RoutedOperation, + safety_route: &SafetyRoute, + ) -> Result<(), RPCError> { + // Get sequencing preference + let sequencing = if detail + .connection_descriptor + .protocol_type() + .is_connection_oriented() + { + Sequencing::EnsureOrdered + } else { + Sequencing::NoPreference + }; + + // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret) + // xxx: punish nodes that send messages that fail to decrypt eventually? How to do this for safety routes? + let node_id_secret = self.routing_table.node_id_secret(); + let dh_secret = self + .crypto + .cached_dh(&safety_route.public_key, &node_id_secret) + .map_err(RPCError::protocol)?; + let body = Crypto::decrypt_aead( + &routed_operation.data, + &routed_operation.nonce, + &dh_secret, + None, + ) + .map_err(RPCError::map_internal( + "decryption of routed operation failed", + ))?; + + // Pass message to RPC system + self.enqueue_safety_routed_message(sequencing, body) + .map_err(RPCError::internal)?; + + Ok(()) + } + + /// Process a routed operation that came in over both a safety route and a private route + #[instrument(level = "trace", skip_all, err)] + fn process_private_routed_operation( + &self, + detail: RPCMessageHeaderDetailDirect, + routed_operation: RoutedOperation, + safety_route: &SafetyRoute, + private_route: &PrivateRoute, + ) -> Result<(), RPCError> { + // Get sender id + let sender_id = detail.envelope.get_sender_id(); + + // Look up the private route and ensure it's one in our spec store + let rss = self.routing_table.route_spec_store(); + let (secret_key, safety_spec) = rss + .validate_signatures( + &private_route.public_key, + &routed_operation.signatures, + &routed_operation.data, + sender_id, + ) + .map_err(RPCError::protocol)? + .ok_or_else(|| RPCError::protocol("signatures did not validate for private route"))?; + + // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret) + // xxx: punish nodes that send messages that fail to decrypt eventually. How to do this for private routes? + let dh_secret = self + .crypto + .cached_dh(&safety_route.public_key, &secret_key) + .map_err(RPCError::protocol)?; + let body = Crypto::decrypt_aead( + &routed_operation.data, + &routed_operation.nonce, + &dh_secret, + None, + ) + .map_err(RPCError::map_internal( + "decryption of routed operation failed", + ))?; + + // Pass message to RPC system + self.enqueue_private_routed_message(private_route.public_key, safety_spec, body) + .map_err(RPCError::internal)?; + + Ok(()) + } + + #[instrument(level = "trace", skip_all, err)] + fn process_routed_operation( &self, detail: RPCMessageHeaderDetailDirect, routed_operation: RoutedOperation, @@ -170,67 +264,18 @@ impl RPCProcessor { // If the private route public key is our node id, then this was sent via safety route to our node directly // so there will be no signatures to validate - let (secret_key, safety_selection) = if private_route.public_key - == self.routing_table.node_id() - { + if private_route.public_key == self.routing_table.node_id() { // The private route was a stub - // Return our secret key and an appropriate safety selection - // - // Note: it is important that we never respond with a safety route to questions that come - // in without a private route. Giving away a safety route when the node id is known is - // a privacy violation! - - // Get sequencing preference - let sequencing = if detail - .connection_descriptor - .protocol_type() - .is_connection_oriented() - { - Sequencing::EnsureOrdered - } else { - Sequencing::NoPreference - }; - ( - self.routing_table.node_id_secret(), - SafetySelection::Unsafe(sequencing), - ) + self.process_safety_routed_operation(detail, routed_operation, safety_route) } else { - // Get sender id - let sender_id = detail.envelope.get_sender_id(); - - // Look up the private route and ensure it's one in our spec store - let rss = self.routing_table.route_spec_store(); - rss.validate_signatures( - &private_route.public_key, - &routed_operation.signatures, - &routed_operation.data, - sender_id, + // Both safety and private routes used, should reply with a safety route + self.process_private_routed_operation( + detail, + routed_operation, + safety_route, + private_route, ) - .map_err(RPCError::protocol)? - .ok_or_else(|| RPCError::protocol("signatures did not validate for private route"))? - }; - - // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret) - // xxx: punish nodes that send messages that fail to decrypt eventually - let dh_secret = self - .crypto - .cached_dh(&safety_route.public_key, &secret_key) - .map_err(RPCError::protocol)?; - let body = Crypto::decrypt_aead( - &routed_operation.data, - &routed_operation.nonce, - &dh_secret, - None, - ) - .map_err(RPCError::map_internal( - "decryption of routed operation failed", - ))?; - - // Pass message to RPC system - self.enqueue_private_routed_message(private_route.public_key, safety_selection, body) - .map_err(RPCError::internal)?; - - Ok(()) + } } #[instrument(level = "trace", skip(self, msg), err)] @@ -312,8 +357,7 @@ impl RPCProcessor { route.operation, &route.safety_route, &private_route, - ) - .await?; + )?; } } else if blob_tag == 0 { // RouteHop @@ -383,8 +427,7 @@ impl RPCProcessor { route.operation, &route.safety_route, private_route, - ) - .await?; + )?; } } } diff --git a/veilid-core/src/rpc_processor/rpc_signal.rs b/veilid-core/src/rpc_processor/rpc_signal.rs index 63fbcb79..0180fb6c 100644 --- a/veilid-core/src/rpc_processor/rpc_signal.rs +++ b/veilid-core/src/rpc_processor/rpc_signal.rs @@ -2,13 +2,26 @@ use super::*; impl RPCProcessor { // Sends a unidirectional signal to a node - // Can be sent via all methods including relays and routes + // Can be sent via relays but not routes. For routed 'signal' like capabilities, use AppMessage. #[instrument(level = "trace", skip(self), ret, err)] pub async fn rpc_call_signal( self, dest: Destination, signal_info: SignalInfo, ) -> Result, RPCError> { + // Ensure destination never has a private route + if matches!( + dest, + Destination::PrivateRoute { + private_route: _, + safety_selection: _ + } + ) { + return Err(RPCError::internal( + "Never send signal requests over private routes", + )); + } + let signal = RPCOperationSignal { signal_info }; let statement = RPCStatement::new(RPCStatementDetail::Signal(signal)); @@ -20,6 +33,15 @@ impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] pub(crate) async fn process_signal(&self, msg: RPCMessage) -> Result<(), RPCError> { + // Can't allow anything other than direct packets here, as handling reverse connections + // or anything like via signals over private routes would deanonymize the route + match &msg.header.detail { + RPCMessageHeaderDetail::Direct(_) => {} + RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { + return Err(RPCError::protocol("signal must be direct")); + } + }; + // Get the statement let signal = match msg.operation.into_kind() { RPCOperationKind::Statement(s) => match s.into_detail() { diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index af02a6c3..ab8b83d8 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -1,30 +1,79 @@ use super::*; +#[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Default)] +pub struct SenderInfo { + pub socket_address: SocketAddress, +} + impl RPCProcessor { // Send StatusQ RPC request, receive StatusA answer - // Can be sent via relays, but not via routes + // Can be sent via relays or routes, but will have less information via routes + // sender: + // unsafe -> node status + // safe -> nothing + // receiver: + // direct -> node status + sender info + // safety -> node status + // private -> nothing #[instrument(level = "trace", skip(self), ret, err)] pub async fn rpc_call_status( self, - peer: NodeRef, - ) -> Result>, RPCError> { - let routing_domain = match peer.best_routing_domain() { - Some(rd) => rd, - None => { - return Ok(NetworkResult::no_connection_other( - "no routing domain for peer", - )) + dest: Destination, + ) -> Result>>, RPCError> { + let (opt_target_nr, routing_domain, node_status) = match dest.get_safety_selection() { + SafetySelection::Unsafe(_) => { + let (opt_target_nr, routing_domain) = match &dest { + Destination::Direct { + target, + safety_selection: _, + } => { + let routing_domain = match target.best_routing_domain() { + Some(rd) => rd, + None => { + return Ok(NetworkResult::no_connection_other( + "no routing domain for target", + )) + } + }; + (Some(target.clone()), routing_domain) + } + Destination::Relay { + relay, + target, + safety_selection: _, + } => { + let opt_target_nr = self.routing_table.lookup_node_ref(*target); + let routing_domain = match relay.best_routing_domain() { + Some(rd) => rd, + None => { + return Ok(NetworkResult::no_connection_other( + "no routing domain for peer", + )) + } + }; + (opt_target_nr, routing_domain) + } + Destination::PrivateRoute { + private_route: _, + safety_selection: _, + } => (None, RoutingDomain::PublicInternet), + }; + + let node_status = Some(self.network_manager().generate_node_status(routing_domain)); + (opt_target_nr, routing_domain, node_status) + } + SafetySelection::Safe(_) => { + let routing_domain = RoutingDomain::PublicInternet; + let node_status = None; + (None, routing_domain, node_status) } }; - let node_status = self.network_manager().generate_node_status(routing_domain); + let status_q = RPCOperationStatusQ { node_status }; let question = RPCQuestion::new(RespondTo::Sender, RPCQuestionDetail::StatusQ(status_q)); // Send the info request - let waitable_reply = network_result_try!( - self.question(Destination::direct(peer.clone()), question) - .await? - ); + let waitable_reply = network_result_try!(self.question(dest.clone(), question).await?); // Note what kind of ping this was and to what peer scope let send_data_kind = waitable_reply.send_data_kind; @@ -45,74 +94,90 @@ impl RPCProcessor { }; // Ensure the returned node status is the kind for the routing domain we asked for - match routing_domain { - RoutingDomain::PublicInternet => { - if !matches!(status_a.node_status, NodeStatus::PublicInternet(_)) { - return Ok(NetworkResult::invalid_message( - "node status doesn't match PublicInternet routing domain", - )); - } - } - RoutingDomain::LocalNetwork => { - if !matches!(status_a.node_status, NodeStatus::LocalNetwork(_)) { - return Ok(NetworkResult::invalid_message( - "node status doesn't match LocalNetwork routing domain", - )); + if let Some(target_nr) = opt_target_nr { + if let Some(node_status) = status_a.node_status { + match routing_domain { + RoutingDomain::PublicInternet => { + if !matches!(node_status, NodeStatus::PublicInternet(_)) { + return Ok(NetworkResult::invalid_message( + "node status doesn't match PublicInternet routing domain", + )); + } + } + RoutingDomain::LocalNetwork => { + if !matches!(node_status, NodeStatus::LocalNetwork(_)) { + return Ok(NetworkResult::invalid_message( + "node status doesn't match LocalNetwork routing domain", + )); + } + } } + + // Update latest node status in routing table + target_nr.update_node_status(node_status); } } - // Update latest node status in routing table - peer.update_node_status(status_a.node_status); - // Report sender_info IP addresses to network manager // Don't need to validate these addresses for the current routing domain // the address itself is irrelevant, and the remote node can lie anyway - if let Some(socket_address) = status_a.sender_info.socket_address { - match send_data_kind { - SendDataKind::Direct(connection_descriptor) => match routing_domain { - RoutingDomain::PublicInternet => self - .network_manager() - .report_public_internet_socket_address( - socket_address, - connection_descriptor, - peer, - ), - RoutingDomain::LocalNetwork => { - self.network_manager().report_local_network_socket_address( - socket_address, - connection_descriptor, - peer, - ) + let mut opt_sender_info = None; + match dest { + Destination::Direct { + target, + safety_selection, + } => { + if matches!(safety_selection, SafetySelection::Unsafe(_)) { + if let Some(sender_info) = status_a.sender_info { + match send_data_kind { + SendDataKind::Direct(connection_descriptor) => { + // Directly requested status that actually gets sent directly and not over a relay will tell us what our IP address appears as + // If this changes, we'd want to know about that to reset the networking stack + match routing_domain { + RoutingDomain::PublicInternet => self + .network_manager() + .report_public_internet_socket_address( + sender_info.socket_address, + connection_descriptor, + target, + ), + RoutingDomain::LocalNetwork => { + self.network_manager().report_local_network_socket_address( + sender_info.socket_address, + connection_descriptor, + target, + ) + } + } + opt_sender_info = Some(sender_info.clone()); + } + SendDataKind::Indirect => { + // Do nothing in this case, as the socket address returned here would be for any node other than ours + } + SendDataKind::Existing(_) => { + // Do nothing in this case, as an existing connection could not have a different public address or it would have been reset + } + }; } - }, - SendDataKind::Indirect => { - // Do nothing in this case, as the socket address returned here would be for any node other than ours - } - SendDataKind::Existing(_) => { - // Do nothing in this case, as an existing connection could not have a different public address or it would have been reset } } - } - - Ok(NetworkResult::value(Answer::new( - latency, - status_a.sender_info, - ))) + Destination::Relay { + relay: _, + target: _, + safety_selection: _, + } + | Destination::PrivateRoute { + private_route: _, + safety_selection: _, + } => { + // sender info is irrelevant over relays and routes + } + }; + Ok(NetworkResult::value(Answer::new(latency, opt_sender_info))) } #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] pub(crate) async fn process_status_q(&self, msg: RPCMessage) -> Result<(), RPCError> { - let detail = match &msg.header.detail { - RPCMessageHeaderDetail::Direct(detail) => detail, - RPCMessageHeaderDetail::PrivateRouted(_) => { - return Err(RPCError::protocol("status_q must be direct")); - } - }; - - let connection_descriptor = detail.connection_descriptor; - let routing_domain = detail.routing_domain; - // Get the question let status_q = match msg.operation.kind() { RPCOperationKind::Question(q) => match q.detail() { @@ -122,36 +187,55 @@ impl RPCProcessor { _ => panic!("not a question"), }; - // Ensure the node status from the question is the kind for the routing domain we received the request in - match routing_domain { - RoutingDomain::PublicInternet => { - if !matches!(status_q.node_status, NodeStatus::PublicInternet(_)) { - log_rpc!(debug "node status doesn't match PublicInternet routing domain"); - return Ok(()); + let (node_status, sender_info) = match &msg.header.detail { + RPCMessageHeaderDetail::Direct(detail) => { + let connection_descriptor = detail.connection_descriptor; + let routing_domain = detail.routing_domain; + + // Ensure the node status from the question is the kind for the routing domain we received the request in + if let Some(node_status) = &status_q.node_status { + match routing_domain { + RoutingDomain::PublicInternet => { + if !matches!(node_status, NodeStatus::PublicInternet(_)) { + log_rpc!(debug "node status doesn't match PublicInternet routing domain"); + return Ok(()); + } + } + RoutingDomain::LocalNetwork => { + if !matches!(node_status, NodeStatus::LocalNetwork(_)) { + log_rpc!(debug "node status doesn't match LocalNetwork routing domain"); + return Ok(()); + } + } + } + + // update node status for the requesting node to our routing table + if let Some(sender_nr) = msg.opt_sender_nr.clone() { + // Update latest node status in routing table for the statusq sender + sender_nr.update_node_status(node_status.clone()); + } } + + // Get the peer address in the returned sender info + let sender_info = SenderInfo { + socket_address: *connection_descriptor.remote_address(), + }; + + // Make status answer + let node_status = self.network_manager().generate_node_status(routing_domain); + (Some(node_status), Some(sender_info)) } - RoutingDomain::LocalNetwork => { - if !matches!(status_q.node_status, NodeStatus::LocalNetwork(_)) { - log_rpc!(debug "node status doesn't match LocalNetwork routing domain"); - return Ok(()); - } + RPCMessageHeaderDetail::SafetyRouted(_) => { + // Make status answer + let node_status = self + .network_manager() + .generate_node_status(RoutingDomain::PublicInternet); + (Some(node_status), None) } - } - - // update node status for the requesting node to our routing table - if let Some(sender_nr) = msg.opt_sender_nr.clone() { - // Update latest node status in routing table for the statusq sender - sender_nr.update_node_status(status_q.node_status.clone()); - } - - // Make status answer - let node_status = self.network_manager().generate_node_status(routing_domain); - - // Get the peer address in the returned sender info - let sender_info = SenderInfo { - socket_address: Some(*connection_descriptor.remote_address()), + RPCMessageHeaderDetail::PrivateRouted(_) => (None, None), }; + // Make status answer let status_a = RPCOperationStatusA { node_status, sender_info, diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 11de3e23..14e3c905 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -35,7 +35,8 @@ impl RPCProcessor { // Wait for receipt match eventual_value.await.take_value().unwrap() { ReceiptEvent::ReturnedPrivate { private_route: _ } - | ReceiptEvent::ReturnedInBand { inbound_noderef: _ } => { + | ReceiptEvent::ReturnedInBand { inbound_noderef: _ } + | ReceiptEvent::ReturnedSafety => { log_net!(debug "validate_dial_info receipt should be returned out-of-band".green()); Ok(false) } diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index e990ebfc..5fe6173a 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -3,6 +3,7 @@ use super::*; use routing_table::*; +use rpc_processor::*; fn get_bucket_entry_state(text: &str) -> Option { if text == "dead" { @@ -397,7 +398,7 @@ impl VeilidAPI { // Dump routing table entry let out = match rpc - .rpc_call_status(nr) + .rpc_call_status(Destination::direct(nr)) .await .map_err(VeilidAPIError::internal)? { diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 16a7ef4f..e30ef957 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -366,11 +366,6 @@ impl BlockId { ///////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Default, Serialize, Deserialize)] -pub struct SenderInfo { - pub socket_address: Option, -} - // Keep member order appropriate for sorting < preference #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)] pub enum DialInfoClass { @@ -420,7 +415,7 @@ pub enum Stability { Reliable, } -/// The choice of safety route including in compiled routes +/// The choice of safety route to include in compiled routes #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub enum SafetySelection { /// Don't use a safety route, only specify the sequencing preference From 941cf9309e22cb469a70fb3581f37dfaefff0c74 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 2 Nov 2022 16:29:29 -0400 Subject: [PATCH 30/67] lifetime cleanup --- veilid-core/src/routing_table/mod.rs | 10 +- .../src/routing_table/route_spec_store.rs | 174 +++++++++--------- 2 files changed, 97 insertions(+), 87 deletions(-) diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 96fb913b..05711f2b 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -34,8 +34,8 @@ pub struct LowLevelPortInfo { pub low_level_protocol_ports: LowLevelProtocolPorts, pub protocol_to_port: ProtocolToPortMapping, } -pub type RoutingTableEntryFilter = - Box FnMut(&'r RoutingTableInner, DHTKey, Option>) -> bool + Send>; +pub type RoutingTableEntryFilter<'t> = + Box>) -> bool + Send + 't>; #[derive(Clone, Debug, Default)] pub struct RoutingTableHealth { @@ -550,7 +550,7 @@ impl RoutingTable { pub fn make_inbound_dial_info_entry_filter<'a>( routing_domain: RoutingDomain, dial_info_filter: DialInfoFilter, - ) -> RoutingTableEntryFilter { + ) -> RoutingTableEntryFilter<'a> { // does it have matching public dial info? Box::new(move |rti, _k, e| { if let Some(e) = e { @@ -575,10 +575,10 @@ impl RoutingTable { } /// Makes a filter that finds nodes capable of dialing a particular outbound dialinfo - pub fn make_outbound_dial_info_entry_filter( + pub fn make_outbound_dial_info_entry_filter<'a>( routing_domain: RoutingDomain, dial_info: DialInfo, - ) -> RoutingTableEntryFilter { + ) -> RoutingTableEntryFilter<'a> { // does the node's outbound capabilities match the dialinfo? Box::new(move |rti, _k, e| { if let Some(e) = e { diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index d99a5620..93b267de 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -138,13 +138,17 @@ fn _get_route_permutation_count(hop_count: usize) -> usize { (3..hop_count).into_iter().fold(2usize, |acc, x| acc * x) } +type PermFunc<'t> = Box Option<(Vec, Vec)> + Send + 't>; + /// get the route permutation at particular 'perm' index, starting at the 'start' index /// for a set of 'hop_count' nodes. the first node is always fixed, and the maximum /// number of permutations is given by get_route_permutation_count() -fn with_route_permutations(hop_count: usize, start: usize, mut f: F) -> bool -where - F: FnMut(&[usize]) -> bool, -{ + +fn with_route_permutations( + hop_count: usize, + start: usize, + f: &PermFunc, +) -> Option<(Vec, Vec)> { if hop_count == 0 { unreachable!(); } @@ -159,20 +163,19 @@ where } // heaps algorithm, but skipping the first element - fn heaps_permutation(permutation: &mut [usize], size: usize, mut f: F) -> bool - where - F: FnMut(&[usize]) -> bool, - { + fn heaps_permutation( + permutation: &mut [usize], + size: usize, + f: &PermFunc, + ) -> Option<(Vec, Vec)> { if size == 1 { - if f(&permutation) { - return true; - } - return false; + return f(&permutation); } for i in 0..size { - if heaps_permutation(permutation, size - 1, &mut f) { - return true; + let out = heaps_permutation(permutation, size - 1, f); + if out.is_some() { + return out; } if size % 2 == 1 { permutation.swap(1, size); @@ -180,7 +183,8 @@ where permutation.swap(1 + i, size); } } - false + + None } // recurse @@ -502,77 +506,81 @@ impl RouteSpecStore { } // Now go through nodes and try to build a route we haven't seen yet + let perm_func = Box::new(|permutation: &[usize]| { + // Get the route cache key + let cache_key = route_permutation_to_hop_cache(&nodes, permutation); + + // Skip routes we have already seen + if inner.cache.hop_cache.contains(&cache_key) { + return None; + } + + // Ensure this route is viable by checking that each node can contact the next one + if directions.contains(Direction::Outbound) { + let our_node_info = rti.get_own_node_info(RoutingDomain::PublicInternet); + let our_node_id = rti.node_id(); + let mut previous_node = &(our_node_id, our_node_info); + let mut reachable = true; + for n in permutation { + let current_node = nodes.get(*n).unwrap(); + let cm = rti.get_contact_method( + RoutingDomain::PublicInternet, + &previous_node.0, + &previous_node.1, + ¤t_node.0, + ¤t_node.1, + DialInfoFilter::all(), + sequencing, + ); + if matches!(cm, ContactMethod::Unreachable) { + reachable = false; + break; + } + previous_node = current_node; + } + if !reachable { + return None; + } + } + if directions.contains(Direction::Inbound) { + let our_node_info = rti.get_own_node_info(RoutingDomain::PublicInternet); + let our_node_id = rti.node_id(); + let mut next_node = &(our_node_id, our_node_info); + let mut reachable = true; + for n in permutation.iter().rev() { + let current_node = nodes.get(*n).unwrap(); + let cm = rti.get_contact_method( + RoutingDomain::PublicInternet, + &next_node.0, + &next_node.1, + ¤t_node.0, + ¤t_node.1, + DialInfoFilter::all(), + sequencing, + ); + if matches!(cm, ContactMethod::Unreachable) { + reachable = false; + break; + } + next_node = current_node; + } + if !reachable { + return None; + } + } + // Keep this route + let route_nodes = permutation.to_vec(); + Some((route_nodes, cache_key)) + }) as PermFunc; + let mut route_nodes: Vec = Vec::new(); let mut cache_key: Vec = Vec::new(); + for start in 0..(nodes.len() - hop_count) { // Try the permutations available starting with 'start' - let done = with_route_permutations(hop_count, start, |permutation: &[usize]| { - // Get the route cache key - cache_key = route_permutation_to_hop_cache(&nodes, permutation); - - // Skip routes we have already seen - if inner.cache.hop_cache.contains(&cache_key) { - return false; - } - - // Ensure this route is viable by checking that each node can contact the next one - if directions.contains(Direction::Outbound) { - let our_node_info = rti.get_own_node_info(RoutingDomain::PublicInternet); - let our_node_id = rti.node_id(); - let mut previous_node = &(our_node_id, our_node_info); - let mut reachable = true; - for n in permutation { - let current_node = nodes.get(*n).unwrap(); - let cm = rti.get_contact_method( - RoutingDomain::PublicInternet, - &previous_node.0, - &previous_node.1, - ¤t_node.0, - ¤t_node.1, - DialInfoFilter::all(), - sequencing, - ); - if matches!(cm, ContactMethod::Unreachable) { - reachable = false; - break; - } - previous_node = current_node; - } - if !reachable { - return false; - } - } - if directions.contains(Direction::Inbound) { - let our_node_info = rti.get_own_node_info(RoutingDomain::PublicInternet); - let our_node_id = rti.node_id(); - let mut next_node = &(our_node_id, our_node_info); - let mut reachable = true; - for n in permutation.iter().rev() { - let current_node = nodes.get(*n).unwrap(); - let cm = rti.get_contact_method( - RoutingDomain::PublicInternet, - &next_node.0, - &next_node.1, - ¤t_node.0, - ¤t_node.1, - DialInfoFilter::all(), - sequencing, - ); - if matches!(cm, ContactMethod::Unreachable) { - reachable = false; - break; - } - next_node = current_node; - } - if !reachable { - return false; - } - } - // Keep this route - route_nodes = permutation.to_vec(); - true - }); - if done { + if let Some((rn, ck)) = with_route_permutations(hop_count, start, &perm_func) { + route_nodes = rn; + cache_key = ck; break; } } @@ -611,6 +619,8 @@ impl RouteSpecStore { sequencing, }; + drop(perm_func); + // Add to cache Self::add_to_cache(&mut inner.cache, cache_key, &rsd); From ee7ccd698d5f7b4baff8a7b19751090622388c32 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 2 Nov 2022 19:37:47 -0400 Subject: [PATCH 31/67] setup fixes --- Cargo.lock | 489 ++++++++++++++++++--------- scripts/earthly/install_capnproto.sh | 33 +- setup_linux.sh | 4 +- 3 files changed, 355 insertions(+), 171 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8c0adabc..8bb019ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,7 +44,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", "once_cell", "version_check", ] @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "arraydeque" @@ -191,9 +191,9 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0da5b41ee986eed3f524c380e6d64965aea573882a8907682ad100f7859305ca" +checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" dependencies = [ "async-channel", "async-executor", @@ -206,16 +206,16 @@ dependencies = [ [[package]] name = "async-io" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7" +checksum = "e8121296a9f05be7f34aa4196b1747243b3b62e048bb7906f644f3fbfc490cf7" dependencies = [ + "async-lock", "autocfg", "concurrent-queue", "futures-lite", "libc", "log", - "once_cell", "parking", "polling", "slab", @@ -226,11 +226,12 @@ dependencies = [ [[package]] name = "async-lock" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" dependencies = [ "event-listener", + "futures-lite", ] [[package]] @@ -336,9 +337,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" dependencies = [ "proc-macro2", "quote", @@ -360,9 +361,9 @@ dependencies = [ [[package]] name = "async-tungstenite" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b71b31561643aa8e7df3effe284fa83ab1a840e52294c5f4bd7bfd8b2becbb" +checksum = "4b750efd83b7e716a015eed5ebb583cda83c52d9b24a8f0125e5c48c3313c9f8" dependencies = [ "async-tls", "futures-io", @@ -447,9 +448,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e3356844c4d6a6d6467b8da2cffb4a2820be256f50a3a386c9d152bab31043" +checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43" dependencies = [ "async-trait", "axum-core", @@ -476,9 +477,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f0c0a60006f2a293d82d571f635042a72edf927539b7685bd62d361963839b" +checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" dependencies = [ "async-trait", "bytes 1.2.1", @@ -513,9 +514,9 @@ checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bindgen" @@ -649,15 +650,15 @@ checksum = "cc12a55e9bd3840279c248c96ecf541d5ba98d6654e08869fe167121384a582c" [[package]] name = "bumpalo" -version = "3.11.0" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "byte-slice-cast" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87c5fdd0166095e1d463fc6cc01aa8ce547ad77a4e84d42eb6762b084e28067e" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" [[package]] name = "byteorder" @@ -727,9 +728,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.73" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "581f5dba903aac52ea3feb5ec4810848460ee833876f1f9b0fdeab1f19091574" [[package]] name = "cesu8" @@ -868,9 +869,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.22" +version = "3.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", "bitflags", @@ -892,13 +893,23 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.48" +version = "0.1.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" +checksum = "db34956e100b30725f2eb215f90d4871051239535632f84fea3bc92722c66b7c" dependencies = [ "cc", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "color-eyre" version = "0.6.2" @@ -1105,7 +1116,7 @@ dependencies = [ "bitflags", "crossterm_winapi", "libc", - "mio 0.8.4", + "mio 0.8.5", "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", @@ -1149,9 +1160,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ "quote", "syn", @@ -1202,7 +1213,7 @@ dependencies = [ "flexi_logger", "lazy_static", "log", - "time 0.3.14", + "time 0.3.16", "unicode-width", ] @@ -1231,7 +1242,7 @@ dependencies = [ "log", "num", "owning_ref", - "time 0.3.14", + "time 0.3.16", "tokio 1.21.2", "toml", "unicode-segmentation", @@ -1274,6 +1285,50 @@ dependencies = [ "zeroize", ] +[[package]] +name = "cxx" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "daemonize" version = "0.4.1" @@ -1296,12 +1351,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02" +checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" dependencies = [ - "darling_core 0.14.1", - "darling_macro 0.14.1", + "darling_core 0.14.2", + "darling_macro 0.14.2", ] [[package]] @@ -1320,9 +1375,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f" +checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" dependencies = [ "fnv", "ident_case", @@ -1344,11 +1399,11 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5" +checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" dependencies = [ - "darling_core 0.14.1", + "darling_core 0.14.2", "quote", "syn", ] @@ -1363,7 +1418,7 @@ dependencies = [ "hashbrown", "lock_api", "once_cell", - "parking_lot_core 0.9.3", + "parking_lot_core 0.9.4", ] [[package]] @@ -1525,7 +1580,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03e7b551eba279bf0fa88b83a46330168c1560a52a94f5126f892f0b364ab3e0" dependencies = [ - "darling 0.14.1", + "darling 0.14.2", "proc-macro2", "quote", "syn", @@ -1684,7 +1739,7 @@ dependencies = [ "regex", "rustversion", "thiserror", - "time 0.3.14", + "time 0.3.16", ] [[package]] @@ -1749,9 +1804,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ "futures-channel", "futures-core", @@ -1764,9 +1819,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", "futures-sink", @@ -1774,15 +1829,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" [[package]] name = "futures-executor" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" dependencies = [ "futures-core", "futures-task", @@ -1791,9 +1846,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" [[package]] name = "futures-lite" @@ -1812,9 +1867,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" dependencies = [ "proc-macro2", "quote", @@ -1823,15 +1878,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" [[package]] name = "futures-task" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" [[package]] name = "futures-timer" @@ -1845,9 +1900,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", @@ -1884,9 +1939,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1964,9 +2019,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" dependencies = [ "bytes 1.2.1", "fnv", @@ -2107,9 +2162,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.20" +version = "0.14.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +checksum = "abfba89e19b959ca163c7752ba59d737c1ceea53a5d31a149c805446fc958064" dependencies = [ "bytes 1.2.1", "futures-channel", @@ -2143,17 +2198,28 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.50" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd911b35d940d2bd0bea0f9100068e5b97b51a1cbe13d13382f132e0365257a0" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" dependencies = [ "android_system_properties", "core-foundation-sys 0.8.3", + "iana-time-zone-haiku", "js-sys", "wasm-bindgen", "winapi 0.3.9", ] +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2332,15 +2398,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "jni" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +checksum = "039022cdf4d7b1cf548d31f60ae783138e5fd42013f6271049d7df7afadef96c" dependencies = [ "cesu8", "combine", @@ -2531,9 +2597,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.134" +version = "0.2.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" [[package]] name = "libloading" @@ -2547,9 +2613,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.25.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f0455f2c1bc9a7caa792907026e469c1d91761fb0ea37cbb16427c77280cf35" +checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" dependencies = [ "cc", "pkg-config", @@ -2568,6 +2634,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2721,14 +2796,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -2775,7 +2850,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", ] [[package]] @@ -2871,9 +2946,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +checksum = "74d0df99cfcd2530b2e694f6e17e7f37b8e26bb23983ac530c0c97408837c631" dependencies = [ "cfg-if 0.1.10", "libc", @@ -3009,6 +3084,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi 0.3.9", +] + [[package]] name = "num" version = "0.4.0" @@ -3136,9 +3221,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "oorandom" @@ -3258,9 +3343,15 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.3.0" +version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "owning_ref" @@ -3327,7 +3418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.3", + "parking_lot_core 0.9.4", ] [[package]] @@ -3346,15 +3437,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3505,9 +3596,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "plotters" @@ -3539,9 +3630,9 @@ dependencies = [ [[package]] name = "polling" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011" +checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -3570,9 +3661,9 @@ checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "prettyplease" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a49e86d2c26a24059894a3afa13fd17d063419b05dfb83f06d9c3566060c3f5a" +checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51" dependencies = [ "proc-macro2", "syn", @@ -3580,9 +3671,9 @@ dependencies = [ [[package]] name = "primitive-types" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cfd65aea0c5fa0bfcc7c9e7ca828c921ef778f43d325325ec84bda371bfa75a" +checksum = "9f3486ccba82358b11a77516035647c34ba167dfa53312630de83b12bd4f3d66" dependencies = [ "fixed-hash", "impl-codec", @@ -3637,9 +3728,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -3783,7 +3874,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", ] [[package]] @@ -3843,7 +3934,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", "redox_syscall", "thiserror", ] @@ -3910,9 +4001,9 @@ dependencies = [ [[package]] name = "rlp" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "999508abb0ae792aabed2460c45b89106d97fe4adac593bdaef433c2605847b5" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" dependencies = [ "bytes 1.2.1", "rustc-hex", @@ -3924,7 +4015,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", "bitflags", "serde", ] @@ -4041,7 +4132,7 @@ version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", "log", "ring", "sct", @@ -4054,7 +4145,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", ] [[package]] @@ -4080,9 +4171,9 @@ dependencies = [ [[package]] name = "scoped-tls" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "scopeguard" @@ -4090,6 +4181,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + [[package]] name = "sct" version = "0.6.1" @@ -4181,9 +4278,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.145" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" dependencies = [ "serde_derive", ] @@ -4209,9 +4306,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.145" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" dependencies = [ "proc-macro2", "quote", @@ -4220,9 +4317,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45" dependencies = [ "itoa", "ryu", @@ -4242,18 +4339,18 @@ dependencies = [ [[package]] name = "serde_test" -version = "1.0.145" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c17d2112159132660b4c5399e274f676fb75a2f8d70b7468f18f045b71138ed" +checksum = "641666500e4e6fba7b91b73651a375cb53579468ab3c38389289b802797cad94" dependencies = [ "serde", ] [[package]] name = "serde_yaml" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8613d593412a0deb7bbd8de9d908efff5a0cb9ccd8f62c641e7b2ed2f57291d1" +checksum = "6d232d893b10de3eb7258ff01974d6ee20663d8e833263c99409d4b13a0209da" dependencies = [ "indexmap", "itoa", @@ -4380,7 +4477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio 0.8.4", + "mio 0.8.5", "signal-hook", ] @@ -4395,9 +4492,9 @@ dependencies = [ [[package]] name = "signature" -version = "1.6.3" +version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb766570a2825fa972bceff0d195727876a9cdf2460ab2e52d455dc2de47fd9" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" [[package]] name = "simplelog" @@ -4418,7 +4515,7 @@ checksum = "48dfff04aade74dd495b007c831cd6f4e0cee19c344dd9dc0884c0289b70a786" dependencies = [ "log", "termcolor", - "time 0.3.14", + "time 0.3.16", ] [[package]] @@ -4515,9 +4612,9 @@ checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" [[package]] name = "syn" -version = "1.0.101" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ "proc-macro2", "quote", @@ -4544,9 +4641,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.26.4" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7890fff842b8db56f2033ebee8f6efe1921475c3830c115995552914fb967580" +checksum = "c375d5fd899e32847b8566e10598d6e9f1d9b55ec6de3cdf9e7da4bdc51371bc" dependencies = [ "cfg-if 1.0.0", "core-foundation-sys 0.8.3", @@ -4588,9 +4685,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" @@ -4634,21 +4731,32 @@ dependencies = [ [[package]] name = "time" -version = "0.3.14" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" +checksum = "0fab5c8b9980850e06d92ddbe3ab839c062c801f3927c0fb8abd6fc8e918fbca" dependencies = [ "itoa", "libc", "num_threads", + "serde", + "time-core", "time-macros", ] [[package]] -name = "time-macros" -version = "0.2.4" +name = "time-core" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb801831d812c562ae7d2bfb531f26e66e4e1f6b17307ba4149c5064710e5b" +dependencies = [ + "time-core", +] [[package]] name = "tiny-keccak" @@ -4722,7 +4830,7 @@ dependencies = [ "bytes 1.2.1", "libc", "memchr", - "mio 0.8.4", + "mio 0.8.5", "num_cpus", "parking_lot 0.12.1", "pin-project-lite 0.2.9", @@ -4755,9 +4863,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" +checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite 0.2.9", @@ -4797,7 +4905,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.13.0", + "base64 0.13.1", "bytes 1.2.1", "futures-core", "futures-util", @@ -4874,9 +4982,9 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" [[package]] name = "tower-service" @@ -4886,9 +4994,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "log", @@ -4915,15 +5023,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.14", + "time 0.3.16", "tracing-subscriber", ] [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", @@ -4932,9 +5040,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -4998,12 +5106,12 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ - "ansi_term", "matchers", + "nu-ansi-term", "once_cell", "regex", "sharded-slab", @@ -5101,7 +5209,7 @@ version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", "byteorder", "bytes 1.2.1", "http", @@ -5146,9 +5254,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-normalization" @@ -5286,7 +5394,7 @@ dependencies = [ "async-std", "async-std-resolver", "async-tls", - "async-tungstenite 0.17.2", + "async-tungstenite 0.18.0", "async_executors", "backtrace", "blake3", @@ -5309,7 +5417,7 @@ dependencies = [ "flume", "futures-util", "generic-array", - "getrandom 0.2.7", + "getrandom 0.2.8", "hashbrown", "hashlink 0.8.0", "hex", @@ -5411,7 +5519,7 @@ version = "0.1.0" dependencies = [ "ansi_term", "async-std", - "async-tungstenite 0.17.2", + "async-tungstenite 0.18.0", "bugsalot", "capnp", "capnp-rpc", @@ -5804,7 +5912,7 @@ dependencies = [ "bitflags", "err-derive", "widestring 1.0.2", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -5820,6 +5928,27 @@ dependencies = [ "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" @@ -5832,6 +5961,12 @@ version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b12add87e2fb192fff3f4f7e4342b3694785d79f3a64e2c20d5ceb5ccbcfc3cd" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + [[package]] name = "windows_i686_gnu" version = "0.36.1" @@ -5844,6 +5979,12 @@ version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c98f2db372c23965c5e0f43896a8f0316dc0fbe48d1aa65bea9bdd295d43c15" +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + [[package]] name = "windows_i686_msvc" version = "0.36.1" @@ -5856,6 +5997,12 @@ version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdf0569be0f2863ab6a12a6ba841fcfa7d107cbc7545a3ebd57685330db0a3ff" +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" @@ -5868,6 +6015,18 @@ version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "905858262c8380a36f32cb8c1990d7e7c3b7a8170e58ed9a98ca6d940b7ea9f1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" @@ -5880,6 +6039,12 @@ version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "890c3c6341d441ffb38f705f47196e3665dc6dd79f6d72fa185d937326730561" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "winreg" version = "0.7.0" diff --git a/scripts/earthly/install_capnproto.sh b/scripts/earthly/install_capnproto.sh index 71aa7d9d..69ae43df 100755 --- a/scripts/earthly/install_capnproto.sh +++ b/scripts/earthly/install_capnproto.sh @@ -1,11 +1,28 @@ #!/bin/bash mkdir /tmp/capnproto-install -cd /tmp/capnproto-install -curl -O https://capnproto.org/capnproto-c++-0.9.1.tar.gz -tar zxf capnproto-c++-0.9.1.tar.gz -cd capnproto-c++-0.9.1 -./configure +pushd /tmp/capnproto-install +curl -O https://capnproto.org/capnproto-c++-0.10.2.tar.gz +tar zxf capnproto-c++-0.10.2.tar.gz +cd capnproto-c++-0.10.2 +./configure --without-openssl make -j6 check -make install -cd / -rm -rf /tmp/capnproto-install +if [ "$EUID" -ne 0 ]; then + if command -v checkinstall &> /dev/null; then + sudo checkinstall -y + cp *.deb ~ + else + sudo make install + + fi + popd + sudo rm -rf /tmp/capnproto-install +else + if command -v checkinstall &> /dev/null; then + checkinstall -y + cp *.deb ~ + else + make install + fi + popd + rm -rf /tmp/capnproto-install +fi diff --git a/setup_linux.sh b/setup_linux.sh index 68c17c16..00e4e647 100755 --- a/setup_linux.sh +++ b/setup_linux.sh @@ -77,5 +77,7 @@ rustup target add aarch64-linux-android armv7-linux-androideabi i686-linux-andro cargo install wasm-bindgen-cli wasm-pack # Ensure packages are installed -sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk llvm wabt capnproto protobuf-compiler +sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk llvm wabt protobuf-compiler checkinstall +# Install capnproto using the same mechanism as our earthly build +$SCRIPTDIR/scripts/earthly/install_capnproto.sh From 74dc92c65794473f0be3e31a29c5484351c67103 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 2 Nov 2022 20:42:07 -0400 Subject: [PATCH 32/67] updates --- Cargo.lock | 32 +++++++++++++++++-------- Earthfile | 40 +++++++++++++++++++++---------- external/cursive | 2 +- external/hashlink | 2 +- scripts/earthly/install_protoc.sh | 26 ++++++++++++++++++++ setup_linux.sh | 4 +++- 6 files changed, 80 insertions(+), 26 deletions(-) create mode 100755 scripts/earthly/install_protoc.sh diff --git a/Cargo.lock b/Cargo.lock index 8bb019ef..a4656778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,6 +49,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464b3811b747f8f7ebc8849c9c728c39f6ac98a055edad93baf9eb330e3f8f9d" +dependencies = [ + "cfg-if 1.0.0", + "getrandom 0.2.8", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.19" @@ -1109,9 +1121,9 @@ dependencies = [ [[package]] name = "crossterm" -version = "0.23.2" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2102ea4f781910f8a5b98dd061f4c2023f479ce7bb1236330099ceb5a93cf17" +checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" dependencies = [ "bitflags", "crossterm_winapi", @@ -1186,9 +1198,9 @@ checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" [[package]] name = "cursive" -version = "0.18.0" +version = "0.20.0" dependencies = [ - "ahash", + "ahash 0.8.1", "async-std", "cfg-if 1.0.0", "crossbeam-channel", @@ -1231,9 +1243,9 @@ dependencies = [ [[package]] name = "cursive_core" -version = "0.3.2" +version = "0.3.5" dependencies = [ - "ahash", + "ahash 0.8.1", "async-std", "crossbeam-channel", "enum-map", @@ -2048,12 +2060,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", + "ahash 0.7.6", ] [[package]] name = "hashlink" -version = "0.8.0" +version = "0.8.1" dependencies = [ "hashbrown", "serde", @@ -4065,7 +4077,7 @@ dependencies = [ "bitflags", "fallible-iterator", "fallible-streaming-iterator", - "hashlink 0.8.1", + "hashlink 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "libsqlite3-sys", "smallvec", ] @@ -5419,7 +5431,7 @@ dependencies = [ "generic-array", "getrandom 0.2.8", "hashbrown", - "hashlink 0.8.0", + "hashlink 0.8.1", "hex", "ifstructs", "igd", diff --git a/Earthfile b/Earthfile index 8ce1a7bd..e8f17b66 100644 --- a/Earthfile +++ b/Earthfile @@ -7,7 +7,7 @@ FROM --platform amd64 ubuntu:16.04 # Install build prerequisites deps-base: RUN apt-get -y update - RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev + RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip # Install Cap'n Proto deps-capnp: @@ -15,9 +15,15 @@ deps-capnp: COPY scripts/earthly/install_capnproto.sh / RUN /bin/bash /install_capnproto.sh; rm /install_capnproto.sh +# Install protoc +deps-protoc: + FROM +deps-capnp + COPY scripts/earthly/install_protoc.sh / + RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh + # Install Rust deps-rust: - FROM +deps-capnp + FROM +deps-protoc ENV RUSTUP_HOME=/usr/local/rustup ENV CARGO_HOME=/usr/local/cargo ENV PATH=/usr/local/cargo/bin:$PATH @@ -49,35 +55,43 @@ deps-android: RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip RUN cd /Android; unzip /Android/cmdline-tools.zip RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;30.0.3 ndk\;22.0.7026061 cmake\;3.18.1 platform-tools platforms\;android-30 + RUN apt-get clean -# Clean up the apt cache to save space -deps: - FROM +deps-android +# Just linux build not android +deps-linux: + FROM +deps-cross RUN apt-get clean -code: - FROM +deps +# Code + Linux deps +code-linux: + FROM +deps-linux + COPY --dir .cargo external files scripts veilid-cli veilid-core veilid-server veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid + WORKDIR /veilid + +# Code + Linux + Android deps +code-android: + FROM +deps-android COPY --dir .cargo external files scripts veilid-cli veilid-core veilid-server veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid WORKDIR /veilid # Clippy only clippy: - FROM +code + FROM +code-linux RUN cargo clippy # Build build-linux-amd64: - FROM +code + FROM +code-linux RUN cargo build --target x86_64-unknown-linux-gnu --release SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu build-linux-arm64: - FROM +code + FROM +code-linux RUN cargo build --target aarch64-unknown-linux-gnu --release SAVE ARTIFACT ./target/aarch64-unknown-linux-gnu AS LOCAL ./target/artifacts/aarch64-unknown-linux-gnu build-android: - FROM +code + FROM +code-android WORKDIR /veilid/veilid-core ENV PATH=$PATH:/Android/Sdk/ndk/22.0.7026061/toolchains/llvm/prebuilt/linux-x86_64/bin/ RUN cargo build --target aarch64-linux-android --release @@ -92,11 +106,11 @@ build-android: # Unit tests unit-tests-linux-amd64: - FROM +code + FROM +code-linux RUN cargo test --target x86_64-unknown-linux-gnu --release unit-tests-linux-arm64: - FROM +code + FROM +code-linux RUN cargo test --target aarch64-unknown-linux-gnu --release # Package diff --git a/external/cursive b/external/cursive index fea04c2f..f1504cf3 160000 --- a/external/cursive +++ b/external/cursive @@ -1 +1 @@ -Subproject commit fea04c2f9bb8c4c9551ca6eb4f2cb1268551120f +Subproject commit f1504cf37a7021454020cda5cfba815755399794 diff --git a/external/hashlink b/external/hashlink index c8da3a58..a089b448 160000 --- a/external/hashlink +++ b/external/hashlink @@ -1 +1 @@ -Subproject commit c8da3a58485c850f4029a58de99b1af83112ba8a +Subproject commit a089b448071ef36633947693b90023c67dc8485f diff --git a/scripts/earthly/install_protoc.sh b/scripts/earthly/install_protoc.sh new file mode 100755 index 00000000..7ba2171a --- /dev/null +++ b/scripts/earthly/install_protoc.sh @@ -0,0 +1,26 @@ +#!/bin/bash +VERSION=21.9 + +mkdir /tmp/protoc-install +pushd /tmp/protoc-install +curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-linux-x86_64.zip +unzip protoc-$VERSION-linux-x86_64.zip +if [ "$EUID" -ne 0 ]; then + if command -v checkinstall &> /dev/null; then + sudo checkinstall --pkgversion=$VERSION -y cp -r bin include /usr/local/ + cp *.deb ~ + else + sudo make install + fi + popd + sudo rm -rf /tmp/protoc-install +else + if command -v checkinstall &> /dev/null; then + checkinstall --pkgversion=$VERSION -y cp -r bin include /usr/local/ + cp *.deb ~ + else + make install + fi + popd + rm -rf /tmp/protoc-install +fi diff --git a/setup_linux.sh b/setup_linux.sh index 00e4e647..87118146 100755 --- a/setup_linux.sh +++ b/setup_linux.sh @@ -77,7 +77,9 @@ rustup target add aarch64-linux-android armv7-linux-androideabi i686-linux-andro cargo install wasm-bindgen-cli wasm-pack # Ensure packages are installed -sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk llvm wabt protobuf-compiler checkinstall +sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk llvm wabt checkinstall # Install capnproto using the same mechanism as our earthly build $SCRIPTDIR/scripts/earthly/install_capnproto.sh +# Install protoc using the same mechanism as our earthly build +$SCRIPTDIR/scripts/earthly/install_protoc.sh From bb03a44e483a1fb47a40d658b2fefd299e4145e9 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 2 Nov 2022 21:54:48 -0400 Subject: [PATCH 33/67] routing domain fixes --- .../routing_table/routing_domain_editor.rs | 23 -------------- .../src/routing_table/routing_domains.rs | 31 ++++++------------- .../src/routing_table/routing_table_inner.rs | 6 ++-- 3 files changed, 13 insertions(+), 47 deletions(-) diff --git a/veilid-core/src/routing_table/routing_domain_editor.rs b/veilid-core/src/routing_table/routing_domain_editor.rs index 65001c09..60e76509 100644 --- a/veilid-core/src/routing_table/routing_domain_editor.rs +++ b/veilid-core/src/routing_table/routing_domain_editor.rs @@ -9,10 +9,6 @@ enum RoutingDomainChange { AddDialInfoDetail { dial_info_detail: DialInfoDetail, }, - SetupNode { - node_id: DHTKey, - node_id_secret: DHTKeySecret, - }, SetupNetwork { outbound_protocols: ProtocolTypeSet, inbound_protocols: ProtocolTypeSet, @@ -84,13 +80,6 @@ impl RoutingDomainEditor { Ok(()) } #[instrument(level = "debug", skip(self))] - pub fn setup_node(&mut self, node_id: DHTKey, node_id_secret: DHTKeySecret) { - self.changes.push(RoutingDomainChange::SetupNode { - node_id, - node_id_secret, - }) - } - #[instrument(level = "debug", skip(self))] pub fn setup_network( &mut self, outbound_protocols: ProtocolTypeSet, @@ -160,18 +149,6 @@ impl RoutingDomainEditor { ); changed = true; } - RoutingDomainChange::SetupNode { - node_id, - node_id_secret, - } => { - debug!( - "[{:?}] setup node: {}", - self.routing_domain, - node_id.encode() - ); - detail.common_mut().setup_node(node_id, node_id_secret); - changed = true; - } RoutingDomainChange::SetupNetwork { outbound_protocols, inbound_protocols, diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 25efeddd..43bd7d44 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -22,8 +22,6 @@ pub enum ContactMethod { #[derive(Debug)] pub struct RoutingDomainDetailCommon { routing_domain: RoutingDomain, - node_id: DHTKey, - node_id_secret: DHTKeySecret, network_class: Option, outbound_protocols: ProtocolTypeSet, inbound_protocols: ProtocolTypeSet, @@ -38,8 +36,6 @@ impl RoutingDomainDetailCommon { pub fn new(routing_domain: RoutingDomain) -> Self { Self { routing_domain, - node_id: Default::default(), - node_id_secret: Default::default(), network_class: Default::default(), outbound_protocols: Default::default(), inbound_protocols: Default::default(), @@ -50,12 +46,6 @@ impl RoutingDomainDetailCommon { } } - // Set from routing table - pub(super) fn setup_node(&mut self, node_id: DHTKey, node_id_secret: DHTKeySecret) { - self.node_id = node_id; - self.node_id_secret = node_id_secret; - self.clear_cache(); - } // Set from network manager pub(super) fn setup_network( &mut self, @@ -66,16 +56,12 @@ impl RoutingDomainDetailCommon { self.outbound_protocols = outbound_protocols; self.inbound_protocols = inbound_protocols; self.address_types = address_types; + self.clear_cache(); } - pub fn node_id(&self) -> DHTKey { - self.node_id - } - pub fn node_id_secret(&self) -> DHTKeySecret { - self.node_id_secret - } pub(super) fn set_network_class(&mut self, network_class: Option) { self.network_class = network_class; + self.clear_cache(); } pub fn network_class(&self) -> Option { self.network_class @@ -95,24 +81,27 @@ impl RoutingDomainDetailCommon { pub(super) fn set_relay_node(&mut self, opt_relay_node: Option) { self.relay_node = opt_relay_node.map(|nr| { nr.filtered_clone(NodeRefFilter::new().with_routing_domain(self.routing_domain)) - }) + }); + self.clear_cache(); } pub fn dial_info_details(&self) -> &Vec { &self.dial_info_details } pub(super) fn clear_dial_info_details(&mut self) { self.dial_info_details.clear(); + self.clear_cache(); } pub(super) fn add_dial_info_detail(&mut self, did: DialInfoDetail) { self.dial_info_details.push(did); self.dial_info_details.sort(); + self.clear_cache(); } pub fn has_valid_own_node_info(&self) -> bool { self.network_class.unwrap_or(NetworkClass::Invalid) != NetworkClass::Invalid } - pub fn with_peer_info(&self, f: F) -> R + pub fn with_peer_info(&self, rti: &RoutingTableInner, f: F) -> R where F: FnOnce(&PeerInfo) -> R, { @@ -120,7 +109,7 @@ impl RoutingDomainDetailCommon { if cpi.is_none() { // Regenerate peer info let pi = PeerInfo::new( - NodeId::new(self.node_id), + NodeId::new(rti.unlocked_inner.node_id), SignedNodeInfo::with_secret( NodeInfo { network_class: self.network_class.unwrap_or(NetworkClass::Invalid), @@ -134,8 +123,8 @@ impl RoutingDomainDetailCommon { .as_ref() .and_then(|rn| rn.make_peer_info(self.routing_domain).map(Box::new)), }, - NodeId::new(self.node_id), - &self.node_id_secret, + NodeId::new(rti.unlocked_inner.node_id), + &rti.unlocked_inner.node_id_secret, ) .unwrap(), ); diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index 2e7c9989..69c48172 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -248,7 +248,7 @@ impl RoutingTableInner { /// Return a copy of our node's peerinfo pub fn get_own_peer_info(&self, routing_domain: RoutingDomain) -> PeerInfo { self.with_routing_domain(routing_domain, |rdd| { - rdd.common().with_peer_info(|pi| pi.clone()) + rdd.common().with_peer_info(self, |pi| pi.clone()) }) } @@ -256,7 +256,7 @@ impl RoutingTableInner { pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedNodeInfo { self.with_routing_domain(routing_domain, |rdd| { rdd.common() - .with_peer_info(|pi| pi.signed_node_info.clone()) + .with_peer_info(self, |pi| pi.signed_node_info.clone()) }) } @@ -264,7 +264,7 @@ impl RoutingTableInner { pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo { self.with_routing_domain(routing_domain, |rdd| { rdd.common() - .with_peer_info(|pi| pi.signed_node_info.node_info.clone()) + .with_peer_info(self, |pi| pi.signed_node_info.node_info.clone()) }) } From f1bf883376e5069d9684da8d38843a587f847a22 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 2 Nov 2022 22:21:22 -0400 Subject: [PATCH 34/67] fix hang --- veilid-core/src/routing_table/mod.rs | 37 ++++++++++++++++++- .../src/routing_table/routing_table_inner.rs | 36 ------------------ 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 05711f2b..ec1508d1 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -500,7 +500,42 @@ impl RoutingTable { } pub fn get_recent_peers(&self) -> Vec<(DHTKey, RecentPeersEntry)> { - self.inner.write().get_recent_peers(self.clone()) + let mut recent_peers = Vec::new(); + let mut dead_peers = Vec::new(); + let mut out = Vec::new(); + + // collect all recent peers + { + let inner = self.inner.read(); + for (k, _v) in &inner.recent_peers { + recent_peers.push(*k); + } + } + + // look up each node and make sure the connection is still live + // (uses same logic as send_data, ensuring last_connection works for UDP) + for e in &recent_peers { + let mut dead = true; + if let Some(nr) = self.lookup_node_ref(*e) { + if let Some(last_connection) = nr.last_connection() { + out.push((*e, RecentPeersEntry { last_connection })); + dead = false; + } + } + if dead { + dead_peers.push(e); + } + } + + // purge dead recent peers + { + let mut inner = self.inner.write(); + for d in dead_peers { + inner.recent_peers.remove(d); + } + } + + out } pub fn touch_recent_peer(&self, node_id: DHTKey, last_connection: ConnectionDescriptor) { diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index 69c48172..36c7240e 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -749,42 +749,6 @@ impl RoutingTableInner { health } - pub fn get_recent_peers( - &mut self, - outer_self: RoutingTable, - ) -> Vec<(DHTKey, RecentPeersEntry)> { - let mut recent_peers = Vec::new(); - let mut dead_peers = Vec::new(); - let mut out = Vec::new(); - - // collect all recent peers - for (k, _v) in &self.recent_peers { - recent_peers.push(*k); - } - - // look up each node and make sure the connection is still live - // (uses same logic as send_data, ensuring last_connection works for UDP) - for e in &recent_peers { - let mut dead = true; - if let Some(nr) = self.lookup_node_ref(outer_self.clone(), *e) { - if let Some(last_connection) = nr.last_connection() { - out.push((*e, RecentPeersEntry { last_connection })); - dead = false; - } - } - if dead { - dead_peers.push(e); - } - } - - // purge dead recent peers - for d in dead_peers { - self.recent_peers.remove(d); - } - - out - } - pub fn touch_recent_peer(&mut self, node_id: DHTKey, last_connection: ConnectionDescriptor) { self.recent_peers .insert(node_id, RecentPeersEntry { last_connection }); From 404f579baa4e42ede06df7ca4f69f787813bda36 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 3 Nov 2022 11:28:29 -0400 Subject: [PATCH 35/67] add tokio console --- .cargo/config.toml | 3 + Cargo.lock | 77 +++++++++++++++++++++++++ veilid-cli/src/client_api_connection.rs | 70 ++++++++++++++++++++-- veilid-cli/src/command_processor.rs | 6 ++ veilid-cli/src/ui.rs | 7 +++ veilid-server/Cargo.toml | 10 ++-- veilid-server/src/cmdline.rs | 17 ++++++ veilid-server/src/main.rs | 17 +++++- veilid-server/src/settings.rs | 10 ++++ veilid-server/src/veilid_logs.rs | 15 +++++ 10 files changed, 221 insertions(+), 11 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index af7d22a8..91fa3b11 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,6 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] + [target.aarch64-unknown-linux-gnu] linker = "aarch64-linux-gnu-gcc" diff --git a/Cargo.lock b/Cargo.lock index a4656778..356bf366 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -973,6 +973,42 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "console-api" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57ff02e8ad8e06ab9731d5dc72dc23bef9200778eae1a89d555d8c42e5d4a86" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a3a81dfaf6b66bce5d159eddae701e3a002f194d378cbf7be5f053c281d9be" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio 1.21.2", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -1040,6 +1076,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "criterion" version = "0.4.0" @@ -1736,6 +1781,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flexi_logger" version = "0.23.3" @@ -2080,6 +2135,19 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "hdrhistogram" +version = "7.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +dependencies = [ + "base64 0.13.1", + "byteorder", + "flate2", + "nom 7.1.1", + "num-traits", +] + [[package]] name = "heck" version = "0.4.0" @@ -2172,6 +2240,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.22" @@ -4849,6 +4923,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "winapi 0.3.9", ] @@ -5532,6 +5607,7 @@ dependencies = [ "ansi_term", "async-std", "async-tungstenite 0.18.0", + "backtrace", "bugsalot", "capnp", "capnp-rpc", @@ -5540,6 +5616,7 @@ dependencies = [ "clap", "color-eyre", "config", + "console-subscriber", "ctrlc", "daemonize", "directories", diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index 682269c3..36276179 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -3,6 +3,7 @@ use crate::tools::*; use crate::veilid_client_capnp::*; use capnp::capability::Promise; use capnp_rpc::{pry, rpc_twoparty_capnp, twoparty, Disconnector, RpcSystem}; +use futures::future::FutureExt; use serde::de::DeserializeOwned; use std::cell::RefCell; use std::net::SocketAddr; @@ -101,6 +102,7 @@ struct ClientApiConnectionInner { disconnector: Option>, server: Option>>, disconnect_requested: bool, + cancel_eventual: Eventual, } type Handle = Rc>; @@ -119,9 +121,19 @@ impl ClientApiConnection { disconnector: None, server: None, disconnect_requested: false, + cancel_eventual: Eventual::new(), })), } } + + pub fn cancel(&self) { + let eventual = { + let inner = self.inner.borrow(); + inner.cancel_eventual.clone() + }; + eventual.resolve(); // don't need to await this + } + async fn process_veilid_state<'a>( &'a mut self, veilid_state: VeilidState, @@ -264,6 +276,34 @@ impl ClientApiConnection { } } + pub fn cancellable(&mut self, p: Promise) -> Promise + where + T: 'static, + { + let (mut cancel_instance, cancel_eventual) = { + let inner = self.inner.borrow(); + ( + inner.cancel_eventual.instance_empty().fuse(), + inner.cancel_eventual.clone(), + ) + }; + let mut p = p.fuse(); + + Promise::from_future(async move { + let out = select! { + a = p => { + a + }, + _ = cancel_instance => { + Err(capnp::Error::failed("cancelled".into())) + } + }; + drop(cancel_instance); + cancel_eventual.reset(); + out + }) + } + pub async fn server_attach(&mut self) -> Result<(), String> { trace!("ClientApiConnection::server_attach"); let server = { @@ -275,7 +315,10 @@ impl ClientApiConnection { .clone() }; let request = server.borrow().attach_request(); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? @@ -296,7 +339,10 @@ impl ClientApiConnection { .clone() }; let request = server.borrow().detach_request(); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? @@ -317,7 +363,10 @@ impl ClientApiConnection { .clone() }; let request = server.borrow().shutdown_request(); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; response.get().map(drop).map_err(map_to_string) } @@ -333,7 +382,10 @@ impl ClientApiConnection { }; let mut request = server.borrow().debug_request(); request.get().set_command(&what); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? @@ -361,7 +413,10 @@ impl ClientApiConnection { request.get().set_layer(&layer); let log_level_json = veilid_core::serialize_json(&log_level); request.get().set_log_level(&log_level_json); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? @@ -384,7 +439,10 @@ impl ClientApiConnection { let mut request = server.borrow().app_call_reply_request(); request.get().set_id(id); request.get().set_message(&msg); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index a9c3877e..ed5f3dd8 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -102,6 +102,12 @@ impl CommandProcessor { } } + pub fn cancel_command(&self) { + trace!("CommandProcessor::cancel_command"); + let capi = self.capi(); + capi.cancel(); + } + pub fn cmd_help(&self, _rest: Option, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_help"); self.ui().add_node_event( diff --git a/veilid-cli/src/ui.rs b/veilid-cli/src/ui.rs index e4ac5da9..b1c93065 100644 --- a/veilid-cli/src/ui.rs +++ b/veilid-cli/src/ui.rs @@ -306,11 +306,18 @@ impl UI { fn run_command(s: &mut Cursive, text: &str) -> Result<(), String> { // disable ui Self::enable_command_ui(s, false); + // run command + s.set_global_callback(cursive::event::Event::Key(Key::Esc), |s| { + let cmdproc = Self::command_processor(s); + cmdproc.cancel_command(); + }); + let cmdproc = Self::command_processor(s); cmdproc.run_command( text, Box::new(|s| { + s.set_global_callback(cursive::event::Event::Key(Key::Esc), UI::quit_handler); Self::enable_command_ui(s, true); }), ) diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 5f8783b9..a78a2026 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -12,9 +12,9 @@ path = "src/main.rs" [features] default = [ "rt-tokio" ] -rt-async-std = [ "veilid-core/rt-async-std", "async-std", "opentelemetry/rt-async-std", "opentelemetry-otlp/grpc-sys"] -rt-tokio = [ "veilid-core/rt-tokio", "tokio", "tokio-stream", "tokio-util", "opentelemetry/rt-tokio"] -tracking = ["veilid-core/tracking"] +rt-async-std = [ "veilid-core/rt-async-std", "async-std", "opentelemetry/rt-async-std", "opentelemetry-otlp/grpc-sys" ] +rt-tokio = [ "veilid-core/rt-tokio", "tokio", "tokio-stream", "tokio-util", "opentelemetry/rt-tokio", "console-subscriber" ] +tracking = [ "veilid-core/tracking" ] [dependencies] veilid-core = { path = "../veilid-core" } @@ -27,11 +27,13 @@ opentelemetry = { version = "^0" } opentelemetry-otlp = { version = "^0" } opentelemetry-semantic-conventions = "^0" async-std = { version = "^1", features = ["unstable"], optional = true } -tokio = { version = "^1", features = ["full"], optional = true } +tokio = { version = "^1", features = ["full", "tracing"], optional = true } +console-subscriber = { version = "^0", optional = true } tokio-stream = { version = "^0", features = ["net"], optional = true } tokio-util = { version = "^0", features = ["compat"], optional = true} async-tungstenite = { version = "^0", features = ["async-tls"] } color-eyre = { version = "^0", default-features = false } +backtrace = "^0" clap = "^3" directories = "^4" capnp = "^0" diff --git a/veilid-server/src/cmdline.rs b/veilid-server/src/cmdline.rs index f1048bbb..58364971 100644 --- a/veilid-server/src/cmdline.rs +++ b/veilid-server/src/cmdline.rs @@ -130,8 +130,20 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result EyreResult<(Settings, ArgMatches)> { }; settingsrw.core.network.bootstrap_nodes = bootstrap_list; } + + if matches.occurrences_of("console") != 0 { + settingsrw.logging.console.enabled = true; + } + drop(settingsrw); // Set specific config settings diff --git a/veilid-server/src/main.rs b/veilid-server/src/main.rs index d97b2666..3856731d 100644 --- a/veilid-server/src/main.rs +++ b/veilid-server/src/main.rs @@ -91,8 +91,23 @@ fn main() -> EyreResult<()> { } // --- Normal Startup --- + let panic_on_shutdown = matches.occurrences_of("panic") != 0; ctrlc::set_handler(move || { - shutdown(); + if panic_on_shutdown { + let orig_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + // invoke the default handler and exit the process + orig_hook(panic_info); + + let backtrace = backtrace::Backtrace::new(); + eprintln!("Backtrace:\n{:?}", backtrace); + + std::process::exit(1); + })); + panic!("panic requested"); + } else { + shutdown(); + } }) .expect("Error setting Ctrl-C handler"); diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index 6aed0875..1ada4a7c 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -41,6 +41,8 @@ logging: enabled: false level: 'trace' grpc_endpoint: 'localhost:4317' + console: + enabled: false testing: subnode_index: 0 core: @@ -416,6 +418,11 @@ pub struct Terminal { pub level: LogLevel, } +#[derive(Debug, Deserialize, Serialize)] +pub struct Console { + pub enabled: bool, +} + #[derive(Debug, Deserialize, Serialize)] pub struct File { pub enabled: bool, @@ -456,6 +463,7 @@ pub struct Logging { pub file: File, pub api: Api, pub otlp: Otlp, + pub console: Console, } #[derive(Debug, Deserialize, Serialize)] @@ -922,6 +930,7 @@ impl Settings { set_config_value!(inner.logging.otlp.enabled, value); set_config_value!(inner.logging.otlp.level, value); set_config_value!(inner.logging.otlp.grpc_endpoint, value); + set_config_value!(inner.logging.console.enabled, value); set_config_value!(inner.testing.subnode_index, value); set_config_value!(inner.core.protected_store.allow_insecure_fallback, value); set_config_value!( @@ -1443,6 +1452,7 @@ mod tests { s.logging.otlp.grpc_endpoint, NamedSocketAddrs::from_str("localhost:4317").unwrap() ); + assert_eq!(s.logging.console.enabled, false); assert_eq!(s.testing.subnode_index, 0); assert_eq!( diff --git a/veilid-server/src/veilid_logs.rs b/veilid-server/src/veilid_logs.rs index 7a08c835..494fda13 100644 --- a/veilid-server/src/veilid_logs.rs +++ b/veilid-server/src/veilid_logs.rs @@ -1,6 +1,8 @@ use crate::settings::*; use crate::*; use cfg_if::*; +#[cfg(feature = "rt-tokio")] +use console_subscriber::ConsoleLayer; use opentelemetry::sdk::*; use opentelemetry::*; use opentelemetry_otlp::WithExportConfig; @@ -36,6 +38,19 @@ impl VeilidLogs { // XXX: //layers.push(tracing_error::ErrorLayer::default().boxed()); + #[cfg(feature = "rt-tokio")] + if settingsr.logging.console.enabled { + let layer = ConsoleLayer::builder() + .with_default_env() + .spawn() + .with_filter( + filter::Targets::new() + .with_target("tokio", Level::TRACE) + .with_target("runtime", Level::TRACE), + ); + layers.push(layer.boxed()); + } + // Terminal logger if settingsr.logging.terminal.enabled { let filter = veilid_core::VeilidLayerFilter::new( From c1644f1015736d927c627847b336182a61b201d2 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 3 Nov 2022 22:02:40 -0400 Subject: [PATCH 36/67] bug fixes --- veilid-core/src/routing_table/mod.rs | 2 + veilid-core/src/routing_table/node_ref.rs | 491 ++++++++++-------- .../src/routing_table/node_ref_filter.rs | 61 +++ .../src/routing_table/routing_table_inner.rs | 10 +- veilid-core/src/veilid_api/mod.rs | 2 +- 5 files changed, 344 insertions(+), 222 deletions(-) create mode 100644 veilid-core/src/routing_table/node_ref_filter.rs diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index ec1508d1..b1d5c4fe 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -2,6 +2,7 @@ mod bucket; mod bucket_entry; mod debug; mod node_ref; +mod node_ref_filter; mod route_spec_store; mod routing_domain_editor; mod routing_domains; @@ -19,6 +20,7 @@ pub use bucket_entry::*; pub use debug::*; use hashlink::LruCache; pub use node_ref::*; +pub use node_ref_filter::*; pub use route_spec_store::*; pub use routing_domain_editor::*; pub use routing_domains::*; diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index c555c172..4b660037 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -6,67 +6,9 @@ use alloc::fmt; // We should ping them with some frequency and 30 seconds is typical timeout const CONNECTIONLESS_TIMEOUT_SECS: u32 = 29; -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct NodeRefFilter { - pub routing_domain_set: RoutingDomainSet, - pub dial_info_filter: DialInfoFilter, -} +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -impl Default for NodeRefFilter { - fn default() -> Self { - Self::new() - } -} - -impl NodeRefFilter { - pub fn new() -> Self { - Self { - routing_domain_set: RoutingDomainSet::all(), - dial_info_filter: DialInfoFilter::all(), - } - } - - pub fn with_routing_domain(mut self, routing_domain: RoutingDomain) -> Self { - self.routing_domain_set = routing_domain.into(); - self - } - pub fn with_routing_domain_set(mut self, routing_domain_set: RoutingDomainSet) -> Self { - self.routing_domain_set = routing_domain_set; - self - } - pub fn with_dial_info_filter(mut self, dial_info_filter: DialInfoFilter) -> Self { - self.dial_info_filter = dial_info_filter; - self - } - pub fn with_protocol_type(mut self, protocol_type: ProtocolType) -> Self { - self.dial_info_filter = self.dial_info_filter.with_protocol_type(protocol_type); - self - } - pub fn with_protocol_type_set(mut self, protocol_set: ProtocolTypeSet) -> Self { - self.dial_info_filter = self.dial_info_filter.with_protocol_type_set(protocol_set); - self - } - pub fn with_address_type(mut self, address_type: AddressType) -> Self { - self.dial_info_filter = self.dial_info_filter.with_address_type(address_type); - self - } - pub fn with_address_type_set(mut self, address_set: AddressTypeSet) -> Self { - self.dial_info_filter = self.dial_info_filter.with_address_type_set(address_set); - self - } - pub fn filtered(mut self, other_filter: &NodeRefFilter) -> Self { - self.routing_domain_set &= other_filter.routing_domain_set; - self.dial_info_filter = self - .dial_info_filter - .filtered(&other_filter.dial_info_filter); - self - } - pub fn is_dead(&self) -> bool { - self.dial_info_filter.is_dead() || self.routing_domain_set.is_empty() - } -} - -pub struct NodeRef { +pub struct NodeRefBaseCommon { routing_table: RoutingTable, node_id: DHTKey, entry: Arc, @@ -76,104 +18,79 @@ pub struct NodeRef { track_id: usize, } -impl NodeRef { - pub fn new( - routing_table: RoutingTable, - node_id: DHTKey, - entry: Arc, - filter: Option, - ) -> Self { - entry.ref_count.fetch_add(1u32, Ordering::Relaxed); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - Self { - routing_table, - node_id, - entry, - filter, - sequencing: Sequencing::NoPreference, - #[cfg(feature = "tracking")] - track_id: entry.track(), - } - } +pub trait NodeRefBase: Sized { + // Common field access + fn common(&self) -> &NodeRefBaseCommon; + fn common_mut(&mut self) -> &mut NodeRefBaseCommon; - // Operate on entry accessors - pub(super) fn operate(&self, f: F) -> T + // Implementation-specific operators + fn operate(&self, f: F) -> T where - F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, - { - let inner = &*self.routing_table.inner.read(); - self.entry.with(inner, f) - } - - pub(super) fn operate_mut(&self, f: F) -> T + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T; + fn operate_mut(&self, f: F) -> T where - F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, - { - let inner = &mut *self.routing_table.inner.write(); - self.entry.with_mut(inner, f) - } + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T; // Filtering - - pub fn filter_ref(&self) -> Option<&NodeRefFilter> { - self.filter.as_ref() + fn filter_ref(&self) -> Option<&NodeRefFilter> { + self.common().filter.as_ref() } - pub fn take_filter(&mut self) -> Option { - self.filter.take() + fn take_filter(&mut self) -> Option { + self.common_mut().filter.take() } - pub fn set_filter(&mut self, filter: Option) { - self.filter = filter + fn set_filter(&mut self, filter: Option) { + self.common_mut().filter = filter } - pub fn set_sequencing(&mut self, sequencing: Sequencing) { - self.sequencing = sequencing; + fn set_sequencing(&mut self, sequencing: Sequencing) { + self.common_mut().sequencing = sequencing; } - pub fn sequencing(&self) -> Sequencing { - self.sequencing + fn sequencing(&self) -> Sequencing { + self.common().sequencing } - pub fn merge_filter(&mut self, filter: NodeRefFilter) { - if let Some(self_filter) = self.filter.take() { - self.filter = Some(self_filter.filtered(&filter)); + fn merge_filter(&mut self, filter: NodeRefFilter) { + let common_mut = self.common_mut(); + if let Some(self_filter) = common_mut.filter.take() { + common_mut.filter = Some(self_filter.filtered(&filter)); } else { - self.filter = Some(filter); + common_mut.filter = Some(filter); } } - pub fn filtered_clone(&self, filter: NodeRefFilter) -> Self { - let mut out = self.clone(); - out.merge_filter(filter); - out - } - - pub fn is_filter_dead(&self) -> bool { - if let Some(filter) = &self.filter { + fn is_filter_dead(&self) -> bool { + if let Some(filter) = &self.common().filter { filter.is_dead() } else { false } } - pub fn routing_domain_set(&self) -> RoutingDomainSet { - self.filter + fn routing_domain_set(&self) -> RoutingDomainSet { + self.common() + .filter .as_ref() .map(|f| f.routing_domain_set) .unwrap_or(RoutingDomainSet::all()) } - pub fn dial_info_filter(&self) -> DialInfoFilter { - self.filter + fn dial_info_filter(&self) -> DialInfoFilter { + self.common() + .filter .as_ref() .map(|f| f.dial_info_filter.clone()) .unwrap_or(DialInfoFilter::all()) } - pub fn best_routing_domain(&self) -> Option { + fn best_routing_domain(&self) -> Option { self.operate(|_rti, e| { e.best_routing_domain( - self.filter + self.common() + .filter .as_ref() .map(|f| f.routing_domain_set) .unwrap_or(RoutingDomainSet::all()), @@ -182,66 +99,66 @@ impl NodeRef { } // Accessors - pub fn routing_table(&self) -> RoutingTable { - self.routing_table.clone() + fn routing_table(&self) -> RoutingTable { + self.common().routing_table.clone() } - pub fn node_id(&self) -> DHTKey { - self.node_id + fn node_id(&self) -> DHTKey { + self.common().node_id } - pub fn has_updated_since_last_network_change(&self) -> bool { + fn has_updated_since_last_network_change(&self) -> bool { self.operate(|_rti, e| e.has_updated_since_last_network_change()) } - pub fn set_updated_since_last_network_change(&self) { + fn set_updated_since_last_network_change(&self) { self.operate_mut(|_rti, e| e.set_updated_since_last_network_change(true)); } - pub fn update_node_status(&self, node_status: NodeStatus) { + fn update_node_status(&self, node_status: NodeStatus) { self.operate_mut(|_rti, e| { e.update_node_status(node_status); }); } - pub fn min_max_version(&self) -> Option<(u8, u8)> { + fn min_max_version(&self) -> Option<(u8, u8)> { self.operate(|_rti, e| e.min_max_version()) } - pub fn set_min_max_version(&self, min_max_version: (u8, u8)) { + fn set_min_max_version(&self, min_max_version: (u8, u8)) { self.operate_mut(|_rti, e| e.set_min_max_version(min_max_version)) } - pub fn state(&self, cur_ts: u64) -> BucketEntryState { + fn state(&self, cur_ts: u64) -> BucketEntryState { self.operate(|_rti, e| e.state(cur_ts)) } - pub fn peer_stats(&self) -> PeerStats { + fn peer_stats(&self) -> PeerStats { self.operate(|_rti, e| e.peer_stats().clone()) } // Per-RoutingDomain accessors - pub fn make_peer_info(&self, routing_domain: RoutingDomain) -> Option { + fn make_peer_info(&self, routing_domain: RoutingDomain) -> Option { self.operate(|_rti, e| e.make_peer_info(self.node_id(), routing_domain)) } - pub fn node_info(&self, routing_domain: RoutingDomain) -> Option { + fn node_info(&self, routing_domain: RoutingDomain) -> Option { self.operate(|_rti, e| e.node_info(routing_domain).cloned()) } - pub fn signed_node_info_has_valid_signature(&self, routing_domain: RoutingDomain) -> bool { + fn signed_node_info_has_valid_signature(&self, routing_domain: RoutingDomain) -> bool { self.operate(|_rti, e| { e.signed_node_info(routing_domain) .map(|sni| sni.has_valid_signature()) .unwrap_or(false) }) } - pub fn has_seen_our_node_info(&self, routing_domain: RoutingDomain) -> bool { + fn has_seen_our_node_info(&self, routing_domain: RoutingDomain) -> bool { self.operate(|_rti, e| e.has_seen_our_node_info(routing_domain)) } - pub fn set_seen_our_node_info(&self, routing_domain: RoutingDomain) { + fn set_seen_our_node_info(&self, routing_domain: RoutingDomain) { self.operate_mut(|_rti, e| e.set_seen_our_node_info(routing_domain, true)); } - pub fn network_class(&self, routing_domain: RoutingDomain) -> Option { + fn network_class(&self, routing_domain: RoutingDomain) -> Option { self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.network_class)) } - pub fn outbound_protocols(&self, routing_domain: RoutingDomain) -> Option { + fn outbound_protocols(&self, routing_domain: RoutingDomain) -> Option { self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.outbound_protocols)) } - pub fn address_types(&self, routing_domain: RoutingDomain) -> Option { + fn address_types(&self, routing_domain: RoutingDomain) -> Option { self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.address_types)) } - pub fn node_info_outbound_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { + fn node_info_outbound_filter(&self, routing_domain: RoutingDomain) -> DialInfoFilter { let mut dif = DialInfoFilter::all(); if let Some(outbound_protocols) = self.outbound_protocols(routing_domain) { dif = dif.with_protocol_type_set(outbound_protocols); @@ -251,34 +168,37 @@ impl NodeRef { } dif } - pub fn relay(&self, routing_domain: RoutingDomain) -> Option { - let target_rpi = self.operate(|_rti, e| { - e.node_info(routing_domain) + fn relay(&self, routing_domain: RoutingDomain) -> Option { + self.operate_mut(|rti, e| { + let opt_target_rpi = e + .node_info(routing_domain) .map(|n| n.relay_peer_info.as_ref().map(|pi| pi.as_ref().clone())) - })?; - target_rpi.and_then(|t| { - // If relay is ourselves, then return None, because we can't relay through ourselves - // and to contact this node we should have had an existing inbound connection - if t.node_id.key == self.routing_table.node_id() { - return None; - } + .flatten(); + opt_target_rpi.and_then(|t| { + // If relay is ourselves, then return None, because we can't relay through ourselves + // and to contact this node we should have had an existing inbound connection + if t.node_id.key == rti.unlocked_inner.node_id { + return None; + } - // Register relay node and return noderef - self.routing_table.register_node_with_signed_node_info( - routing_domain, - t.node_id.key, - t.signed_node_info, - false, - ) + // Register relay node and return noderef + rti.register_node_with_signed_node_info( + self.routing_table(), + routing_domain, + t.node_id.key, + t.signed_node_info, + false, + ) + }) }) } // Filtered accessors - pub fn first_filtered_dial_info_detail(&self) -> Option { + fn first_filtered_dial_info_detail(&self) -> Option { let routing_domain_set = self.routing_domain_set(); let dial_info_filter = self.dial_info_filter(); - let (sort, dial_info_filter) = match self.sequencing { + let (sort, dial_info_filter) = match self.common().sequencing { Sequencing::NoPreference => (None, dial_info_filter), Sequencing::PreferOrdered => ( Some(DialInfoDetail::ordered_sequencing_sort), @@ -305,11 +225,11 @@ impl NodeRef { }) } - pub fn all_filtered_dial_info_details(&self) -> Vec { + fn all_filtered_dial_info_details(&self) -> Vec { let routing_domain_set = self.routing_domain_set(); let dial_info_filter = self.dial_info_filter(); - let (sort, dial_info_filter) = match self.sequencing { + let (sort, dial_info_filter) = match self.common().sequencing { Sequencing::NoPreference => (None, dial_info_filter), Sequencing::PreferOrdered => ( Some(DialInfoDetail::ordered_sequencing_sort), @@ -338,43 +258,47 @@ impl NodeRef { out } - pub fn last_connection(&self) -> Option { + fn last_connection(&self) -> Option { // Get the last connections and the last time we saw anything with this connection // Filtered first and then sorted by most recent - let last_connections = self.operate(|rti, e| e.last_connections(rti, self.filter.clone())); + self.operate(|rti, e| { + let last_connections = e.last_connections(rti, self.common().filter.clone()); - // Do some checks to ensure these are possibly still 'live' - for (last_connection, last_seen) in last_connections { - // Should we check the connection table? - if last_connection.protocol_type().is_connection_oriented() { - // Look the connection up in the connection manager and see if it's still there - let connection_manager = self.routing_table.network_manager().connection_manager(); - if connection_manager.get_connection(last_connection).is_some() { - return Some(last_connection); - } - } else { - // If this is not connection oriented, then we check our last seen time - // to see if this mapping has expired (beyond our timeout) - let cur_ts = intf::get_timestamp(); - if (last_seen + (CONNECTIONLESS_TIMEOUT_SECS as u64 * 1_000_000u64)) >= cur_ts { - return Some(last_connection); + // Do some checks to ensure these are possibly still 'live' + for (last_connection, last_seen) in last_connections { + // Should we check the connection table? + if last_connection.protocol_type().is_connection_oriented() { + // Look the connection up in the connection manager and see if it's still there + let connection_manager = + rti.unlocked_inner.network_manager.connection_manager(); + if connection_manager.get_connection(last_connection).is_some() { + return Some(last_connection); + } + } else { + // If this is not connection oriented, then we check our last seen time + // to see if this mapping has expired (beyond our timeout) + let cur_ts = intf::get_timestamp(); + if (last_seen + (CONNECTIONLESS_TIMEOUT_SECS as u64 * 1_000_000u64)) >= cur_ts { + return Some(last_connection); + } } } - } - None + None + }) } - pub fn clear_last_connections(&self) { + fn clear_last_connections(&self) { self.operate_mut(|_rti, e| e.clear_last_connections()) } - pub fn set_last_connection(&self, connection_descriptor: ConnectionDescriptor, ts: u64) { - self.operate_mut(|_rti, e| e.set_last_connection(connection_descriptor, ts)); - self.routing_table - .touch_recent_peer(self.node_id(), connection_descriptor); + fn set_last_connection(&self, connection_descriptor: ConnectionDescriptor, ts: u64) { + self.operate_mut(|rti, e| { + e.set_last_connection(connection_descriptor, ts); + rti.touch_recent_peer(self.common().node_id, connection_descriptor); + }) } - pub fn has_any_dial_info(&self) -> bool { + fn has_any_dial_info(&self) -> bool { self.operate(|_rti, e| { for rtd in RoutingDomain::all() { if let Some(ni) = e.node_info(rtd) { @@ -387,25 +311,25 @@ impl NodeRef { }) } - pub fn stats_question_sent(&self, ts: u64, bytes: u64, expects_answer: bool) { + fn stats_question_sent(&self, ts: u64, bytes: u64, expects_answer: bool) { self.operate_mut(|rti, e| { rti.transfer_stats_accounting().add_up(bytes); e.question_sent(ts, bytes, expects_answer); }) } - pub fn stats_question_rcvd(&self, ts: u64, bytes: u64) { + fn stats_question_rcvd(&self, ts: u64, bytes: u64) { self.operate_mut(|rti, e| { rti.transfer_stats_accounting().add_down(bytes); e.question_rcvd(ts, bytes); }) } - pub fn stats_answer_sent(&self, bytes: u64) { + fn stats_answer_sent(&self, bytes: u64) { self.operate_mut(|rti, e| { rti.transfer_stats_accounting().add_up(bytes); e.answer_sent(bytes); }) } - pub fn stats_answer_rcvd(&self, send_ts: u64, recv_ts: u64, bytes: u64) { + fn stats_answer_rcvd(&self, send_ts: u64, recv_ts: u64, bytes: u64) { self.operate_mut(|rti, e| { rti.transfer_stats_accounting().add_down(bytes); rti.latency_stats_accounting() @@ -413,54 +337,118 @@ impl NodeRef { e.answer_rcvd(send_ts, recv_ts, bytes); }) } - pub fn stats_question_lost(&self) { + fn stats_question_lost(&self) { self.operate_mut(|_rti, e| { e.question_lost(); }) } - pub fn stats_failed_to_send(&self, ts: u64, expects_answer: bool) { + fn stats_failed_to_send(&self, ts: u64, expects_answer: bool) { self.operate_mut(|_rti, e| { e.failed_to_send(ts, expects_answer); }) } } -impl Clone for NodeRef { - fn clone(&self) -> Self { - self.entry.ref_count.fetch_add(1u32, Ordering::Relaxed); +//////////////////////////////////////////////////////////////////////////////////// + +/// Reference to a routing table entry +/// Keeps entry in the routing table until all references are gone +pub struct NodeRef { + common: NodeRefBaseCommon, +} + +impl NodeRef { + pub fn new( + routing_table: RoutingTable, + node_id: DHTKey, + entry: Arc, + filter: Option, + ) -> Self { + entry.ref_count.fetch_add(1u32, Ordering::Relaxed); Self { - routing_table: self.routing_table.clone(), - node_id: self.node_id, - entry: self.entry.clone(), - filter: self.filter.clone(), - sequencing: self.sequencing, - #[cfg(feature = "tracking")] - track_id: e.track(), + common: NodeRefBaseCommon { + routing_table, + node_id, + entry, + filter, + sequencing: Sequencing::NoPreference, + #[cfg(feature = "tracking")] + track_id: entry.track(), + }, + } + } + + pub fn filtered_clone(&self, filter: NodeRefFilter) -> Self { + let mut out = self.clone(); + out.merge_filter(filter); + out + } + + pub fn locked<'a>(&self, rti: &'a mut RoutingTableInner) -> NodeRefLocked<'a> { + NodeRefLocked::new(rti, self.clone()) + } +} + +impl NodeRefBase for NodeRef { + fn common(&self) -> &NodeRefBaseCommon { + &self.common + } + + fn common_mut(&mut self) -> &mut NodeRefBaseCommon { + &mut self.common + } + + fn operate(&self, f: F) -> T + where + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, + { + let inner = &*self.common.routing_table.inner.read(); + self.common.entry.with(inner, f) + } + + fn operate_mut(&self, f: F) -> T + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, + { + let inner = &mut *self.common.routing_table.inner.write(); + self.common.entry.with_mut(inner, f) + } +} + +impl Clone for NodeRef { + fn clone(&self) -> Self { + self.common + .entry + .ref_count + .fetch_add(1u32, Ordering::Relaxed); + + Self { + common: NodeRefBaseCommon { + routing_table: self.common.routing_table.clone(), + node_id: self.common.node_id, + entry: self.common.entry.clone(), + filter: self.common.filter.clone(), + sequencing: self.common.sequencing, + #[cfg(feature = "tracking")] + track_id: self.common.entry.write().track(), + }, } } } -// impl PartialEq for NodeRef { -// fn eq(&self, other: &Self) -> bool { -// self.node_id == other.node_id -// } -// } - -// impl Eq for NodeRef {} - impl fmt::Display for NodeRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.node_id.encode()) + write!(f, "{}", self.common.node_id.encode()) } } impl fmt::Debug for NodeRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("NodeRef") - .field("node_id", &self.node_id) - .field("filter", &self.filter) - .field("sequencing", &self.sequencing) + .field("node_id", &self.common.node_id) + .field("filter", &self.common.filter) + .field("sequencing", &self.common.sequencing) .finish() } } @@ -468,12 +456,79 @@ impl fmt::Debug for NodeRef { impl Drop for NodeRef { fn drop(&mut self) { #[cfg(feature = "tracking")] - self.operate(|e| e.untrack(self.track_id)); + self.common.entry.write().untrack(self.track_id); // drop the noderef and queue a bucket kick if it was the last one - let new_ref_count = self.entry.ref_count.fetch_sub(1u32, Ordering::Relaxed) - 1; + let new_ref_count = self + .common + .entry + .ref_count + .fetch_sub(1u32, Ordering::Relaxed) + - 1; if new_ref_count == 0 { - self.routing_table.queue_bucket_kick(self.node_id); + self.common + .routing_table + .queue_bucket_kick(self.common.node_id); } } } + +//////////////////////////////////////////////////////////////////////////////////// + +/// Locked reference to a routing table entry +/// For internal use inside the RoutingTable module where you have +/// already locked a RoutingTableInner +/// Keeps entry in the routing table until all references are gone +pub struct NodeRefLocked<'a> { + inner: Mutex<&'a mut RoutingTableInner>, + nr: NodeRef, +} + +impl<'a> NodeRefLocked<'a> { + pub fn new(inner: &'a mut RoutingTableInner, nr: NodeRef) -> Self { + Self { + inner: Mutex::new(inner), + nr, + } + } +} + +impl<'a> NodeRefBase for NodeRefLocked<'a> { + fn common(&self) -> &NodeRefBaseCommon { + &self.nr.common + } + + fn common_mut(&mut self) -> &mut NodeRefBaseCommon { + &mut self.nr.common + } + + fn operate(&self, f: F) -> T + where + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, + { + let inner = &*self.inner.lock(); + self.nr.common.entry.with(inner, f) + } + + fn operate_mut(&self, f: F) -> T + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, + { + let inner = &mut *self.inner.lock(); + self.nr.common.entry.with_mut(inner, f) + } +} + +impl<'a> fmt::Display for NodeRefLocked<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.nr) + } +} + +impl<'a> fmt::Debug for NodeRefLocked<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NodeRefLocked") + .field("nr", &self.nr) + .finish() + } +} diff --git a/veilid-core/src/routing_table/node_ref_filter.rs b/veilid-core/src/routing_table/node_ref_filter.rs new file mode 100644 index 00000000..934d93a1 --- /dev/null +++ b/veilid-core/src/routing_table/node_ref_filter.rs @@ -0,0 +1,61 @@ +use super::*; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct NodeRefFilter { + pub routing_domain_set: RoutingDomainSet, + pub dial_info_filter: DialInfoFilter, +} + +impl Default for NodeRefFilter { + fn default() -> Self { + Self::new() + } +} + +impl NodeRefFilter { + pub fn new() -> Self { + Self { + routing_domain_set: RoutingDomainSet::all(), + dial_info_filter: DialInfoFilter::all(), + } + } + + pub fn with_routing_domain(mut self, routing_domain: RoutingDomain) -> Self { + self.routing_domain_set = routing_domain.into(); + self + } + pub fn with_routing_domain_set(mut self, routing_domain_set: RoutingDomainSet) -> Self { + self.routing_domain_set = routing_domain_set; + self + } + pub fn with_dial_info_filter(mut self, dial_info_filter: DialInfoFilter) -> Self { + self.dial_info_filter = dial_info_filter; + self + } + pub fn with_protocol_type(mut self, protocol_type: ProtocolType) -> Self { + self.dial_info_filter = self.dial_info_filter.with_protocol_type(protocol_type); + self + } + pub fn with_protocol_type_set(mut self, protocol_set: ProtocolTypeSet) -> Self { + self.dial_info_filter = self.dial_info_filter.with_protocol_type_set(protocol_set); + self + } + pub fn with_address_type(mut self, address_type: AddressType) -> Self { + self.dial_info_filter = self.dial_info_filter.with_address_type(address_type); + self + } + pub fn with_address_type_set(mut self, address_set: AddressTypeSet) -> Self { + self.dial_info_filter = self.dial_info_filter.with_address_type_set(address_set); + self + } + pub fn filtered(mut self, other_filter: &NodeRefFilter) -> Self { + self.routing_domain_set &= other_filter.routing_domain_set; + self.dial_info_filter = self + .dial_info_filter + .filtered(&other_filter.dial_info_filter); + self + } + pub fn is_dead(&self) -> bool { + self.dial_info_filter.is_dead() || self.routing_domain_set.is_empty() + } +} diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index 36c7240e..cab84df6 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -720,7 +720,7 @@ impl RoutingTableInner { }); if let Some(nr) = &out { // set the most recent node address for connection finding and udp replies - nr.set_last_connection(descriptor, timestamp); + nr.locked(self).set_last_connection(descriptor, timestamp); } out } @@ -841,12 +841,16 @@ impl RoutingTableInner { Vec::<(DHTKey, Option>)>::with_capacity(self.bucket_entry_count + 1); // add our own node (only one of there with the None entry) + let mut filtered = false; for filter in &mut filters { - if filter(self, self.unlocked_inner.node_id, None) { - nodes.push((self.unlocked_inner.node_id, None)); + if !filter(self, self.unlocked_inner.node_id, None) { + filtered = true; break; } } + if !filtered { + nodes.push((self.unlocked_inner.node_id, None)); + } // add all nodes from buckets self.with_entries(cur_ts, BucketEntryState::Unreliable, |rti, k, v| { diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index e30ef957..b28bc49b 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -25,7 +25,7 @@ pub use intf::BlockStore; pub use intf::ProtectedStore; pub use intf::TableStore; pub use network_manager::NetworkManager; -pub use routing_table::RoutingTable; +pub use routing_table::{NodeRef, NodeRefBase, RoutingTable}; use core::fmt; use core_context::{api_shutdown, VeilidCoreContext}; From 9f917af7675f68f913ef05e374ea21bff7ad9880 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 3 Nov 2022 22:52:18 -0400 Subject: [PATCH 37/67] trace fix --- veilid-core/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/lib.rs b/veilid-core/src/lib.rs index 1acea791..adc8d656 100644 --- a/veilid-core/src/lib.rs +++ b/veilid-core/src/lib.rs @@ -64,12 +64,14 @@ pub fn veilid_version() -> (u32, u32, u32) { #[cfg(target_os = "android")] pub use intf::utils::android::{veilid_core_setup_android, veilid_core_setup_android_no_log}; -pub static DEFAULT_LOG_IGNORE_LIST: [&str; 19] = [ +pub static DEFAULT_LOG_IGNORE_LIST: [&str; 21] = [ "mio", "h2", "hyper", "tower", "tonic", + "tokio", + "runtime", "tokio_util", "want", "serial_test", From 60c46485301b6312e2d502571e47842059b60e04 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 4 Nov 2022 12:58:13 -0400 Subject: [PATCH 38/67] fix signed node info --- veilid-core/src/routing_table/bucket_entry.rs | 29 ++++++++++--------- veilid-core/src/rpc_processor/rpc_status.rs | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index f858fbd7..6bad7999 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -181,20 +181,23 @@ impl BucketEntryInner { // See if we have an existing signed_node_info to update or not if let Some(current_sni) = opt_current_sni { - // If the timestamp hasn't changed or is less, ignore this update - if signed_node_info.timestamp <= current_sni.timestamp { - // If we received a node update with the same timestamp - // we can make this node live again, but only if our network has recently changed - // which may make nodes that were unreachable now reachable with the same dialinfo - if !self.updated_since_last_network_change - && signed_node_info.timestamp == current_sni.timestamp - { - // No need to update the signednodeinfo though since the timestamp is the same - // Touch the node and let it try to live again - self.updated_since_last_network_change = true; - self.touch_last_seen(intf::get_timestamp()); + // Always allow overwriting invalid/unsigned node + if current_sni.has_valid_signature() { + // If the timestamp hasn't changed or is less, ignore this update + if signed_node_info.timestamp <= current_sni.timestamp { + // If we received a node update with the same timestamp + // we can make this node live again, but only if our network has recently changed + // which may make nodes that were unreachable now reachable with the same dialinfo + if !self.updated_since_last_network_change + && signed_node_info.timestamp == current_sni.timestamp + { + // No need to update the signednodeinfo though since the timestamp is the same + // Touch the node and let it try to live again + self.updated_since_last_network_change = true; + self.touch_last_seen(intf::get_timestamp()); + } + return; } - return; } } diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index ab8b83d8..6725ddfa 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -149,7 +149,6 @@ impl RPCProcessor { ) } } - opt_sender_info = Some(sender_info.clone()); } SendDataKind::Indirect => { // Do nothing in this case, as the socket address returned here would be for any node other than ours @@ -158,6 +157,7 @@ impl RPCProcessor { // Do nothing in this case, as an existing connection could not have a different public address or it would have been reset } }; + opt_sender_info = Some(sender_info.clone()); } } } From ed0049dc22713a31842893dbab2695cf3871a4d1 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 4 Nov 2022 19:29:44 -0400 Subject: [PATCH 39/67] xfer --- .../src/intf/native/protected_store.rs | 59 ++-- veilid-core/src/intf/wasm/protected_store.rs | 139 +++++---- veilid-core/src/network_manager/mod.rs | 2 +- veilid-core/src/network_manager/native/mod.rs | 10 +- .../native/network_class_discovery.rs | 6 +- veilid-core/src/network_manager/wasm/mod.rs | 17 +- .../src/routing_table/route_spec_store.rs | 133 +++++---- veilid-core/src/rpc_processor/mod.rs | 8 +- veilid-core/src/veilid_api/debug.rs | 265 ++++++++++++++++++ 9 files changed, 497 insertions(+), 142 deletions(-) diff --git a/veilid-core/src/intf/native/protected_store.rs b/veilid-core/src/intf/native/protected_store.rs index f4452cda..d88e31fb 100644 --- a/veilid-core/src/intf/native/protected_store.rs +++ b/veilid-core/src/intf/native/protected_store.rs @@ -2,6 +2,7 @@ use crate::xx::*; use crate::*; use data_encoding::BASE64URL_NOPAD; use keyring_manager::*; +use serde::{Deserialize, Serialize}; use std::path::Path; pub struct ProtectedStoreInner { @@ -31,15 +32,18 @@ impl ProtectedStore { #[instrument(level = "trace", skip(self), err)] pub async fn delete_all(&self) -> EyreResult<()> { // Delete all known keys - if self.remove_user_secret_string("node_id").await? { + if self.remove_user_secret("node_id").await? { debug!("deleted protected_store key 'node_id'"); } - if self.remove_user_secret_string("node_id_secret").await? { + if self.remove_user_secret("node_id_secret").await? { debug!("deleted protected_store key 'node_id_secret'"); } - if self.remove_user_secret_string("_test_key").await? { + if self.remove_user_secret("_test_key").await? { debug!("deleted protected_store key '_test_key'"); } + if self.remove_user_secret("RouteSpecStore").await? { + debug!("deleted protected_store key 'RouteSpecStore'"); + } Ok(()) } @@ -139,19 +143,30 @@ impl ProtectedStore { } } - #[instrument(level = "trace", skip(self), ret, err)] - pub async fn remove_user_secret_string(&self, key: &str) -> EyreResult { - let inner = self.inner.lock(); - match inner - .keyring_manager - .as_ref() - .ok_or_else(|| eyre!("Protected store not initialized"))? - .with_keyring(&self.service_name(), key, |kr| kr.delete_value()) - { - Ok(_) => Ok(true), - Err(KeyringError::NoPasswordFound) => Ok(false), - Err(e) => Err(eyre!("Failed to remove user secret: {}", e)), - } + #[instrument(level = "trace", skip(self, value), ret, err)] + pub async fn save_user_secret_cbor(&self, key: &str, value: &T) -> EyreResult + where + T: Serialize, + { + let v = serde_cbor::to_vec(value).wrap_err("couldn't store as CBOR")?; + self.save_user_secret(&key, &v).await + } + + #[instrument(level = "trace", skip(self), err)] + pub async fn load_user_secret_cbor(&self, key: &str) -> EyreResult> + where + T: for<'de> Deserialize<'de>, + { + let out = self.load_user_secret(key).await?; + let b = match out { + Some(v) => v, + None => { + return Ok(None); + } + }; + + let obj = serde_cbor::from_slice::(&b).wrap_err("failed to deserialize")?; + Ok(Some(obj)) } #[instrument(level = "trace", skip(self, value), ret, err)] @@ -195,6 +210,16 @@ impl ProtectedStore { #[instrument(level = "trace", skip(self), ret, err)] pub async fn remove_user_secret(&self, key: &str) -> EyreResult { - self.remove_user_secret_string(key).await + let inner = self.inner.lock(); + match inner + .keyring_manager + .as_ref() + .ok_or_else(|| eyre!("Protected store not initialized"))? + .with_keyring(&self.service_name(), key, |kr| kr.delete_value()) + { + Ok(_) => Ok(true), + Err(KeyringError::NoPasswordFound) => Ok(false), + Err(e) => Err(eyre!("Failed to remove user secret: {}", e)), + } } } diff --git a/veilid-core/src/intf/wasm/protected_store.rs b/veilid-core/src/intf/wasm/protected_store.rs index 51a7619b..ab527dbf 100644 --- a/veilid-core/src/intf/wasm/protected_store.rs +++ b/veilid-core/src/intf/wasm/protected_store.rs @@ -17,38 +17,40 @@ extern "C" { fn keytar_deletePassword(service: &str, account: &str) -> Result; } - #[derive(Clone)] pub struct ProtectedStore { config: VeilidConfig, } impl ProtectedStore { - pub fn new(config: VeilidConfig) -> Self { - Self { - config, - } + Self { config } } + #[instrument(level = "trace", skip(self), err)] pub async fn delete_all(&self) -> EyreResult<()> { // Delete all known keys - if self.remove_user_secret_string("node_id").await? { + if self.remove_user_secret("node_id").await? { debug!("deleted protected_store key 'node_id'"); } - if self.remove_user_secret_string("node_id_secret").await? { + if self.remove_user_secret("node_id_secret").await? { debug!("deleted protected_store key 'node_id_secret'"); } - if self.remove_user_secret_string("_test_key").await? { + if self.remove_user_secret("_test_key").await? { debug!("deleted protected_store key '_test_key'"); } + if self.remove_user_secret("RouteSpecStore").await? { + debug!("deleted protected_store key 'RouteSpecStore'"); + } Ok(()) } + #[instrument(level = "debug", skip(self), err)] pub async fn init(&self) -> EyreResult<()> { Ok(()) } + #[instrument(level = "debug", skip(self))] pub async fn terminate(&self) {} fn keyring_name(&self) -> String { @@ -69,12 +71,13 @@ impl ProtectedStore { } } + #[instrument(level = "trace", skip(self, value), ret, err)] pub async fn save_user_secret_string(&self, key: &str, value: &str) -> EyreResult { if is_nodejs() { let prev = match JsFuture::from( keytar_getPassword(self.keyring_name().as_str(), key) - .map_err(map_jsvalue_error) - .wrap_err("exception thrown")?, + .map_err(map_jsvalue_error) + .wrap_err("exception thrown")?, ) .await { @@ -84,8 +87,8 @@ impl ProtectedStore { match JsFuture::from( keytar_setPassword(self.keyring_name().as_str(), key, value) - .map_err(map_jsvalue_error) - .wrap_err("exception thrown")?, + .map_err(map_jsvalue_error) + .wrap_err("exception thrown")?, ) .await { @@ -134,6 +137,7 @@ impl ProtectedStore { } } + #[instrument(level = "trace", skip(self), err)] pub async fn load_user_secret_string(&self, key: &str) -> EyreResult> { if is_nodejs() { let prev = match JsFuture::from( @@ -181,7 +185,73 @@ impl ProtectedStore { } } - pub async fn remove_user_secret_string(&self, key: &str) -> EyreResult { + #[instrument(level = "trace", skip(self, value), ret, err)] + pub async fn save_user_secret_cbor(&self, key: &str, value: &T) -> EyreResult + where + T: Serialize, + { + let v = serde_cbor::to_vec(value).wrap_err("couldn't store as CBOR")?; + self.save_user_secret(&key, &v).await + } + + #[instrument(level = "trace", skip(self), err)] + pub async fn load_user_secret_cbor(&self, key: &str) -> EyreResult> + where + T: for<'de> Deserialize<'de>, + { + let out = self.load_user_secret(key).await?; + let b = match out { + Some(v) => v, + None => { + return Ok(None); + } + }; + + let obj = serde_cbor::from_slice::(&b).wrap_err("failed to deserialize")?; + Ok(Some(obj)) + } + + #[instrument(level = "trace", skip(self, value), ret, err)] + pub async fn save_user_secret(&self, key: &str, value: &[u8]) -> EyreResult { + let mut s = BASE64URL_NOPAD.encode(value); + s.push('!'); + + self.save_user_secret_string(key, s.as_str()).await + } + + #[instrument(level = "trace", skip(self), err)] + pub async fn load_user_secret(&self, key: &str) -> EyreResult>> { + let mut s = match self.load_user_secret_string(key).await? { + Some(s) => s, + None => { + return Ok(None); + } + }; + + if s.pop() != Some('!') { + bail!("User secret is not a buffer"); + } + + let mut bytes = Vec::::new(); + let res = BASE64URL_NOPAD.decode_len(s.len()); + match res { + Ok(l) => { + bytes.resize(l, 0u8); + } + Err(_) => { + bail!("Failed to decode"); + } + } + + let res = BASE64URL_NOPAD.decode_mut(s.as_bytes(), &mut bytes); + match res { + Ok(_) => Ok(Some(bytes)), + Err(_) => bail!("Failed to decode"), + } + } + + #[instrument(level = "trace", skip(self), ret, err)] + pub async fn remove_user_secret(&self, key: &str) -> EyreResult { if is_nodejs() { match JsFuture::from( keytar_deletePassword(self.keyring_name().as_str(), key) @@ -231,45 +301,4 @@ impl ProtectedStore { unimplemented!(); } } - - pub async fn save_user_secret(&self, key: &str, value: &[u8]) -> EyreResult { - let mut s = BASE64URL_NOPAD.encode(value); - s.push('!'); - - self.save_user_secret_string(key, s.as_str()).await - } - - pub async fn load_user_secret(&self, key: &str) -> EyreResult>> { - let mut s = match self.load_user_secret_string(key).await? { - Some(s) => s, - None => { - return Ok(None); - } - }; - - if s.pop() != Some('!') { - bail!("User secret is not a buffer"); - } - - let mut bytes = Vec::::new(); - let res = BASE64URL_NOPAD.decode_len(s.len()); - match res { - Ok(l) => { - bytes.resize(l, 0u8); - } - Err(_) => { - bail!("Failed to decode"); - } - } - - let res = BASE64URL_NOPAD.decode_mut(s.as_bytes(), &mut bytes); - match res { - Ok(_) => Ok(Some(bytes)), - Err(_) => bail!("Failed to decode"), - } - } - - pub async fn remove_user_secret(&self, key: &str) -> EyreResult { - self.remove_user_secret_string(key).await - } -} \ No newline at end of file +} diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index a7166f5e..21d0f5eb 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1747,7 +1747,7 @@ impl NetworkManager { // Ignore these reports if we are currently detecting public dial info let net = self.net(); - if net.doing_public_dial_info_check() { + if net.needs_public_dial_info_check() { return; } diff --git a/veilid-core/src/network_manager/native/mod.rs b/veilid-core/src/network_manager/native/mod.rs index 7994a752..f13e3fb7 100644 --- a/veilid-core/src/network_manager/native/mod.rs +++ b/veilid-core/src/network_manager/native/mod.rs @@ -63,8 +63,6 @@ struct NetworkInner { enable_ipv6_local: bool, /// set if we need to calculate our public dial info again needs_public_dial_info_check: bool, - /// set during the actual execution of the public dial info check to ensure we don't do it more than once - doing_public_dial_info_check: bool, /// the punishment closure to enax public_dial_info_check_punishment: Option>, /// udp socket record for bound-first sockets, which are used to guarantee a port is available before @@ -116,7 +114,6 @@ impl Network { network_started: false, network_needs_restart: false, needs_public_dial_info_check: false, - doing_public_dial_info_check: false, public_dial_info_check_punishment: None, protocol_config: Default::default(), static_public_dialinfo: ProtocolTypeSet::empty(), @@ -871,16 +868,11 @@ impl Network { inner.public_dial_info_check_punishment = punishment; } - fn needs_public_dial_info_check(&self) -> bool { + pub fn needs_public_dial_info_check(&self) -> bool { let inner = self.inner.lock(); inner.needs_public_dial_info_check } - pub fn doing_public_dial_info_check(&self) -> bool { - let inner = self.inner.lock(); - inner.doing_public_dial_info_check - } - ////////////////////////////////////////// #[instrument(level = "trace", skip(self), err)] diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/network_class_discovery.rs index c24ec733..2b8b8aca 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/network_class_discovery.rs @@ -883,15 +883,11 @@ impl Network { l: u64, t: u64, ) -> EyreResult<()> { - // Note that we are doing the public dial info check - // We don't have to check this for concurrency, since this routine is run in a TickTask/SingleFuture - self.inner.lock().doing_public_dial_info_check = true; - // Do the public dial info check let out = self.do_public_dial_info_check(stop_token, l, t).await; // Done with public dial info check - self.inner.lock().doing_public_dial_info_check = false; + self.inner.lock().needs_public_dial_info_check = false; out } diff --git a/veilid-core/src/network_manager/wasm/mod.rs b/veilid-core/src/network_manager/wasm/mod.rs index d630f598..2489e191 100644 --- a/veilid-core/src/network_manager/wasm/mod.rs +++ b/veilid-core/src/network_manager/wasm/mod.rs @@ -46,7 +46,7 @@ impl Network { NetworkUnlockedInner { network_manager, routing_table, - connection_manager + connection_manager, } } @@ -58,7 +58,11 @@ impl Network { Self { config: network_manager.config(), inner: Arc::new(Mutex::new(Self::new_inner())), - unlocked_inner: Arc::new(Self::new_unlocked_inner(network_manager, routing_table, connection_manager)) + unlocked_inner: Arc::new(Self::new_unlocked_inner( + network_manager, + routing_table, + connection_manager, + )), } } @@ -319,12 +323,15 @@ impl Network { } ////////////////////////////////////////// - - pub fn set_needs_public_dial_info_check(&self, _punishment: Option>) { + + pub fn set_needs_public_dial_info_check( + &self, + _punishment: Option>, + ) { // } - pub fn doing_public_dial_info_check(&self) -> bool { + pub fn needs_public_dial_info_check(&self) -> bool { false } diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 93b267de..5c2344eb 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -155,7 +155,7 @@ fn with_route_permutations( // initial permutation let mut permutation: Vec = Vec::with_capacity(hop_count); for n in 0..hop_count { - permutation[n] = start + n; + permutation.push(start + n); } // if we have one hop or two, then there's only one permutation if hop_count == 1 || hop_count == 2 { @@ -229,22 +229,17 @@ impl RouteSpecStore { // Load secrets from pstore let pstore = routing_table.network_manager().protected_store(); + let out: Vec<(DHTKey, DHTKeySecret)> = pstore + .load_user_secret_cbor("RouteSpecStore") + .await? + .unwrap_or_default(); + let mut dead_keys = Vec::new(); - for (k, v) in &mut content.details { - if let Some(secret_key) = pstore - .load_user_secret(&format!("RouteSpecStore_{}", k.encode())) - .await? - { - match secret_key.try_into() { - Ok(s) => { - v.secret_key = DHTKeySecret::new(s); - } - Err(_) => { - dead_keys.push(*k); - } - } + for (k, v) in out { + if let Some(rsd) = content.details.get_mut(&k) { + rsd.secret_key = v; } else { - dead_keys.push(*k); + dead_keys.push(k); } } for k in dead_keys { @@ -293,18 +288,14 @@ impl RouteSpecStore { .routing_table .network_manager() .protected_store(); + + let mut out: Vec<(DHTKey, DHTKeySecret)> = Vec::with_capacity(content.details.len()); for (k, v) in &content.details { - if pstore - .save_user_secret( - &format!("RouteSpecStore_{}", k.encode()), - &v.secret_key.bytes, - ) - .await? - { - panic!("route spec should not already have secret key saved"); - } + out.push((*k, v.secret_key)); } + let _ = pstore.save_user_secret_cbor("RouteSpecStore", &out).await?; // ignore if this previously existed or not + Ok(()) } @@ -346,6 +337,14 @@ impl RouteSpecStore { inner.content.details.get_mut(public_key) } + /// Purge the route spec store + pub async fn purge(&self) -> EyreResult<()> { + let inner = &mut *self.inner.lock(); + inner.content = Default::default(); + inner.cache = Default::default(); + self.save().await + } + /// Create a new route /// Prefers nodes that are not currently in use by another route /// The route is not yet tested for its reachability @@ -675,7 +674,7 @@ impl RouteSpecStore { ))) } - pub fn release_route(&self, public_key: DHTKey) { + pub fn release_route(&self, public_key: DHTKey) -> EyreResult<()> { let mut inner = self.inner.lock(); if let Some(detail) = inner.content.details.remove(&public_key) { // Remove from hop cache @@ -710,8 +709,9 @@ impl RouteSpecStore { } } } else { - panic!("can't release route that was never allocated"); + bail!("can't release route that was never allocated"); } + Ok(()) } /// Find first matching unpublished route that fits into the selection criteria @@ -739,6 +739,22 @@ impl RouteSpecStore { None } + /// List all routes + pub fn list_routes(&self) -> Vec { + let inner = self.inner.lock(); + let mut out = Vec::with_capacity(inner.content.details.len()); + for detail in &inner.content.details { + out.push(*detail.0); + } + out + } + + /// Get the debug description of a route + pub fn debug_route(&self, key: &DHTKey) -> Option { + let inner = &*self.inner.lock(); + Self::detail(inner, key).map(|rsd| format!("{:#?}", rsd)) + } + ////////////////////////////////////////////////////////////////////// /// Compiles a safety route to the private route, with caching @@ -973,17 +989,18 @@ impl RouteSpecStore { /// Assemble private route for publication pub fn assemble_private_route( &self, - rti: &RoutingTableInner, - routing_table: RoutingTable, key: &DHTKey, + optimize: Option, ) -> EyreResult { let inner = &*self.inner.lock(); + let routing_table = self.unlocked_inner.routing_table.clone(); + let rti = &*routing_table.inner.read(); - let rsd = Self::detail(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; + let rsd = Self::detail(inner, key).ok_or_else(|| eyre!("route does not exist"))?; // See if we can optimize this compilation yet // We don't want to include full nodeinfo if we don't have to - let optimize = rsd.reachable; + let optimize = optimize.unwrap_or(rsd.reachable); // Make innermost route hop to our own node let mut route_hop = RouteHop { @@ -1053,79 +1070,79 @@ impl RouteSpecStore { /// Mark route as published /// When first deserialized, routes must be re-published in order to ensure they remain /// in the RouteSpecStore. - pub fn mark_route_published(&mut self, key: &DHTKey) -> EyreResult<()> { + pub fn mark_route_published(&self, key: &DHTKey, published: bool) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - Self::detail_mut(inner, &key) + Self::detail_mut(inner, key) .ok_or_else(|| eyre!("route does not exist"))? - .published = true; + .published = published; Ok(()) } /// Mark route as reachable /// When first deserialized, routes must be re-tested for reachability /// This can be used to determine if routes need to be sent with full peerinfo or can just use a node id - pub fn mark_route_reachable(&mut self, key: &DHTKey) -> EyreResult<()> { + pub fn mark_route_reachable(&self, key: &DHTKey, reachable: bool) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - Self::detail_mut(inner, &key) + Self::detail_mut(inner, key) .ok_or_else(|| eyre!("route does not exist"))? - .published = true; + .reachable = reachable; Ok(()) } /// Mark route as checked - pub fn touch_route_checked(&mut self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { + pub fn touch_route_checked(&self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - Self::detail_mut(inner, &key) + Self::detail_mut(inner, key) .ok_or_else(|| eyre!("route does not exist"))? .last_checked_ts = Some(cur_ts); Ok(()) } /// Mark route as used - pub fn touch_route_used(&mut self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { + pub fn touch_route_used(&self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - Self::detail_mut(inner, &key) + Self::detail_mut(inner, key) .ok_or_else(|| eyre!("route does not exist"))? .last_used_ts = Some(cur_ts); Ok(()) } /// Record latency on the route - pub fn record_latency(&mut self, key: &DHTKey, latency: u64) -> EyreResult<()> { + pub fn record_latency(&self, key: &DHTKey, latency: u64) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - let rsd = Self::detail_mut(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; + let rsd = Self::detail_mut(inner, key).ok_or_else(|| eyre!("route does not exist"))?; rsd.latency_stats = rsd.latency_stats_accounting.record_latency(latency); Ok(()) } /// Get the calculated latency stats - pub fn latency_stats(&mut self, key: &DHTKey) -> EyreResult { + pub fn latency_stats(&self, key: &DHTKey) -> EyreResult { let inner = &mut *self.inner.lock(); - Ok(Self::detail_mut(inner, &key) + Ok(Self::detail_mut(inner, key) .ok_or_else(|| eyre!("route does not exist"))? .latency_stats .clone()) } /// Add download transfers to route - pub fn add_down(&mut self, key: &DHTKey, bytes: u64) -> EyreResult<()> { + pub fn add_down(&self, key: &DHTKey, bytes: u64) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - let rsd = Self::detail_mut(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; + let rsd = Self::detail_mut(inner, key).ok_or_else(|| eyre!("route does not exist"))?; rsd.transfer_stats_accounting.add_down(bytes); Ok(()) } /// Add upload transfers to route - pub fn add_up(&mut self, key: &DHTKey, bytes: u64) -> EyreResult<()> { + pub fn add_up(&self, key: &DHTKey, bytes: u64) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - let rsd = Self::detail_mut(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; + let rsd = Self::detail_mut(inner, key).ok_or_else(|| eyre!("route does not exist"))?; rsd.transfer_stats_accounting.add_up(bytes); Ok(()) } /// Process transfer statistics to get averages - pub fn roll_transfers(&mut self, last_ts: u64, cur_ts: u64) { + pub fn roll_transfers(&self, last_ts: u64, cur_ts: u64) { let inner = &mut *self.inner.lock(); for rsd in inner.content.details.values_mut() { rsd.transfer_stats_accounting.roll_transfers( @@ -1135,4 +1152,22 @@ impl RouteSpecStore { ); } } + + /// Convert private route to binary blob + pub fn private_route_to_blob(private_route: &PrivateRoute) -> EyreResult> { + let mut pr_message = ::capnp::message::Builder::new_default(); + let mut pr_builder = pr_message.init_root::(); + encode_private_route(&private_route, &mut pr_builder) + .wrap_err("failed to encode private route")?; + builder_to_vec(pr_message).wrap_err("failed to convert builder to vec") + } + + /// Convert binary blob to private route + pub fn blob_to_private_route(blob: Vec) -> EyreResult { + let reader = ::capnp::message::Reader::new(RPCMessageData::new(blob), Default::default()); + let pr_reader = reader + .get_root::() + .wrap_err("failed to make reader for private_route")?; + decode_private_route(&pr_reader).wrap_err("failed to decode private route") + } } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 4d43c1cc..e0fc17a6 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -90,10 +90,16 @@ struct RPCMessageHeader { impl RPCMessageHeader {} #[derive(Debug)] -struct RPCMessageData { +pub struct RPCMessageData { contents: Vec, // rpc messages must be a canonicalized single segment } +impl RPCMessageData { + pub fn new(contents: Vec) -> Self { + Self { contents } + } +} + impl ReaderSegments for RPCMessageData { fn get_segment(&self, idx: u32) -> Option<&[u8]> { if idx > 0 { diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 5fe6173a..4ae72f28 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -16,12 +16,41 @@ fn get_bucket_entry_state(text: &str) -> Option { None } } + +fn get_string(text: &str) -> Option { + Some(text.to_owned()) +} + +fn get_route_id(rss: RouteSpecStore) -> impl Fn(&str) -> Option { + return move |text: &str| { + match DHTKey::try_decode(text).ok() { + Some(key) => { + let routes = rss.list_routes(); + if routes.contains(&key) { + return Some(key); + } + } + None => { + let routes = rss.list_routes(); + for r in routes { + let rkey = r.encode(); + if rkey.starts_with(text) { + return Some(r); + } + } + } + } + None + }; +} + fn get_number(text: &str) -> Option { usize::from_str(text).ok() } fn get_dht_key(text: &str) -> Option { DHTKey::try_decode(text).ok() } + fn get_protocol_type(text: &str) -> Option { let lctext = text.to_ascii_lowercase(); if lctext == "udp" { @@ -36,6 +65,41 @@ fn get_protocol_type(text: &str) -> Option { None } } +fn get_sequencing(text: &str) -> Option { + let seqtext = text.to_ascii_lowercase(); + if seqtext == "np" { + Some(Sequencing::NoPreference) + } else if seqtext == "ord" { + Some(Sequencing::PreferOrdered) + } else if seqtext == "*ord" { + Some(Sequencing::EnsureOrdered) + } else { + None + } +} +fn get_stability(text: &str) -> Option { + let sttext = text.to_ascii_lowercase(); + if sttext == "ll" { + Some(Stability::LowLatency) + } else if sttext == "rel" { + Some(Stability::Reliable) + } else { + None + } +} +fn get_direction_set(text: &str) -> Option { + let dstext = text.to_ascii_lowercase(); + if dstext == "in" { + Some(Direction::Inbound.into()) + } else if dstext == "out" { + Some(Direction::Outbound.into()) + } else if dstext == "inout" { + Some(DirectionSet::all()) + } else { + None + } +} + fn get_address_type(text: &str) -> Option { let lctext = text.to_ascii_lowercase(); if lctext == "ipv4" { @@ -251,6 +315,13 @@ impl VeilidAPI { .purge_last_connections(); Ok("Connections purged".to_owned()) + } else if args[0] == "routes" { + // Purge route spec store + let rss = self.network_manager()?.routing_table().route_spec_store(); + match rss.purge().await { + Ok(_) => Ok("Routes purged".to_owned()), + Err(e) => Ok(format!("Routes purged but failed to save: {}", e)), + } } else { Err(VeilidAPIError::InvalidArgument { context: "debug_purge".to_owned(), @@ -420,6 +491,191 @@ impl VeilidAPI { Ok(format!("{:#?}", out)) } + async fn debug_route_allocate(&self, args: Vec) -> Result { + // [ord|*ord] [rel] [] [in|out] + + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + let rss = routing_table.route_spec_store(); + let config = self.config().unwrap(); + let default_route_hop_count = { + let c = config.get(); + c.network.rpc.default_route_hop_count as usize + }; + + let mut ai = 1; + let mut sequencing = Sequencing::NoPreference; + let mut stability = Stability::LowLatency; + let mut hop_count = default_route_hop_count; + let mut directions = DirectionSet::all(); + + while ai < args.len() { + if let Ok(seq) = + get_debug_argument_at(&args, ai, "debug_route", "sequencing", get_sequencing) + { + sequencing = seq; + } else if let Ok(sta) = + get_debug_argument_at(&args, ai, "debug_route", "stability", get_stability) + { + stability = sta; + } else if let Ok(hc) = + get_debug_argument_at(&args, ai, "debug_route", "hop_count", get_number) + { + hop_count = hc; + } else if let Ok(ds) = + get_debug_argument_at(&args, ai, "debug_route", "direction_set", get_direction_set) + { + directions = ds; + } else { + return Ok(format!("Invalid argument specified: {}", args[ai])); + } + ai += 1; + } + + // Allocate route + let out = match rss.allocate_route(stability, sequencing, hop_count, directions) { + Ok(Some(v)) => format!("{}", v.encode()), + Ok(None) => format!(""), + Err(e) => { + format!("Route allocation failed: {}", e) + } + }; + + Ok(out) + } + async fn debug_route_release(&self, args: Vec) -> Result { + // + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + let rss = routing_table.route_spec_store(); + + let route_id = get_debug_argument_at(&args, 1, "debug_route", "route_id", get_dht_key)?; + + // Release route + let out = match rss.release_route(route_id) { + Ok(()) => format!("Released"), + Err(e) => { + format!("Route release failed: {}", e) + } + }; + + Ok(out) + } + async fn debug_route_publish(&self, args: Vec) -> Result { + // [full] + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + let rss = routing_table.route_spec_store(); + + let route_id = get_debug_argument_at(&args, 1, "debug_route", "route_id", get_dht_key)?; + let full = { + if args.len() > 2 { + let full_val = get_debug_argument_at(&args, 2, "debug_route", "full", get_string)? + .to_ascii_lowercase(); + if full_val == "full" { + true + } else { + return Err(VeilidAPIError::invalid_argument( + "debug_route", + "full", + full_val, + )); + } + } else { + false + } + }; + + // Publish route + let out = match rss.assemble_private_route(&route_id, Some(!full)) { + Ok(private_route) => { + if let Err(e) = rss.mark_route_published(&route_id, true) { + return Ok(format!("Couldn't mark route published: {}", e)); + } + // Convert to blob + let blob_data = RouteSpecStore::private_route_to_blob(&private_route) + .map_err(VeilidAPIError::internal)?; + data_encoding::BASE64URL_NOPAD.encode(&blob_data) + } + Err(e) => { + format!("Couldn't assemble private route: {}", e) + } + }; + + Ok(out) + } + async fn debug_route_unpublish(&self, args: Vec) -> Result { + // + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + let rss = routing_table.route_spec_store(); + + let route_id = get_debug_argument_at(&args, 1, "debug_route", "route_id", get_dht_key)?; + + // Unpublish route + let out = if let Err(e) = rss.mark_route_published(&route_id, false) { + return Ok(format!("Couldn't mark route unpublished: {}", e)); + } else { + "Route unpublished".to_owned() + }; + Ok(out) + } + async fn debug_route_print(&self, args: Vec) -> Result { + // + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + let rss = routing_table.route_spec_store(); + + let route_id = get_debug_argument_at(&args, 1, "debug_route", "route_id", get_dht_key)?; + + match rss.debug_route(&route_id) { + Some(s) => Ok(s), + None => Ok("Route does not exist".to_owned()), + } + } + async fn debug_route_list(&self, _args: Vec) -> Result { + // + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + let rss = routing_table.route_spec_store(); + + let routes = rss.list_routes(); + let mut out = format!("Routes: (count = {}):\n", routes.len()); + for r in routes { + out.push_str(&format!("{}\n", r.encode())); + } + Ok(out) + } + async fn debug_route_import(&self, _args: Vec) -> Result { + // + let out = format!(""); + return Ok(out); + } + + async fn debug_route(&self, args: String) -> Result { + let args: Vec = args.split_whitespace().map(|s| s.to_owned()).collect(); + + let command = get_debug_argument_at(&args, 0, "debug_route", "command", get_string)?; + + if command == "allocate" { + self.debug_route_allocate(args).await + } else if command == "release" { + self.debug_route_release(args).await + } else if command == "publish" { + self.debug_route_publish(args).await + } else if command == "unpublish" { + self.debug_route_unpublish(args).await + } else if command == "print" { + self.debug_route_print(args).await + } else if command == "list" { + self.debug_route_list(args).await + } else if command == "import" { + self.debug_route_import(args).await + } else { + Ok(">>> Unknown command\n".to_owned()) + } + } + pub async fn debug_help(&self, _args: String) -> Result { Ok(r#">>> Debug commands: help @@ -435,6 +691,13 @@ impl VeilidAPI { restart network ping [protocol_type][address_type][routing_domain] contact [protocol_type [address_type]] + route allocate [ord|*ord] [rel] [] [in|out] + route release + route publish [full] + route unpublish + route print + route list + route import "# .to_owned()) } @@ -476,6 +739,8 @@ impl VeilidAPI { self.debug_config(rest).await } else if arg == "restart" { self.debug_restart(rest).await + } else if arg == "route" { + self.debug_route(rest).await } else { Ok(">>> Unknown command\n".to_owned()) } From dbd9d874345f79bfab12b0231ac04ddfa52353f8 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 4 Nov 2022 19:39:02 -0400 Subject: [PATCH 40/67] fix async --- veilid-core/src/routing_table/route_spec_store.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 5c2344eb..7cf45488 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -339,9 +339,11 @@ impl RouteSpecStore { /// Purge the route spec store pub async fn purge(&self) -> EyreResult<()> { - let inner = &mut *self.inner.lock(); - inner.content = Default::default(); - inner.cache = Default::default(); + { + let inner = &mut *self.inner.lock(); + inner.content = Default::default(); + inner.cache = Default::default(); + } self.save().await } From b2dd3bf9b7502ef337dda98edbe2a6da42f5e069 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 4 Nov 2022 19:58:01 -0400 Subject: [PATCH 41/67] fixes --- veilid-core/src/intf/native/protected_store.rs | 4 ++-- veilid-core/src/intf/wasm/protected_store.rs | 4 ++-- veilid-core/src/routing_table/route_spec_store.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/veilid-core/src/intf/native/protected_store.rs b/veilid-core/src/intf/native/protected_store.rs index d88e31fb..3ab88a12 100644 --- a/veilid-core/src/intf/native/protected_store.rs +++ b/veilid-core/src/intf/native/protected_store.rs @@ -143,7 +143,7 @@ impl ProtectedStore { } } - #[instrument(level = "trace", skip(self, value), ret, err)] + #[instrument(level = "trace", skip(self, value))] pub async fn save_user_secret_cbor(&self, key: &str, value: &T) -> EyreResult where T: Serialize, @@ -152,7 +152,7 @@ impl ProtectedStore { self.save_user_secret(&key, &v).await } - #[instrument(level = "trace", skip(self), err)] + #[instrument(level = "trace", skip(self))] pub async fn load_user_secret_cbor(&self, key: &str) -> EyreResult> where T: for<'de> Deserialize<'de>, diff --git a/veilid-core/src/intf/wasm/protected_store.rs b/veilid-core/src/intf/wasm/protected_store.rs index ab527dbf..54ba80fa 100644 --- a/veilid-core/src/intf/wasm/protected_store.rs +++ b/veilid-core/src/intf/wasm/protected_store.rs @@ -185,7 +185,7 @@ impl ProtectedStore { } } - #[instrument(level = "trace", skip(self, value), ret, err)] + #[instrument(level = "trace", skip(self, value))] pub async fn save_user_secret_cbor(&self, key: &str, value: &T) -> EyreResult where T: Serialize, @@ -194,7 +194,7 @@ impl ProtectedStore { self.save_user_secret(&key, &v).await } - #[instrument(level = "trace", skip(self), err)] + #[instrument(level = "trace", skip(self))] pub async fn load_user_secret_cbor(&self, key: &str) -> EyreResult> where T: for<'de> Deserialize<'de>, diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 7cf45488..3044b122 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -282,7 +282,7 @@ impl RouteSpecStore { let rsstdb = table_store.open("RouteSpecStore", 1).await?; rsstdb.store_cbor(0, b"content", &content).await?; - // Keep secrets in protected store as well + // // Keep secrets in protected store as well let pstore = self .unlocked_inner .routing_table From cd892d077ae791ee51fe643919c940b70278078c Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 5 Nov 2022 18:50:20 -0400 Subject: [PATCH 42/67] fixes --- .../src/routing_table/route_spec_store.rs | 24 +- veilid-core/src/rpc_processor/mod.rs | 44 ++- veilid-core/src/rpc_processor/rpc_route.rs | 22 +- veilid-core/src/veilid_api/debug.rs | 301 ++++++++++++------ 4 files changed, 258 insertions(+), 133 deletions(-) diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 3044b122..e0345437 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -699,7 +699,11 @@ impl RouteSpecStore { } } // Remove from end nodes cache - match inner.cache.used_nodes.entry(*detail.hops.last().unwrap()) { + match inner + .cache + .used_end_nodes + .entry(*detail.hops.last().unwrap()) + { std::collections::hash_map::Entry::Occupied(mut o) => { *o.get_mut() -= 1; if *o.get() == 0 { @@ -707,7 +711,7 @@ impl RouteSpecStore { } } std::collections::hash_map::Entry::Vacant(_) => { - panic!("used_nodes cache should have contained hop"); + panic!("used_end_nodes cache should have contained hop"); } } } else { @@ -1161,12 +1165,24 @@ impl RouteSpecStore { let mut pr_builder = pr_message.init_root::(); encode_private_route(&private_route, &mut pr_builder) .wrap_err("failed to encode private route")?; - builder_to_vec(pr_message).wrap_err("failed to convert builder to vec") + + let mut buffer = vec![]; + capnp::serialize_packed::write_message(&mut buffer, &pr_message) + .wrap_err("failed to convert builder to vec")?; + Ok(buffer) + + // builder_to_vec(pr_message).wrap_err("failed to convert builder to vec") } /// Convert binary blob to private route pub fn blob_to_private_route(blob: Vec) -> EyreResult { - let reader = ::capnp::message::Reader::new(RPCMessageData::new(blob), Default::default()); + let reader = capnp::serialize_packed::read_message( + blob.as_slice(), + capnp::message::ReaderOptions::new(), + ) + .wrap_err("failed to make message reader")?; + + //let reader = ::capnp::message::Reader::new(RPCMessageData::new(blob), Default::default()); let pr_reader = reader .get_root::() .wrap_err("failed to make reader for private_route")?; diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index e0fc17a6..f2186b05 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -30,7 +30,6 @@ pub use rpc_status::*; use super::*; use crate::crypto::*; use crate::xx::*; -use capnp::message::ReaderSegments; use futures_util::StreamExt; use network_manager::*; use receipt_manager::*; @@ -98,18 +97,28 @@ impl RPCMessageData { pub fn new(contents: Vec) -> Self { Self { contents } } -} -impl ReaderSegments for RPCMessageData { - fn get_segment(&self, idx: u32) -> Option<&[u8]> { - if idx > 0 { - None - } else { - Some(self.contents.as_slice()) - } + pub fn get_reader( + &self, + ) -> Result, RPCError> { + capnp::serialize_packed::read_message( + self.contents.as_slice(), + capnp::message::ReaderOptions::new(), + ) + .map_err(RPCError::protocol) } } +// impl ReaderSegments for RPCMessageData { +// fn get_segment(&self, idx: u32) -> Option<&[u8]> { +// if idx > 0 { +// None +// } else { +// Some(self.contents.as_slice()) +// } +// } +// } + #[derive(Debug)] struct RPCMessageEncoded { header: RPCMessageHeader, @@ -127,12 +136,17 @@ pub fn builder_to_vec<'a, T>(builder: capnp::message::Builder) -> Result(reader: &capnp::message::Reader) -> Result, RPCError> @@ -899,7 +913,7 @@ impl RPCProcessor { // Decode the RPC message let operation = { - let reader = capnp::message::Reader::new(encoded_msg.data, Default::default()); + let reader = encoded_msg.data.get_reader()?; let op_reader = reader .get_root::() .map_err(RPCError::protocol) @@ -945,7 +959,7 @@ impl RPCProcessor { RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { // Decode the RPC message let operation = { - let reader = capnp::message::Reader::new(encoded_msg.data, Default::default()); + let reader = encoded_msg.data.get_reader()?; let op_reader = reader .get_root::() .map_err(RPCError::protocol) diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 257a2b71..56848b18 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -325,15 +325,8 @@ impl RPCProcessor { .cached_dh(&route.safety_route.public_key, &node_id_secret) .map_err(RPCError::protocol)?; let dec_blob_data = Crypto::decrypt_aead(blob_data, &d.nonce, &dh_secret, None) - .map_err(RPCError::map_internal( - "decryption of safety route hop failed", - ))?; - let dec_blob_reader = capnp::message::Reader::new( - RPCMessageData { - contents: dec_blob_data, - }, - Default::default(), - ); + .map_err(RPCError::protocol)?; + let dec_blob_reader = RPCMessageData::new(dec_blob_data).get_reader()?; // Decode the blob appropriately if blob_tag == 1 { @@ -387,15 +380,8 @@ impl RPCProcessor { .map_err(RPCError::protocol)?; let dec_blob_data = Crypto::decrypt_aead(&next_hop.blob, &next_hop.nonce, &dh_secret, None) - .map_err(RPCError::map_internal( - "decryption of private route hop failed", - ))?; - let dec_blob_reader = capnp::message::Reader::new( - RPCMessageData { - contents: dec_blob_data, - }, - Default::default(), - ); + .map_err(RPCError::protocol)?; + let dec_blob_reader = RPCMessageData::new(dec_blob_data).get_reader()?; // Decode next RouteHop let route_hop = { diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 4ae72f28..ee00b36e 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -2,9 +2,19 @@ // Debugging use super::*; +use data_encoding::BASE64URL_NOPAD; use routing_table::*; use rpc_processor::*; +#[derive(Default, Debug)] +struct DebugCache { + imported_routes: Vec, +} + +static DEBUG_CACHE: Mutex = Mutex::new(DebugCache { + imported_routes: Vec::new(), +}); + fn get_bucket_entry_state(text: &str) -> Option { if text == "dead" { Some(BucketEntryState::Dead) @@ -44,6 +54,126 @@ fn get_route_id(rss: RouteSpecStore) -> impl Fn(&str) -> Option { }; } +fn get_safety_selection(text: &str, rss: RouteSpecStore) -> Option { + if text.len() == 0 { + return None; + } + if &text[0..1] == "-" { + // Unsafe + let text = &text[1..]; + let seq = get_sequencing(text).unwrap_or(Sequencing::NoPreference); + Some(SafetySelection::Unsafe(seq)) + } else { + // Safe + let mut preferred_route = None; + let mut hop_count = 2; + let mut stability = Stability::LowLatency; + let mut sequencing = Sequencing::NoPreference; + for x in text.split(",") { + let x = x.trim(); + if let Some(pr) = get_route_id(rss.clone())(x) { + preferred_route = Some(pr) + } + if let Some(n) = get_number(x) { + hop_count = n; + } + if let Some(s) = get_stability(x) { + stability = s; + } + if let Some(s) = get_sequencing(x) { + sequencing = s; + } + } + let ss = SafetySpec { + preferred_route, + hop_count, + stability, + sequencing, + }; + Some(SafetySelection::Safe(ss)) + } +} + +fn get_node_ref_modifiers(mut node_ref: NodeRef) -> impl FnOnce(&str) -> Option { + move |text| { + for m in text.split("/") { + if let Some(pt) = get_protocol_type(m) { + node_ref.merge_filter(NodeRefFilter::new().with_protocol_type(pt)); + } else if let Some(at) = get_address_type(m) { + node_ref.merge_filter(NodeRefFilter::new().with_address_type(at)); + } else if let Some(rd) = get_routing_domain(m) { + node_ref.merge_filter(NodeRefFilter::new().with_routing_domain(rd)); + } else { + return None; + } + } + Some(node_ref) + } +} + +fn get_destination(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option { + move |text| { + // Safety selection + let (text, ss) = if let Some((first, second)) = text.split_once('+') { + let ss = get_safety_selection(second, routing_table.route_spec_store())?; + (first, Some(ss)) + } else { + (text, None) + }; + if text.len() == 0 { + return None; + } + if &text[0..1] == "#" { + // Private route + let text = &text[1..]; + let n = get_number(text)?; + let dc = DEBUG_CACHE.lock(); + let r = dc.imported_routes.get(n)?; + Some(Destination::private_route( + r.clone(), + ss.unwrap_or(SafetySelection::Unsafe(Sequencing::NoPreference)), + )) + } else { + let (text, mods) = text + .split_once('/') + .map(|x| (x.0, Some(x.1))) + .unwrap_or((text, None)); + if let Some((first, second)) = text.split_once('@') { + // Relay + let relay_id = get_dht_key(second)?; + let mut relay_nr = routing_table.lookup_node_ref(relay_id)?; + let target_id = get_dht_key(first)?; + + if let Some(mods) = mods { + relay_nr = get_node_ref_modifiers(relay_nr)(mods)?; + } + + let mut d = Destination::relay(relay_nr, target_id); + if let Some(ss) = ss { + d = d.with_safety(ss) + } + + Some(d) + } else { + // Direct + let target_id = get_dht_key(text)?; + let mut target_nr = routing_table.lookup_node_ref(target_id)?; + + if let Some(mods) = mods { + target_nr = get_node_ref_modifiers(target_nr)(mods)?; + } + + let mut d = Destination::direct(target_nr); + if let Some(ss) = ss { + d = d.with_safety(ss) + } + + Some(d) + } + } + } +} + fn get_number(text: &str) -> Option { usize::from_str(text).ok() } @@ -51,6 +181,22 @@ fn get_dht_key(text: &str) -> Option { DHTKey::try_decode(text).ok() } +fn get_node_ref(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option { + move |text| { + let (text, mods) = text + .split_once('/') + .map(|x| (x.0, Some(x.1))) + .unwrap_or((text, None)); + + let node_id = get_dht_key(text)?; + let mut nr = routing_table.lookup_node_ref(node_id)?; + if let Some(mods) = mods { + nr = get_node_ref_modifiers(nr)(mods)?; + } + Some(nr) + } +} + fn get_protocol_type(text: &str) -> Option { let lctext = text.to_ascii_lowercase(); if lctext == "udp" { @@ -366,55 +512,19 @@ impl VeilidAPI { async fn debug_contact(&self, args: String) -> Result { let args: Vec = args.split_whitespace().map(|s| s.to_owned()).collect(); - let node_id = get_debug_argument_at(&args, 0, "debug_contact", "node_id", get_dht_key)?; - let network_manager = self.network_manager()?; let routing_table = network_manager.routing_table(); - let mut nr = match routing_table.lookup_node_ref(node_id) { - Some(nr) => nr, - None => return Ok("Node id not found in routing table".to_owned()), - }; - - let mut ai = 1; - let mut routing_domain = None; - while ai < args.len() { - if let Ok(pt) = get_debug_argument_at( - &args, - ai, - "debug_contact", - "protocol_type", - get_protocol_type, - ) { - nr.merge_filter(NodeRefFilter::new().with_protocol_type(pt)); - } else if let Ok(at) = - get_debug_argument_at(&args, ai, "debug_contact", "address_type", get_address_type) - { - nr.merge_filter(NodeRefFilter::new().with_address_type(at)); - } else if let Ok(rd) = get_debug_argument_at( - &args, - ai, - "debug_contact", - "routing_domain", - get_routing_domain, - ) { - if routing_domain.is_none() { - routing_domain = Some(rd); - } else { - return Ok("Multiple routing domains specified".to_owned()); - } - } else { - return Ok(format!("Invalid argument specified: {}", args[ai])); - } - ai += 1; - } - - if let Some(routing_domain) = routing_domain { - nr.merge_filter(NodeRefFilter::new().with_routing_domain(routing_domain)) - } + let node_ref = get_debug_argument_at( + &args, + 0, + "debug_contact", + "node_ref", + get_node_ref(routing_table), + )?; let cm = network_manager - .get_node_contact_method(nr) + .get_node_contact_method(node_ref) .map_err(VeilidAPIError::internal)?; Ok(format!("{:#?}", cm)) @@ -427,49 +537,17 @@ impl VeilidAPI { let args: Vec = args.split_whitespace().map(|s| s.to_owned()).collect(); - let node_id = get_debug_argument_at(&args, 0, "debug_ping", "node_id", get_dht_key)?; - - let mut nr = match routing_table.lookup_node_ref(node_id) { - Some(nr) => nr, - None => return Ok("Node id not found in routing table".to_owned()), - }; - - let mut ai = 1; - let mut routing_domain = None; - while ai < args.len() { - if let Ok(pt) = - get_debug_argument_at(&args, ai, "debug_ping", "protocol_type", get_protocol_type) - { - nr.merge_filter(NodeRefFilter::new().with_protocol_type(pt)); - } else if let Ok(at) = - get_debug_argument_at(&args, ai, "debug_ping", "address_type", get_address_type) - { - nr.merge_filter(NodeRefFilter::new().with_address_type(at)); - } else if let Ok(rd) = get_debug_argument_at( - &args, - ai, - "debug_ping", - "routing_domain", - get_routing_domain, - ) { - if routing_domain.is_none() { - routing_domain = Some(rd); - } else { - return Ok("Multiple routing domains specified".to_owned()); - } - } else { - return Ok(format!("Invalid argument specified: {}", args[ai])); - } - ai += 1; - } - - if let Some(routing_domain) = routing_domain { - nr.merge_filter(NodeRefFilter::new().with_routing_domain(routing_domain)) - } + let dest = get_debug_argument_at( + &args, + 0, + "debug_ping", + "destination", + get_destination(routing_table), + )?; // Dump routing table entry let out = match rpc - .rpc_call_status(Destination::direct(nr)) + .rpc_call_status(dest) .await .map_err(VeilidAPIError::internal)? { @@ -595,7 +673,14 @@ impl VeilidAPI { // Convert to blob let blob_data = RouteSpecStore::private_route_to_blob(&private_route) .map_err(VeilidAPIError::internal)?; - data_encoding::BASE64URL_NOPAD.encode(&blob_data) + let out = BASE64URL_NOPAD.encode(&blob_data); + info!( + "Published route {} as {} bytes:\n{}", + route_id.encode(), + blob_data.len(), + out + ); + format!("Published route {}", route_id.encode()) } Err(e) => { format!("Couldn't assemble private route: {}", e) @@ -646,9 +731,21 @@ impl VeilidAPI { } Ok(out) } - async fn debug_route_import(&self, _args: Vec) -> Result { + async fn debug_route_import(&self, args: Vec) -> Result { // - let out = format!(""); + + let blob = get_debug_argument_at(&args, 1, "debug_route", "blob", get_string)?; + let blob_dec = BASE64URL_NOPAD + .decode(blob.as_bytes()) + .map_err(VeilidAPIError::generic)?; + let pr = + RouteSpecStore::blob_to_private_route(blob_dec).map_err(VeilidAPIError::generic)?; + + let mut dc = DEBUG_CACHE.lock(); + let n = dc.imported_routes.len(); + let out = format!("Private route #{} imported: {}", n, pr.public_key); + dc.imported_routes.push(pr); + return Ok(out); } @@ -682,22 +779,34 @@ impl VeilidAPI { buckets [dead|reliable] dialinfo entries [dead|reliable] [limit] - entry + entry nodeinfo config [key [new value]] purge attach detach restart network - ping [protocol_type][address_type][routing_domain] - contact [protocol_type [address_type]] + ping + contact [] route allocate [ord|*ord] [rel] [] [in|out] - route release - route publish [full] - route unpublish - route print - route list - route import + release + publish [full] + unpublish + print + list + import + + is: + * direct: [+][] + * relay: @[+][] + * private: #[+] + is: + * unsafe: -[ord|*ord] + * safe: [route][,ord|*ord][,rel][,] + is: [/][/][/] + is: udp|tcp|ws|wss + is: ipv4|ipv6 + is: public|local "# .to_owned()) } From 0e7f3e1c3c6d954d9e76304fd73ade337deea999 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 5 Nov 2022 19:41:18 -0400 Subject: [PATCH 43/67] fix colliding net connections --- veilid-core/src/network_manager/connection_manager.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index d6350612..e83f069d 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -232,7 +232,8 @@ impl ConnectionManager { }); // Wait for the killed connections to end their recv loops let did_kill = !killed.is_empty(); - for k in killed { + for mut k in killed { + k.close(); k.await; } did_kill From a54da973937b86cbd679bcbfaa3097ea914f396b Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 6 Nov 2022 16:07:56 -0500 Subject: [PATCH 44/67] remove nodejs support --- Cargo.lock | 12 +-- veilid-core/Cargo.toml | 5 +- veilid-core/src/crypto/mod.rs | 8 +- veilid-core/src/intf/native/table_store.rs | 32 ++++-- veilid-core/src/intf/table_db.rs | 101 ++++++++++++++++-- veilid-core/src/intf/wasm/protected_store.rs | 73 ++----------- veilid-core/src/intf/wasm/system.rs | 42 ++++---- veilid-core/src/intf/wasm/table_store.rs | 37 +++++-- veilid-core/src/intf/wasm/utils/mod.rs | 33 ------ veilid-core/src/routing_table/bucket.rs | 28 +++++ veilid-core/src/routing_table/bucket_entry.rs | 19 +++- veilid-core/src/routing_table/mod.rs | 85 ++++++++++++++- .../src/routing_table/route_spec_store.rs | 26 ++++- .../src/routing_table/routing_table_inner.rs | 7 +- .../src/tests/common/test_table_store.rs | 61 +++++------ .../src/tests/common/test_veilid_config.rs | 8 +- veilid-core/src/veilid_api/debug.rs | 5 +- veilid-core/src/xx/mod.rs | 75 +++++-------- 18 files changed, 375 insertions(+), 282 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 356bf366..507882b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2755,15 +2755,6 @@ dependencies = [ "value-bag", ] -[[package]] -name = "lru" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909" -dependencies = [ - "hashbrown", -] - [[package]] name = "lru-cache" version = "0.1.2" @@ -5505,7 +5496,6 @@ dependencies = [ "futures-util", "generic-array", "getrandom 0.2.8", - "hashbrown", "hashlink 0.8.1", "hex", "ifstructs", @@ -5515,11 +5505,11 @@ dependencies = [ "js-sys", "json", "keyring-manager", + "keyvaluedb", "keyvaluedb-sqlite", "keyvaluedb-web", "lazy_static", "libc", - "lru", "maplit", "ndk 0.6.0", "ndk-glue", diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index b59c6c7c..cb828dfc 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -58,7 +58,7 @@ digest = "0.9.0" rtnetlink = { version = "^0", default-features = false, optional = true } async-std-resolver = { version = "^0", optional = true } trust-dns-resolver = { version = "^0", optional = true } - +keyvaluedb = { path = "../external/keyvaluedb/keyvaluedb" } # Dependencies for native builds only # Linux, Windows, Mac, iOS, Android @@ -72,7 +72,6 @@ async-tungstenite = { version = "^0", features = ["async-tls"] } maplit = "^1" config = { version = "^0", features = ["yaml"] } keyring-manager = { path = "../external/keyring-manager" } -lru = "^0" async-tls = "^0.11" igd = { path = "../external/rust-igd" } webpki = "^0" @@ -96,8 +95,6 @@ nix = "^0" wasm-bindgen = "^0" js-sys = "^0" wasm-bindgen-futures = "^0" -hashbrown = "^0" -lru = {version = "^0", features = ["hashbrown"] } no-std-net = { path = "../external/no-std-net", features = ["serde"] } keyvaluedb-web = { path = "../external/keyvaluedb/keyvaluedb-web" } data-encoding = { version = "^2", default_features = false, features = ["alloc"] } diff --git a/veilid-core/src/crypto/mod.rs b/veilid-core/src/crypto/mod.rs index c7836445..4184c54b 100644 --- a/veilid-core/src/crypto/mod.rs +++ b/veilid-core/src/crypto/mod.rs @@ -119,12 +119,12 @@ impl Crypto { // load caches if they are valid for this node id let mut db = table_store.open("crypto_caches", 1).await?; - let caches_valid = match db.load(0, b"node_id").await? { + let caches_valid = match db.load(0, b"node_id")? { Some(v) => v.as_slice() == node_id.bytes, None => false, }; if caches_valid { - if let Some(b) = db.load(0, b"dh_cache").await? { + if let Some(b) = db.load(0, b"dh_cache")? { let mut inner = self.inner.lock(); bytes_to_cache(&b, &mut inner.dh_cache); } @@ -132,7 +132,7 @@ impl Crypto { drop(db); table_store.delete("crypto_caches").await?; db = table_store.open("crypto_caches", 1).await?; - db.store(0, b"node_id", &node_id.bytes).await?; + db.store(0, b"node_id", &node_id.bytes)?; } // Schedule flushing @@ -159,7 +159,7 @@ impl Crypto { }; let db = table_store.open("crypto_caches", 1).await?; - db.store(0, b"dh_cache", &cache_bytes).await?; + db.store(0, b"dh_cache", &cache_bytes)?; Ok(()) } diff --git a/veilid-core/src/intf/native/table_store.rs b/veilid-core/src/intf/native/table_store.rs index 041c4182..5d5ceb14 100644 --- a/veilid-core/src/intf/native/table_store.rs +++ b/veilid-core/src/intf/native/table_store.rs @@ -8,6 +8,8 @@ struct TableStoreInner { opened: BTreeMap>>, } +/// Veilid Table Storage +/// Database for storing key value pairs persistently across runs #[derive(Clone)] pub struct TableStore { config: VeilidConfig, @@ -20,31 +22,38 @@ impl TableStore { opened: BTreeMap::new(), } } - pub fn new(config: VeilidConfig) -> Self { + pub(crate) fn new(config: VeilidConfig) -> Self { Self { config, inner: Arc::new(Mutex::new(Self::new_inner())), } } - pub async fn delete_all(&self) -> EyreResult<()> { - // Delete all known keys - self.delete("crypto_caches").await?; + /// Delete all known tables + pub async fn delete_all(&self) { + if let Err(e) = self.delete("crypto_caches").await { + error!("failed to delete 'crypto_caches': {}", e); + } + if let Err(e) = self.delete("RouteSpecStore").await { + error!("failed to delete 'RouteSpecStore': {}", e); + } + if let Err(e) = self.delete("routing_table").await { + error!("failed to delete 'routing_table': {}", e); + } + } + + pub(crate) async fn init(&self) -> EyreResult<()> { Ok(()) } - pub async fn init(&self) -> EyreResult<()> { - Ok(()) - } - - pub async fn terminate(&self) { + pub(crate) async fn terminate(&self) { assert!( self.inner.lock().opened.is_empty(), "all open databases should have been closed" ); } - pub fn on_table_db_drop(&self, table: String) { + pub(crate) fn on_table_db_drop(&self, table: String) { let mut inner = self.inner.lock(); if inner.opened.remove(&table).is_none() { unreachable!("should have removed an item"); @@ -82,6 +91,8 @@ impl TableStore { }) } + /// Get or create a TableDB database table. If the column count is greater than an + /// existing TableDB's column count, the database will be upgraded to add the missing columns pub async fn open(&self, name: &str, column_count: u32) -> EyreResult { let table_name = self.get_table_name(name)?; @@ -121,6 +132,7 @@ impl TableStore { Ok(table_db) } + /// Delete a TableDB table by name pub async fn delete(&self, name: &str) -> EyreResult { let table_name = self.get_table_name(name)?; diff --git a/veilid-core/src/intf/table_db.rs b/veilid-core/src/intf/table_db.rs index d3bdc7d7..09fbe5a3 100644 --- a/veilid-core/src/intf/table_db.rs +++ b/veilid-core/src/intf/table_db.rs @@ -5,8 +5,10 @@ use serde::{Deserialize, Serialize}; cfg_if! { if #[cfg(target_arch = "wasm32")] { use keyvaluedb_web::*; + use keyvaluedb::*; } else { use keyvaluedb_sqlite::*; + use keyvaluedb::*; } } @@ -28,7 +30,7 @@ pub struct TableDB { } impl TableDB { - pub fn new(table: String, table_store: TableStore, database: Database) -> Self { + pub(super) fn new(table: String, table_store: TableStore, database: Database) -> Self { Self { inner: Arc::new(Mutex::new(TableDBInner { table, @@ -38,22 +40,24 @@ impl TableDB { } } - pub fn try_new_from_weak_inner(weak_inner: Weak>) -> Option { + pub(super) fn try_new_from_weak_inner(weak_inner: Weak>) -> Option { weak_inner.upgrade().map(|table_db_inner| Self { inner: table_db_inner, }) } - pub fn weak_inner(&self) -> Weak> { + pub(super) fn weak_inner(&self) -> Weak> { Arc::downgrade(&self.inner) } - pub async fn get_column_count(&self) -> EyreResult { + /// Get the total number of columns in the TableDB + pub fn get_column_count(&self) -> EyreResult { let db = &self.inner.lock().database; db.num_columns().wrap_err("failed to get column count: {}") } - pub async fn get_keys(&self, col: u32) -> EyreResult>> { + /// Get the list of keys in a column of the TableDB + pub fn get_keys(&self, col: u32) -> EyreResult>> { let db = &self.inner.lock().database; let mut out: Vec> = Vec::new(); db.iter(col, None, &mut |kv| { @@ -64,14 +68,25 @@ impl TableDB { Ok(out) } - pub async fn store(&self, col: u32, key: &[u8], value: &[u8]) -> EyreResult<()> { + /// Start a TableDB write transaction. The transaction object must be committed or rolled back before dropping. + pub fn transact<'a>(&'a self) -> TableDBTransaction<'a> { + let dbt = { + let db = &self.inner.lock().database; + db.transaction() + }; + TableDBTransaction::new(self, dbt) + } + + /// Store a key with a value in a column in the TableDB. Performs a single transaction immediately. + pub fn store(&self, col: u32, key: &[u8], value: &[u8]) -> EyreResult<()> { let db = &self.inner.lock().database; let mut dbt = db.transaction(); dbt.put(col, key, value); db.write(dbt).wrap_err("failed to store key") } - pub async fn store_cbor(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()> + /// Store a key in CBOR format with a value in a column in the TableDB. Performs a single transaction immediately. + pub fn store_cbor(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()> where T: Serialize, { @@ -83,12 +98,14 @@ impl TableDB { db.write(dbt).wrap_err("failed to store key") } - pub async fn load(&self, col: u32, key: &[u8]) -> EyreResult>> { + /// Read a key from a column in the TableDB immediately. + pub fn load(&self, col: u32, key: &[u8]) -> EyreResult>> { let db = &self.inner.lock().database; db.get(col, key).wrap_err("failed to get key") } - pub async fn load_cbor(&self, col: u32, key: &[u8]) -> EyreResult> + /// Read a key from a column in the TableDB immediately, in CBOR format. + pub fn load_cbor(&self, col: u32, key: &[u8]) -> EyreResult> where T: for<'de> Deserialize<'de>, { @@ -104,7 +121,8 @@ impl TableDB { Ok(Some(obj)) } - pub async fn delete(&self, col: u32, key: &[u8]) -> EyreResult { + /// Delete key with from a column in the TableDB + pub fn delete(&self, col: u32, key: &[u8]) -> EyreResult { let db = &self.inner.lock().database; let found = db.get(col, key).wrap_err("failed to get key")?; match found { @@ -118,3 +136,66 @@ impl TableDB { } } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// A TableDB transaction +/// Atomically commits a group of writes or deletes to the TableDB +pub struct TableDBTransaction<'a> { + db: &'a TableDB, + dbt: Option, + _phantom: core::marker::PhantomData<&'a ()>, +} + +impl<'a> TableDBTransaction<'a> { + fn new(db: &'a TableDB, dbt: DBTransaction) -> Self { + Self { + db, + dbt: Some(dbt), + _phantom: Default::default(), + } + } + + /// Commit the transaction. Performs all actions atomically. + pub fn commit(mut self) -> EyreResult<()> { + self.db + .inner + .lock() + .database + .write(self.dbt.take().unwrap()) + .wrap_err("commit failed") + } + + /// Rollback the transaction. Does nothing to the TableDB. + pub fn rollback(mut self) { + self.dbt = None; + } + + /// Store a key with a value in a column in the TableDB + pub fn store(&mut self, col: u32, key: &[u8], value: &[u8]) { + self.dbt.as_mut().unwrap().put(col, key, value); + } + + /// Store a key in CBOR format with a value in a column in the TableDB + pub fn store_cbor(&mut self, col: u32, key: &[u8], value: &T) -> EyreResult<()> + where + T: Serialize, + { + let v = serde_cbor::to_vec(value).wrap_err("couldn't store as CBOR")?; + self.dbt.as_mut().unwrap().put(col, key, v.as_slice()); + Ok(()) + } + + /// Delete key with from a column in the TableDB + pub fn delete(&mut self, col: u32, key: &[u8]) { + self.dbt.as_mut().unwrap().delete(col, key); + } +} + +impl<'a> Drop for TableDBTransaction<'a> { + fn drop(&mut self) { + if self.dbt.is_some() { + warn!("Dropped transaction without commit or rollback"); + } + } +} diff --git a/veilid-core/src/intf/wasm/protected_store.rs b/veilid-core/src/intf/wasm/protected_store.rs index 54ba80fa..8dd3f427 100644 --- a/veilid-core/src/intf/wasm/protected_store.rs +++ b/veilid-core/src/intf/wasm/protected_store.rs @@ -3,20 +3,11 @@ use crate::xx::*; use crate::*; use data_encoding::BASE64URL_NOPAD; use js_sys::*; +use send_wrapper::*; +use serde::{Deserialize, Serialize}; use wasm_bindgen_futures::*; use web_sys::*; -#[wasm_bindgen] -extern "C" { - #[wasm_bindgen(catch, js_name = setPassword, js_namespace = ["global", "wasmhost", "keytar"])] - fn keytar_setPassword(service: &str, account: &str, password: &str) - -> Result; - #[wasm_bindgen(catch, js_name = getPassword, js_namespace = ["global", "wasmhost", "keytar"])] - fn keytar_getPassword(service: &str, account: &str) -> Result; - #[wasm_bindgen(catch, js_name = deletePassword, js_namespace = ["global", "wasmhost", "keytar"])] - fn keytar_deletePassword(service: &str, account: &str) -> Result; -} - #[derive(Clone)] pub struct ProtectedStore { config: VeilidConfig, @@ -71,33 +62,9 @@ impl ProtectedStore { } } - #[instrument(level = "trace", skip(self, value), ret, err)] + //#[instrument(level = "trace", skip(self, value), ret, err)] pub async fn save_user_secret_string(&self, key: &str, value: &str) -> EyreResult { - if is_nodejs() { - let prev = match JsFuture::from( - keytar_getPassword(self.keyring_name().as_str(), key) - .map_err(map_jsvalue_error) - .wrap_err("exception thrown")?, - ) - .await - { - Ok(v) => v.is_truthy(), - Err(_) => false, - }; - - match JsFuture::from( - keytar_setPassword(self.keyring_name().as_str(), key, value) - .map_err(map_jsvalue_error) - .wrap_err("exception thrown")?, - ) - .await - { - Ok(_) => {} - Err(_) => bail!("Failed to set password"), - } - - Ok(prev) - } else if is_browser() { + if is_browser() { let win = match window() { Some(w) => w, None => { @@ -139,24 +106,7 @@ impl ProtectedStore { #[instrument(level = "trace", skip(self), err)] pub async fn load_user_secret_string(&self, key: &str) -> EyreResult> { - if is_nodejs() { - let prev = match JsFuture::from( - keytar_getPassword(self.keyring_name().as_str(), key) - .map_err(map_jsvalue_error) - .wrap_err("exception thrown")?, - ) - .await - { - Ok(p) => p, - Err(_) => JsValue::UNDEFINED, - }; - - if prev.is_undefined() || prev.is_null() { - return Ok(None); - } - - Ok(prev.as_string()) - } else if is_browser() { + if is_browser() { let win = match window() { Some(w) => w, None => { @@ -252,18 +202,7 @@ impl ProtectedStore { #[instrument(level = "trace", skip(self), ret, err)] pub async fn remove_user_secret(&self, key: &str) -> EyreResult { - if is_nodejs() { - match JsFuture::from( - keytar_deletePassword(self.keyring_name().as_str(), key) - .map_err(map_jsvalue_error) - .wrap_err("exception thrown")?, - ) - .await - { - Ok(v) => Ok(v.is_truthy()), - Err(_) => bail!("Failed to delete"), - } - } else if is_browser() { + if is_browser() { let win = match window() { Some(w) => w, None => { diff --git a/veilid-core/src/intf/wasm/system.rs b/veilid-core/src/intf/wasm/system.rs index f30f8a81..00158ad8 100644 --- a/veilid-core/src/intf/wasm/system.rs +++ b/veilid-core/src/intf/wasm/system.rs @@ -19,10 +19,8 @@ extern "C" { pub fn get_timestamp() -> u64 { if utils::is_browser() { return (Date::now() * 1000.0f64) as u64; - } else if utils::is_nodejs() { - return (Date::now() * 1000.0f64) as u64; } else { - panic!("WASM requires browser or nodejs environment"); + panic!("WASM requires browser environment"); } } @@ -85,18 +83,22 @@ pub fn spawn(future: impl Future + Send + 'static) -> MustJoi where Out: Send + 'static, { - MustJoinHandle::new(Bindgen - .spawn_handle(future) - .expect("wasm-bindgen-futures spawn should never error out")) + MustJoinHandle::new( + Bindgen + .spawn_handle(future) + .expect("wasm-bindgen-futures spawn should never error out"), + ) } pub fn spawn_local(future: impl Future + 'static) -> MustJoinHandle where Out: 'static, { - MustJoinHandle::new(Bindgen - .spawn_handle_local(future) - .expect("wasm-bindgen-futures spawn_local should never error out")) + MustJoinHandle::new( + Bindgen + .spawn_handle_local(future) + .expect("wasm-bindgen-futures spawn_local should never error out"), + ) } // pub fn spawn_with_local_set( @@ -114,10 +116,10 @@ where { Bindgen .spawn_handle_local(future) - .expect("wasm-bindgen-futures spawn_local should never error out").detach() + .expect("wasm-bindgen-futures spawn_local should never error out") + .detach() } - pub fn interval(freq_ms: u32, callback: F) -> SendPinBoxFuture<()> where F: Fn() -> FUT + Send + Sync + 'static, @@ -160,12 +162,12 @@ pub async fn get_outbound_relay_peer() -> Option { // pub async fn get_pwa_web_server_config() -> { // if utils::is_browser() { - + // let win = window().unwrap(); // let doc = win.document().unwrap(); // let html_document = document.dyn_into::().unwrap(); // let cookie = html_document.cookie().unwrap(); - + // // let wait_millis = if millis > u32::MAX { // // i32::MAX // // } else { @@ -177,22 +179,14 @@ pub async fn get_outbound_relay_peer() -> Option { // // .unwrap(); // // }); -// // JsFuture::from(promise).await.unwrap(); -// } else if utils::is_nodejs() { -// // let promise = Promise::new(&mut |yes, _| { -// // nodejs_global_set_timeout_with_callback_and_timeout_and_arguments_0(&yes, millis) -// // .unwrap(); -// // }); - // // JsFuture::from(promise).await.unwrap(); // } else { -// panic!("WASM requires browser or nodejs environment"); -// } +// panic!("WASM requires browser environment"); +// } // } - pub async fn txt_lookup>(_host: S) -> EyreResult> { - bail!("wasm does not support txt lookup") + bail!("wasm does not support txt lookup") } pub async fn ptr_lookup(_ip_addr: IpAddr) -> EyreResult { diff --git a/veilid-core/src/intf/wasm/table_store.rs b/veilid-core/src/intf/wasm/table_store.rs index 2e0aaa03..b49481ff 100644 --- a/veilid-core/src/intf/wasm/table_store.rs +++ b/veilid-core/src/intf/wasm/table_store.rs @@ -22,7 +22,7 @@ impl TableStore { opened: BTreeMap::new(), } } - pub fn new(config: VeilidConfig) -> Self { + pub(crate) fn new(config: VeilidConfig) -> Self { Self { config, inner: Arc::new(Mutex::new(Self::new_inner())), @@ -30,12 +30,25 @@ impl TableStore { } } - pub async fn init(&self) -> EyreResult<()> { + /// Delete all known tables + pub async fn delete_all(&self) { + if let Err(e) = self.delete("crypto_caches").await { + error!("failed to delete 'crypto_caches': {}", e); + } + if let Err(e) = self.delete("RouteSpecStore").await { + error!("failed to delete 'RouteSpecStore': {}", e); + } + if let Err(e) = self.delete("routing_table").await { + error!("failed to delete 'routing_table': {}", e); + } + } + + pub(crate) async fn init(&self) -> EyreResult<()> { let _async_guard = self.async_lock.lock().await; Ok(()) } - pub async fn terminate(&self) { + pub(crate) async fn terminate(&self) { let _async_guard = self.async_lock.lock().await; assert!( self.inner.lock().opened.len() == 0, @@ -43,7 +56,7 @@ impl TableStore { ); } - pub fn on_table_db_drop(&self, table: String) { + pub(crate) fn on_table_db_drop(&self, table: String) { let mut inner = self.inner.lock(); match inner.opened.remove(&table) { Some(_) => (), @@ -69,12 +82,14 @@ impl TableStore { }) } + /// Get or create a TableDB database table. If the column count is greater than an + /// existing TableDB's column count, the database will be upgraded to add the missing columns pub async fn open(&self, name: &str, column_count: u32) -> EyreResult { let _async_guard = self.async_lock.lock().await; let table_name = self.get_table_name(name)?; { - let mut inner = self.inner.lock(); + let mut inner = self.inner.lock(); if let Some(table_db_weak_inner) = inner.opened.get(&table_name) { match TableDB::try_new_from_weak_inner(table_db_weak_inner.clone()) { Some(tdb) => { @@ -89,7 +104,10 @@ impl TableStore { let db = Database::open(table_name.clone(), column_count) .await .wrap_err("failed to open tabledb")?; - info!("opened table store '{}' with table name '{:?}' with {} columns", name, table_name, column_count); + info!( + "opened table store '{}' with table name '{:?}' with {} columns", + name, table_name, column_count + ); let table_db = TableDB::new(table_name.clone(), self.clone(), db); @@ -101,11 +119,12 @@ impl TableStore { Ok(table_db) } + /// Delete a TableDB table by name pub async fn delete(&self, name: &str) -> EyreResult { let _async_guard = self.async_lock.lock().await; trace!("TableStore::delete {}", name); let table_name = self.get_table_name(name)?; - + { let inner = self.inner.lock(); if inner.opened.contains_key(&table_name) { @@ -117,9 +136,7 @@ impl TableStore { } } - if utils::is_nodejs() { - unimplemented!(); - } else if utils::is_browser() { + if utils::is_browser() { let out = match Database::delete(table_name.clone()).await { Ok(_) => true, Err(_) => false, diff --git a/veilid-core/src/intf/wasm/utils/mod.rs b/veilid-core/src/intf/wasm/utils/mod.rs index 5a4f853f..fcec19f7 100644 --- a/veilid-core/src/intf/wasm/utils/mod.rs +++ b/veilid-core/src/intf/wasm/utils/mod.rs @@ -15,21 +15,6 @@ extern "C" { pub fn alert(s: &str); } -pub fn is_nodejs() -> bool { - static CACHE: AtomicI8 = AtomicI8::new(-1); - let cache = CACHE.load(Ordering::Relaxed); - if cache != -1 { - return cache != 0; - } - - let res = js_sys::eval("process.release.name === 'node'") - .map(|res| res.is_truthy()) - .unwrap_or_default(); - - CACHE.store(res as i8, Ordering::Relaxed); - res -} - pub fn is_browser() -> bool { static CACHE: AtomicI8 = AtomicI8::new(-1); let cache = CACHE.load(Ordering::Relaxed); @@ -60,24 +45,6 @@ pub fn is_browser() -> bool { // res // } -// pub fn node_require(module: &str) -> JsValue { -// if !is_nodejs() { -// return JsValue::UNDEFINED; -// } - -// let mut home = env!("CARGO_MANIFEST_DIR"); -// if home.len() == 0 { -// home = "."; -// } - -// match js_sys::eval(format!("require(\"{}/{}\")", home, module).as_str()) { -// Ok(v) => v, -// Err(e) => { -// panic!("node_require failed: {:?}", e); -// } -// } -// } - #[derive(ThisError, Debug, Clone, Eq, PartialEq)] #[error("JsValue error")] pub struct JsValueError(String); diff --git a/veilid-core/src/routing_table/bucket.rs b/veilid-core/src/routing_table/bucket.rs index 48199bfc..7ffee2f7 100644 --- a/veilid-core/src/routing_table/bucket.rs +++ b/veilid-core/src/routing_table/bucket.rs @@ -8,6 +8,8 @@ pub struct Bucket { } pub(super) type EntriesIter<'a> = alloc::collections::btree_map::Iter<'a, DHTKey, Arc>; +type BucketData = (Vec<(DHTKey, Vec)>, Option); + fn state_ordering(state: BucketEntryState) -> usize { match state { BucketEntryState::Dead => 0, @@ -25,6 +27,32 @@ impl Bucket { } } + pub(super) fn load_bucket(&mut self, data: &[u8]) -> EyreResult<()> { + let bucket_data: BucketData = + serde_cbor::from_slice::(data).wrap_err("failed to deserialize bucket")?; + + for (k, d) in bucket_data.0 { + let entryinner = serde_cbor::from_slice::(&d) + .wrap_err("failed to deserialize bucket entry")?; + self.entries + .insert(k, Arc::new(BucketEntry::new_with_inner(entryinner))); + } + + self.newest_entry = bucket_data.1; + + Ok(()) + } + pub(super) fn save_bucket(&self) -> EyreResult> { + let mut entry_vec = Vec::new(); + for (k, v) in &self.entries { + let entry_bytes = v.with_mut_inner(|e| serde_cbor::to_vec(e))?; + entry_vec.push((*k, entry_bytes)); + } + let bucket_data: BucketData = (entry_vec, self.newest_entry.clone()); + let out = serde_cbor::to_vec(&bucket_data)?; + Ok(out) + } + pub(super) fn add_entry(&mut self, node_id: DHTKey) -> NodeRef { log_rtab!("Node added: {}", node_id.encode()); diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 6bad7999..422e030a 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -1,5 +1,6 @@ use super::*; use core::sync::atomic::{AtomicU32, Ordering}; +use serde::{Deserialize, Serialize}; /// Reliable pings are done with increased spacing between pings @@ -42,7 +43,7 @@ pub enum BucketEntryState { struct LastConnectionKey(ProtocolType, AddressType); /// Bucket entry information specific to the LocalNetwork RoutingDomain -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct BucketEntryPublicInternet { /// The PublicInternet node info signed_node_info: Option>, @@ -53,7 +54,7 @@ pub struct BucketEntryPublicInternet { } /// Bucket entry information specific to the LocalNetwork RoutingDomain -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct BucketEntryLocalNetwork { /// The LocalNetwork node info signed_node_info: Option>, @@ -63,19 +64,24 @@ pub struct BucketEntryLocalNetwork { node_status: Option, } -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct BucketEntryInner { min_max_version: Option<(u8, u8)>, updated_since_last_network_change: bool, + #[serde(skip)] last_connections: BTreeMap, public_internet: BucketEntryPublicInternet, local_network: BucketEntryLocalNetwork, peer_stats: PeerStats, + #[serde(skip)] latency_stats_accounting: LatencyStatsAccounting, + #[serde(skip)] transfer_stats_accounting: TransferStatsAccounting, #[cfg(feature = "tracking")] + #[serde(skip)] next_track_id: usize, #[cfg(feature = "tracking")] + #[serde(skip)] node_ref_tracks: HashMap, } @@ -657,6 +663,13 @@ impl BucketEntry { } } + pub(super) fn new_with_inner(inner: BucketEntryInner) -> Self { + Self { + ref_count: AtomicU32::new(0), + inner: RwLock::new(inner), + } + } + // Note, that this requires -also- holding the RoutingTable read lock, as an // immutable reference to RoutingTableInner must be passed in to get this // This ensures that an operation on the routing table can not change entries diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index b1d5c4fe..3ab5d8d5 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -154,6 +154,20 @@ impl RoutingTable { pub async fn init(&self) -> EyreResult<()> { debug!("starting routing table init"); + // Set up routing buckets + { + let mut inner = self.inner.write(); + inner.init_buckets(self.clone()); + } + + // Load bucket entries from table db if possible + debug!("loading routing table entries"); + if let Err(e) = self.load_buckets().await { + log_rtab!(warn "Error loading buckets from storage: {}. Resetting.", e); + let mut inner = self.inner.write(); + inner.init_buckets(self.clone()); + } + // Set up routespecstore debug!("starting route spec store init"); let route_spec_store = match RouteSpecStore::load(self.clone()).await { @@ -165,10 +179,10 @@ impl RoutingTable { }; debug!("finished route spec store init"); - let mut inner = self.inner.write(); - inner.init(self.clone())?; - - inner.route_spec_store = Some(route_spec_store); + { + let mut inner = self.inner.write(); + inner.route_spec_store = Some(route_spec_store); + } debug!("finished routing table init"); Ok(()) @@ -188,6 +202,12 @@ impl RoutingTable { error!("kick_buckets_task not stopped: {}", e); } + // Load bucket entries from table db if possible + debug!("saving routing table entries"); + if let Err(e) = self.save_buckets().await { + error!("failed to save routing table entries: {}", e); + } + debug!("saving route spec store"); let rss = { let mut inner = self.inner.write(); @@ -201,12 +221,67 @@ impl RoutingTable { debug!("shutting down routing table"); let mut inner = self.inner.write(); - inner.terminate(); *inner = RoutingTableInner::new(self.unlocked_inner.clone()); debug!("finished routing table terminate"); } + async fn save_buckets(&self) -> EyreResult<()> { + // Serialize all entries + let mut bucketvec: Vec> = Vec::new(); + { + let inner = &*self.inner.read(); + for bucket in &inner.buckets { + bucketvec.push(bucket.save_bucket()?) + } + } + let table_store = self.network_manager().table_store(); + let tdb = table_store.open("routing_table", 1).await?; + let bucket_count = bucketvec.len(); + let mut dbx = tdb.transact(); + if let Err(e) = dbx.store_cbor(0, b"bucket_count", &bucket_count) { + dbx.rollback(); + return Err(e); + } + + for (n, b) in bucketvec.iter().enumerate() { + dbx.store(0, format!("bucket_{}", n).as_bytes(), b) + } + dbx.commit()?; + Ok(()) + } + + async fn load_buckets(&self) -> EyreResult<()> { + // Deserialize all entries + let inner = &mut *self.inner.write(); + + let tstore = self.network_manager().table_store(); + let tdb = tstore.open("routing_table", 1).await?; + let Some(bucket_count): Option = tdb.load_cbor(0, b"bucket_count")? else { + log_rtab!(debug "no bucket count in saved routing table"); + return Ok(()); + }; + if bucket_count != inner.buckets.len() { + // Must have the same number of buckets + warn!("bucket count is different, not loading routing table"); + return Ok(()); + } + let mut bucketdata_vec: Vec> = Vec::new(); + for n in 0..bucket_count { + let Some(bucketdata): Option> = + tdb.load(0, format!("bucket_{}", n).as_bytes())? else { + warn!("bucket data not loading, skipping loading routing table"); + return Ok(()); + }; + bucketdata_vec.push(bucketdata); + } + for n in 0..bucket_count { + inner.buckets[n].load_bucket(&bucketdata_vec[n])?; + } + + Ok(()) + } + /// Set up the local network routing domain with our local routing table configuration pub fn configure_local_network_routing_domain(&self, local_networks: Vec<(IpAddr, IpAddr)>) { log_net!(debug "configure_local_network_routing_domain: {:#?}", local_networks); diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index e0345437..75d4f586 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -225,7 +225,23 @@ impl RouteSpecStore { let table_store = routing_table.network_manager().table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; let mut content: RouteSpecStoreContent = - rsstdb.load_cbor(0, b"content").await?.unwrap_or_default(); + rsstdb.load_cbor(0, b"content")?.unwrap_or_default(); + + // Look up all route hop noderefs since we can't serialize those + let mut dead_keys = Vec::new(); + for (k, rsd) in &mut content.details { + for h in &rsd.hops { + let Some(nr) = routing_table.lookup_node_ref(*h) else { + dead_keys.push(*k); + break; + }; + rsd.hop_node_refs.push(nr); + } + } + for k in dead_keys { + log_rtab!(debug "no entry, killing off private route: {}", k.encode()); + content.details.remove(&k); + } // Load secrets from pstore let pstore = routing_table.network_manager().protected_store(); @@ -280,7 +296,7 @@ impl RouteSpecStore { .network_manager() .table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; - rsstdb.store_cbor(0, b"content", &content).await?; + rsstdb.store_cbor(0, b"content", &content)?; // // Keep secrets in protected store as well let pstore = self @@ -1168,10 +1184,9 @@ impl RouteSpecStore { let mut buffer = vec![]; capnp::serialize_packed::write_message(&mut buffer, &pr_message) + .map_err(RPCError::internal) .wrap_err("failed to convert builder to vec")?; Ok(buffer) - - // builder_to_vec(pr_message).wrap_err("failed to convert builder to vec") } /// Convert binary blob to private route @@ -1180,11 +1195,12 @@ impl RouteSpecStore { blob.as_slice(), capnp::message::ReaderOptions::new(), ) + .map_err(RPCError::internal) .wrap_err("failed to make message reader")?; - //let reader = ::capnp::message::Reader::new(RPCMessageData::new(blob), Default::default()); let pr_reader = reader .get_root::() + .map_err(RPCError::internal) .wrap_err("failed to make reader for private_route")?; decode_private_route(&pr_reader).wrap_err("failed to decode private route") } diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index cab84df6..6435cd1e 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -323,19 +323,16 @@ impl RoutingTableInner { } } - pub fn init(&mut self, routing_table: RoutingTable) -> EyreResult<()> { + pub fn init_buckets(&mut self, routing_table: RoutingTable) { // Size the buckets (one per bit) + self.buckets.clear(); self.buckets.reserve(DHT_KEY_LENGTH * 8); for _ in 0..DHT_KEY_LENGTH * 8 { let bucket = Bucket::new(routing_table.clone()); self.buckets.push(bucket); } - - Ok(()) } - pub fn terminate(&mut self) {} - pub fn configure_local_network_routing_domain( &mut self, local_networks: Vec<(IpAddr, IpAddr)>, diff --git a/veilid-core/src/tests/common/test_table_store.rs b/veilid-core/src/tests/common/test_table_store.rs index 9a4b73a9..f10f322e 100644 --- a/veilid-core/src/tests/common/test_table_store.rs +++ b/veilid-core/src/tests/common/test_table_store.rs @@ -59,78 +59,67 @@ pub async fn test_store_delete_load(ts: TableStore) { ); assert_eq!( - db.load(0, b"foo").await.unwrap(), + db.load(0, b"foo").unwrap(), None, "should not load missing key" ); assert!( - db.store(1, b"foo", b"1234567890").await.is_ok(), + db.store(1, b"foo", b"1234567890").is_ok(), "should store new key" ); assert_eq!( - db.load(0, b"foo").await.unwrap(), + db.load(0, b"foo").unwrap(), None, "should not load missing key" ); - assert_eq!( - db.load(1, b"foo").await.unwrap(), - Some(b"1234567890".to_vec()) - ); + assert_eq!(db.load(1, b"foo").unwrap(), Some(b"1234567890".to_vec())); assert!( - db.store(1, b"bar", b"FNORD").await.is_ok(), + db.store(1, b"bar", b"FNORD").is_ok(), "should store new key" ); assert!( - db.store(0, b"bar", b"ABCDEFGHIJKLMNOPQRSTUVWXYZ") - .await - .is_ok(), + db.store(0, b"bar", b"ABCDEFGHIJKLMNOPQRSTUVWXYZ").is_ok(), "should store new key" ); assert!( - db.store(2, b"bar", b"FNORD").await.is_ok(), + db.store(2, b"bar", b"FNORD").is_ok(), "should store new key" ); assert!( - db.store(2, b"baz", b"QWERTY").await.is_ok(), + db.store(2, b"baz", b"QWERTY").is_ok(), "should store new key" ); assert!( - db.store(2, b"bar", b"QWERTYUIOP").await.is_ok(), + db.store(2, b"bar", b"QWERTYUIOP").is_ok(), "should store new key" ); - assert_eq!(db.load(1, b"bar").await.unwrap(), Some(b"FNORD".to_vec())); + assert_eq!(db.load(1, b"bar").unwrap(), Some(b"FNORD".to_vec())); assert_eq!( - db.load(0, b"bar").await.unwrap(), + db.load(0, b"bar").unwrap(), Some(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ".to_vec()) ); - assert_eq!( - db.load(2, b"bar").await.unwrap(), - Some(b"QWERTYUIOP".to_vec()) - ); - assert_eq!(db.load(2, b"baz").await.unwrap(), Some(b"QWERTY".to_vec())); + assert_eq!(db.load(2, b"bar").unwrap(), Some(b"QWERTYUIOP".to_vec())); + assert_eq!(db.load(2, b"baz").unwrap(), Some(b"QWERTY".to_vec())); - assert_eq!(db.delete(1, b"bar").await.unwrap(), true); - assert_eq!(db.delete(1, b"bar").await.unwrap(), false); + assert_eq!(db.delete(1, b"bar").unwrap(), true); + assert_eq!(db.delete(1, b"bar").unwrap(), false); assert!( - db.delete(4, b"bar").await.is_err(), + db.delete(4, b"bar").is_err(), "can't delete from column that doesn't exist" ); drop(db); let db = ts.open("test", 3).await.expect("should have opened"); - assert_eq!(db.load(1, b"bar").await.unwrap(), None); + assert_eq!(db.load(1, b"bar").unwrap(), None); assert_eq!( - db.load(0, b"bar").await.unwrap(), + db.load(0, b"bar").unwrap(), Some(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ".to_vec()) ); - assert_eq!( - db.load(2, b"bar").await.unwrap(), - Some(b"QWERTYUIOP".to_vec()) - ); - assert_eq!(db.load(2, b"baz").await.unwrap(), Some(b"QWERTY".to_vec())); + assert_eq!(db.load(2, b"bar").unwrap(), Some(b"QWERTYUIOP".to_vec())); + assert_eq!(db.load(2, b"baz").unwrap(), Some(b"QWERTY".to_vec())); } pub async fn test_cbor(ts: TableStore) { @@ -140,11 +129,11 @@ pub async fn test_cbor(ts: TableStore) { let db = ts.open("test", 3).await.expect("should have opened"); let (dht_key, _) = generate_secret(); - assert!(db.store_cbor(0, b"asdf", &dht_key).await.is_ok()); + assert!(db.store_cbor(0, b"asdf", &dht_key).is_ok()); - assert_eq!(db.load_cbor::(0, b"qwer").await.unwrap(), None); + assert_eq!(db.load_cbor::(0, b"qwer").unwrap(), None); - let d = match db.load_cbor::(0, b"asdf").await { + let d = match db.load_cbor::(0, b"asdf") { Ok(x) => x, Err(e) => { panic!("couldn't decode cbor: {}", e); @@ -153,12 +142,12 @@ pub async fn test_cbor(ts: TableStore) { assert_eq!(d, Some(dht_key), "keys should be equal"); assert!( - db.store(1, b"foo", b"1234567890").await.is_ok(), + db.store(1, b"foo", b"1234567890").is_ok(), "should store new key" ); assert!( - db.load_cbor::(1, b"foo").await.is_err(), + db.load_cbor::(1, b"foo").is_err(), "should fail to load cbor" ); } diff --git a/veilid-core/src/tests/common/test_veilid_config.rs b/veilid-core/src/tests/common/test_veilid_config.rs index c9f52197..423d93fa 100644 --- a/veilid-core/src/tests/common/test_veilid_config.rs +++ b/veilid-core/src/tests/common/test_veilid_config.rs @@ -203,11 +203,11 @@ fn config_callback(key: String) -> ConfigCallbackReturn { "network.routing_table.limit_attached_good" => Ok(Box::new(8u32)), "network.routing_table.limit_attached_weak" => Ok(Box::new(4u32)), "network.rpc.concurrency" => Ok(Box::new(2u32)), - "network.rpc.queue_size" => Ok(Box::new(128u32)), + "network.rpc.queue_size" => Ok(Box::new(1024u32)), "network.rpc.max_timestamp_behind_ms" => Ok(Box::new(Some(10_000u32))), "network.rpc.max_timestamp_ahead_ms" => Ok(Box::new(Some(10_000u32))), "network.rpc.timeout_ms" => Ok(Box::new(10_000u32)), - "network.rpc.max_route_hop_count" => Ok(Box::new(7u8)), + "network.rpc.max_route_hop_count" => Ok(Box::new(4u8)), "network.rpc.default_route_hop_count" => Ok(Box::new(2u8)), "network.dht.resolve_node_timeout_ms" => Ok(Box::new(Option::::None)), "network.dht.resolve_node_count" => Ok(Box::new(20u32)), @@ -323,9 +323,9 @@ pub async fn test_config() { assert_eq!(inner.network.bootstrap, Vec::::new()); assert_eq!(inner.network.bootstrap_nodes, Vec::::new()); assert_eq!(inner.network.rpc.concurrency, 2u32); - assert_eq!(inner.network.rpc.queue_size, 128u32); + assert_eq!(inner.network.rpc.queue_size, 1024u32); assert_eq!(inner.network.rpc.timeout_ms, 10_000u32); - assert_eq!(inner.network.rpc.max_route_hop_count, 7u8); + assert_eq!(inner.network.rpc.max_route_hop_count, 4u8); assert_eq!(inner.network.rpc.default_route_hop_count, 2u8); assert_eq!(inner.network.routing_table.limit_over_attached, 64u32); assert_eq!(inner.network.routing_table.limit_fully_attached, 32u32); diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index ee00b36e..9bc78d7d 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -55,10 +55,7 @@ fn get_route_id(rss: RouteSpecStore) -> impl Fn(&str) -> Option { } fn get_safety_selection(text: &str, rss: RouteSpecStore) -> Option { - if text.len() == 0 { - return None; - } - if &text[0..1] == "-" { + if text.len() != 0 && &text[0..1] == "-" { // Unsafe let text = &text[1..]; let seq = get_sequencing(text).unwrap_or(Sequencing::NoPreference); diff --git a/veilid-core/src/xx/mod.rs b/veilid-core/src/xx/mod.rs index d9f48ffd..a4c8e008 100644 --- a/veilid-core/src/xx/mod.rs +++ b/veilid-core/src/xx/mod.rs @@ -41,61 +41,42 @@ pub type PinBoxFutureLifetime<'a, T> = PinBox + 'a>; pub type SendPinBoxFuture = PinBox + Send + 'static>; pub type SendPinBoxFutureLifetime<'a, T> = PinBox + Send + 'a>; +pub use std::borrow::{Cow, ToOwned}; +pub use std::boxed::Box; +pub use std::cell::RefCell; +pub use std::cmp; +pub use std::collections::btree_map::BTreeMap; +pub use std::collections::btree_set::BTreeSet; +pub use std::collections::hash_map::HashMap; +pub use std::collections::hash_set::HashSet; +pub use std::collections::LinkedList; +pub use std::collections::VecDeque; +pub use std::convert::{TryFrom, TryInto}; +pub use std::fmt; +pub use std::future::Future; +pub use std::mem; +pub use std::ops::{Fn, FnMut, FnOnce}; +pub use std::pin::Pin; +pub use std::rc::Rc; +pub use std::string::String; +pub use std::sync::atomic::{AtomicBool, Ordering}; +pub use std::sync::{Arc, Weak}; +pub use std::task; +pub use std::time::Duration; +pub use std::vec::Vec; + cfg_if! { if #[cfg(target_arch = "wasm32")] { - pub use alloc::string::String; - pub use alloc::vec::Vec; - pub use alloc::collections::LinkedList; - pub use alloc::collections::VecDeque; - pub use alloc::collections::btree_map::BTreeMap; - pub use alloc::collections::btree_set::BTreeSet; - pub use hashbrown::hash_map::HashMap; - pub use hashbrown::hash_set::HashSet; - pub use alloc::boxed::Box; - pub use alloc::borrow::{Cow, ToOwned}; pub use wasm_bindgen::prelude::*; - pub use core::cmp; - pub use core::convert::{TryFrom, TryInto}; - pub use core::mem; - pub use core::fmt; - pub use alloc::rc::Rc; - pub use core::cell::RefCell; - pub use core::task; - pub use core::future::Future; - pub use core::time::Duration; - pub use core::pin::Pin; - pub use core::sync::atomic::{Ordering, AtomicBool}; - pub use alloc::sync::{Arc, Weak}; - pub use core::ops::{FnOnce, FnMut, Fn}; + pub use async_lock::Mutex as AsyncMutex; pub use async_lock::MutexGuard as AsyncMutexGuard; pub use async_lock::MutexGuardArc as AsyncMutexGuardArc; - pub use no_std_net::{ SocketAddr, SocketAddrV4, SocketAddrV6, ToSocketAddrs, IpAddr, Ipv4Addr, Ipv6Addr }; pub use async_executors::JoinHandle as LowLevelJoinHandle; + + pub use no_std_net::{ SocketAddr, SocketAddrV4, SocketAddrV6, ToSocketAddrs, IpAddr, Ipv4Addr, Ipv6Addr }; } else { - pub use std::string::String; - pub use std::vec::Vec; - pub use std::collections::LinkedList; - pub use std::collections::VecDeque; - pub use std::collections::btree_map::BTreeMap; - pub use std::collections::btree_set::BTreeSet; - pub use std::collections::hash_map::HashMap; - pub use std::collections::hash_set::HashSet; - pub use std::boxed::Box; - pub use std::borrow::{Cow, ToOwned}; - pub use std::cmp; - pub use std::convert::{TryFrom, TryInto}; - pub use std::mem; - pub use std::fmt; - pub use std::sync::atomic::{Ordering, AtomicBool}; - pub use std::sync::{Arc, Weak}; - pub use std::rc::Rc; - pub use std::cell::RefCell; - pub use std::task; - pub use std::future::Future; - pub use std::time::Duration; - pub use std::pin::Pin; - pub use std::ops::{FnOnce, FnMut, Fn}; + cfg_if! { if #[cfg(feature="rt-async-std")] { pub use async_std::sync::Mutex as AsyncMutex; From e672ae03195f37b814b7b4d1b0f56f0aa29a59e5 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 9 Nov 2022 17:11:35 -0500 Subject: [PATCH 45/67] checkpoint --- Cargo.lock | 94 +- veilid-core/Cargo.toml | 5 +- veilid-core/proto/veilid.capnp | 43 +- veilid-core/src/attachment_manager.rs | 2 +- veilid-core/src/crypto/key.rs | 12 +- .../src/intf/native/protected_store.rs | 43 +- veilid-core/src/intf/table_db.rs | 66 +- veilid-core/src/intf/wasm/protected_store.rs | 16 +- veilid-core/src/network_manager/tasks.rs | 2 +- veilid-core/src/routing_table/bucket.rs | 44 +- veilid-core/src/routing_table/bucket_entry.rs | 55 +- veilid-core/src/routing_table/mod.rs | 19 +- veilid-core/src/routing_table/node_ref.rs | 74 +- .../src/routing_table/route_spec_store.rs | 14 +- .../src/routing_table/routing_domains.rs | 50 +- .../src/routing_table/routing_table_inner.rs | 7 +- veilid-core/src/rpc_processor/coders/mod.rs | 4 + .../src/rpc_processor/coders/network_class.rs | 3 +- .../src/rpc_processor/coders/node_info.rs | 27 +- .../coders/operations/operation.rs | 2 +- .../coders/operations/operation_find_block.rs | 4 +- .../coders/operations/operation_find_node.rs | 2 +- .../coders/operations/operation_get_value.rs | 2 +- .../operations/operation_node_info_update.rs | 2 +- .../coders/operations/operation_set_value.rs | 2 +- .../operations/operation_supply_block.rs | 2 +- .../operations/operation_watch_value.rs | 2 +- .../src/rpc_processor/coders/peer_info.rs | 8 +- .../coders/private_safety_route.rs | 2 +- .../src/rpc_processor/coders/signal_info.rs | 4 +- .../coders/signed_direct_node_info.rs | 43 + .../rpc_processor/coders/signed_node_info.rs | 50 +- .../coders/signed_relayed_node_info.rs | 67 + veilid-core/src/rpc_processor/mod.rs | 10 +- .../src/tests/common/test_table_store.rs | 18 +- .../src/tests/common/test_veilid_core.rs | 71 + veilid-core/src/veilid_api/debug.rs | 1 - veilid-core/src/veilid_api/mod.rs | 1147 +++++++++++++---- .../src/veilid_api/serialize_helpers.rs | 80 ++ 39 files changed, 1676 insertions(+), 423 deletions(-) create mode 100644 veilid-core/src/rpc_processor/coders/signed_direct_node_info.rs create mode 100644 veilid-core/src/rpc_processor/coders/signed_relayed_node_info.rs diff --git a/Cargo.lock b/Cargo.lock index 507882b7..5ed60b92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -672,6 +672,27 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" +[[package]] +name = "bytecheck" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d11cac2c12b5adc6570dad2ee1b87eff4955dac476fe12d81e5fdd352e52406f" +dependencies = [ + "bytecheck_derive", + "ptr_meta", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13e576ebe98e605500b3c8041bb888e966653577172df6dd97398714eb30b9bf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "byteorder" version = "1.4.3" @@ -3871,6 +3892,26 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -4051,6 +4092,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rend" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79af64b4b6362ffba04eef3a4e10829718a4896dac19daa741851c86781edf95" +dependencies = [ + "bytecheck", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -4076,6 +4126,31 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rkyv" +version = "0.7.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cec2b3485b07d96ddfd3134767b8a447b45ea4eb91448d0a35180ec0ffd5ed15" +dependencies = [ + "bytecheck", + "hashbrown", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eaedadc88b53e36dd32d940ed21ae4d850d5916f2581526921f553a72ac34c4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "rlp" version = "0.5.2" @@ -4274,6 +4349,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "secrecy" version = "0.7.0" @@ -4371,6 +4452,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_bytes" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc50e8183eeeb6178dcb167ae34a8051d63535023ae38b5d8d12beae193d37b" +dependencies = [ + "serde", +] + [[package]] name = "serde_cbor" version = "0.11.2" @@ -5477,6 +5567,7 @@ dependencies = [ "backtrace", "blake3", "bugsalot", + "bytecheck", "capnp", "capnpc", "cfg-if 1.0.0", @@ -5520,6 +5611,7 @@ dependencies = [ "owo-colors", "parking_lot 0.12.1", "rand 0.7.3", + "rkyv", "rtnetlink", "rusqlite", "rust-fsm", @@ -5529,7 +5621,7 @@ dependencies = [ "send_wrapper 0.6.0", "serde", "serde-big-array", - "serde_cbor", + "serde_bytes", "serde_json", "serial_test", "simplelog 0.12.0", diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index cb828dfc..905ccad1 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -59,6 +59,9 @@ rtnetlink = { version = "^0", default-features = false, optional = true } async-std-resolver = { version = "^0", optional = true } trust-dns-resolver = { version = "^0", optional = true } keyvaluedb = { path = "../external/keyvaluedb/keyvaluedb" } +serde_bytes = { version = "^0" } +rkyv = { version = "^0", default_features = false, features = ["std", "alloc", "strict", "size_64", "archive_le", "validation"] } +bytecheck = "^0" # Dependencies for native builds only # Linux, Windows, Mac, iOS, Android @@ -82,7 +85,6 @@ futures-util = { version = "^0", default-features = false, features = ["async-aw keyvaluedb-sqlite = { path = "../external/keyvaluedb/keyvaluedb-sqlite" } data-encoding = { version = "^2" } serde = { version = "^1", features = ["derive" ] } -serde_cbor = { version = "^0" } serde_json = { version = "^1" } socket2 = "^0" bugsalot = "^0" @@ -99,7 +101,6 @@ no-std-net = { path = "../external/no-std-net", features = ["serde"] } keyvaluedb-web = { path = "../external/keyvaluedb/keyvaluedb-web" } data-encoding = { version = "^2", default_features = false, features = ["alloc"] } serde = { version = "^1", default-features = false, features = ["derive", "alloc"] } -serde_cbor = { version = "^0", default-features = false, features = ["alloc"] } serde_json = { version = "^1", default-features = false, features = ["alloc"] } getrandom = { version = "^0", features = ["js"] } ws_stream_wasm = "^0" diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index b3c19e91..c3f00bcf 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -173,7 +173,7 @@ struct ValueKey { # } struct ValueData { - data @0 :Data; # value or subvalue contents in CBOR format + data @0 :Data; # value or subvalue contents seq @1 :ValueSeqNum; # sequence number of value } @@ -181,9 +181,10 @@ struct ValueData { ############################## enum NetworkClass { - inboundCapable @0; # I = Inbound capable without relay, may require signal - outboundOnly @1; # O = Outbound only, inbound relay required except with reverse connect signal - webApp @2; # W = PWA, outbound relay is required in most cases + invalid @0; # X = Invalid network class, network is not yet set up + inboundCapable @1; # I = Inbound capable without relay, may require signal + outboundOnly @2; # O = Outbound only, inbound relay required except with reverse connect signal + webApp @3; # W = PWA, outbound relay is required in most cases } enum DialInfoClass { @@ -232,6 +233,10 @@ struct AddressTypeSet { ipv6 @1 :Bool; } +struct SenderInfo { + socketAddress @0 :SocketAddress; # socket address that for the sending peer +} + struct NodeInfo { networkClass @0 :NetworkClass; # network class of this node outboundProtocols @1 :ProtocolTypeSet; # protocols that can go outbound @@ -239,17 +244,27 @@ struct NodeInfo { minVersion @3 :UInt8; # minimum protocol version for rpc maxVersion @4 :UInt8; # maximum protocol version for rpc dialInfoDetailList @5 :List(DialInfoDetail); # inbound dial info details for this node - relayPeerInfo @6 :PeerInfo; # (optional) relay peer info for this node +} + +struct SignedDirectNodeInfo { + nodeInfo @0 :NodeInfo; # node info + timestamp @1 :UInt64; # when signed node info was generated + signature @2 :Signature; # signature +} + +struct SignedRelayedNodeInfo { + nodeInfo @0 :NodeInfo; # node info + relayId @1 :NodeID; # node id for relay + relayInfo @2 :SignedDirectNodeInfo; # signed node info for relay + timestamp @3 :UInt64; # when signed node info was generated + signature @4 :Signature; # signature } struct SignedNodeInfo { - nodeInfo @0 :NodeInfo; # node info - signature @1 :Signature; # signature - timestamp @2 :UInt64; # when signed node info was generated -} - -struct SenderInfo { - socketAddress @0 :SocketAddress; # socket address that for the sending peer + union { + direct @0 :SignedDirectNodeInfo; # node info for nodes reachable without a relay + relayed @1 :SignedRelayedNodeInfo; # node info for nodes requiring a relay + } } struct PeerInfo { @@ -326,7 +341,7 @@ struct OperationGetValueA { struct OperationSetValueQ { key @0 :ValueKey; # key for value to update - value @1 :ValueData; # value or subvalue contents in CBOR format (older or equal seq number gets dropped) + value @1 :ValueData; # value or subvalue contents (older or equal seq number gets dropped) } struct OperationSetValueA { @@ -347,7 +362,7 @@ struct OperationWatchValueA { struct OperationValueChanged { key @0 :ValueKey; # key for value that changed - value @1 :ValueData; # value or subvalue contents in CBOR format with sequence number + value @1 :ValueData; # value or subvalue contents with sequence number } struct OperationSupplyBlockQ { diff --git a/veilid-core/src/attachment_manager.rs b/veilid-core/src/attachment_manager.rs index 8c304e44..8afafcdd 100644 --- a/veilid-core/src/attachment_manager.rs +++ b/veilid-core/src/attachment_manager.rs @@ -9,7 +9,7 @@ use core::fmt; use serde::*; state_machine! { - derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize) + derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,) pub Attachment(Detached) //--- Detached(AttachRequested) => Attaching [StartAttachment], diff --git a/veilid-core/src/crypto/key.rs b/veilid-core/src/crypto/key.rs index c0c1934f..79ae26e5 100644 --- a/veilid-core/src/crypto/key.rs +++ b/veilid-core/src/crypto/key.rs @@ -12,7 +12,6 @@ use digest::generic_array::typenum::U64; use digest::{Digest, Output}; use ed25519_dalek::{Keypair, PublicKey, Signature}; use generic_array::GenericArray; -use serde::{Deserialize, Serialize}; ////////////////////////////////////////////////////////////////////// @@ -39,13 +38,14 @@ pub const DHT_SIGNATURE_LENGTH_ENCODED: usize = 86; macro_rules! byte_array_type { ($name:ident, $size:expr) => { - #[derive(Clone, Copy)] + #[derive(Clone, Copy, RkyvArchive, RkyvSerialize, RkyvDeserialize)] + #[archive_attr(repr(C), derive(CheckBytes))] pub struct $name { pub bytes: [u8; $size], pub valid: bool, } - impl Serialize for $name { + impl serde::Serialize for $name { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, @@ -56,16 +56,16 @@ macro_rules! byte_array_type { } else { s = "".to_owned(); } - s.serialize(serializer) + serde::Serialize::serialize(&s, serializer) } } - impl<'de> Deserialize<'de> for $name { + impl<'de> serde::Deserialize<'de> for $name { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { - let s = String::deserialize(deserializer)?; + let s = ::deserialize(deserializer)?; if s == "" { return Ok($name::default()); } diff --git a/veilid-core/src/intf/native/protected_store.rs b/veilid-core/src/intf/native/protected_store.rs index 3ab88a12..a784189e 100644 --- a/veilid-core/src/intf/native/protected_store.rs +++ b/veilid-core/src/intf/native/protected_store.rs @@ -2,7 +2,6 @@ use crate::xx::*; use crate::*; use data_encoding::BASE64URL_NOPAD; use keyring_manager::*; -use serde::{Deserialize, Serialize}; use std::path::Path; pub struct ProtectedStoreInner { @@ -144,18 +143,31 @@ impl ProtectedStore { } #[instrument(level = "trace", skip(self, value))] - pub async fn save_user_secret_cbor(&self, key: &str, value: &T) -> EyreResult + pub async fn save_user_secret_rkyv(&self, key: &str, value: &T) -> EyreResult where - T: Serialize, + T: RkyvSerialize>, { - let v = serde_cbor::to_vec(value).wrap_err("couldn't store as CBOR")?; + let v = to_rkyv(value)?; + self.save_user_secret(&key, &v).await + } + + #[instrument(level = "trace", skip(self, value))] + pub async fn save_user_secret_json(&self, key: &str, value: &T) -> EyreResult + where + T: serde::Serialize, + { + let v = serde_json::to_vec(value)?; self.save_user_secret(&key, &v).await } #[instrument(level = "trace", skip(self))] - pub async fn load_user_secret_cbor(&self, key: &str) -> EyreResult> + pub async fn load_user_secret_rkyv(&self, key: &str) -> EyreResult> where - T: for<'de> Deserialize<'de>, + T: RkyvArchive, + ::Archived: + for<'t> bytecheck::CheckBytes>, + ::Archived: + rkyv::Deserialize, { let out = self.load_user_secret(key).await?; let b = match out { @@ -165,7 +177,24 @@ impl ProtectedStore { } }; - let obj = serde_cbor::from_slice::(&b).wrap_err("failed to deserialize")?; + let obj = from_rkyv(b)?; + Ok(Some(obj)) + } + + #[instrument(level = "trace", skip(self))] + pub async fn load_user_secret_json(&self, key: &str) -> EyreResult> + where + T: for<'de> serde::de::Deserialize<'de>, + { + let out = self.load_user_secret(key).await?; + let b = match out { + Some(v) => v, + None => { + return Ok(None); + } + }; + + let obj = serde_json::from_slice(&b)?; Ok(Some(obj)) } diff --git a/veilid-core/src/intf/table_db.rs b/veilid-core/src/intf/table_db.rs index 09fbe5a3..ddf6cf72 100644 --- a/veilid-core/src/intf/table_db.rs +++ b/veilid-core/src/intf/table_db.rs @@ -85,12 +85,25 @@ impl TableDB { db.write(dbt).wrap_err("failed to store key") } - /// Store a key in CBOR format with a value in a column in the TableDB. Performs a single transaction immediately. - pub fn store_cbor(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()> + /// Store a key in rkyv format with a value in a column in the TableDB. Performs a single transaction immediately. + pub fn store_rkyv(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()> where - T: Serialize, + T: RkyvSerialize>, { - let v = serde_cbor::to_vec(value).wrap_err("couldn't store as CBOR")?; + let v = to_rkyv(value)?; + + let db = &self.inner.lock().database; + let mut dbt = db.transaction(); + dbt.put(col, key, v.as_slice()); + db.write(dbt).wrap_err("failed to store key") + } + + /// Store a key in json format with a value in a column in the TableDB. Performs a single transaction immediately. + pub fn store_json(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()> + where + T: serde::Serialize, + { + let v = serde_json::to_vec(value)?; let db = &self.inner.lock().database; let mut dbt = db.transaction(); @@ -104,10 +117,14 @@ impl TableDB { db.get(col, key).wrap_err("failed to get key") } - /// Read a key from a column in the TableDB immediately, in CBOR format. - pub fn load_cbor(&self, col: u32, key: &[u8]) -> EyreResult> + /// Read an rkyv key from a column in the TableDB immediately + pub fn load_rkyv(&self, col: u32, key: &[u8]) -> EyreResult> where - T: for<'de> Deserialize<'de>, + T: RkyvArchive, + ::Archived: + for<'t> bytecheck::CheckBytes>, + ::Archived: + rkyv::Deserialize, { let db = &self.inner.lock().database; let out = db.get(col, key).wrap_err("failed to get key")?; @@ -117,7 +134,24 @@ impl TableDB { return Ok(None); } }; - let obj = serde_cbor::from_slice::(&b).wrap_err("failed to deserialize")?; + let obj = from_rkyv(b)?; + Ok(Some(obj)) + } + + /// Read an serde-json key from a column in the TableDB immediately + pub fn load_json(&self, col: u32, key: &[u8]) -> EyreResult> + where + T: for<'de> serde::Deserialize<'de>, + { + let db = &self.inner.lock().database; + let out = db.get(col, key).wrap_err("failed to get key")?; + let b = match out { + Some(v) => v, + None => { + return Ok(None); + } + }; + let obj = serde_json::from_slice(&b)?; Ok(Some(obj)) } @@ -176,12 +210,22 @@ impl<'a> TableDBTransaction<'a> { self.dbt.as_mut().unwrap().put(col, key, value); } - /// Store a key in CBOR format with a value in a column in the TableDB - pub fn store_cbor(&mut self, col: u32, key: &[u8], value: &T) -> EyreResult<()> + /// Store a key in rkyv format with a value in a column in the TableDB + pub fn store_rkyv(&mut self, col: u32, key: &[u8], value: &T) -> EyreResult<()> + where + T: RkyvSerialize>, + { + let v = to_rkyv(value)?; + self.dbt.as_mut().unwrap().put(col, key, v.as_slice()); + Ok(()) + } + + /// Store a key in rkyv format with a value in a column in the TableDB + pub fn store_json(&mut self, col: u32, key: &[u8], value: &T) -> EyreResult<()> where T: Serialize, { - let v = serde_cbor::to_vec(value).wrap_err("couldn't store as CBOR")?; + let v = serde_json::to_vec(value)?; self.dbt.as_mut().unwrap().put(col, key, v.as_slice()); Ok(()) } diff --git a/veilid-core/src/intf/wasm/protected_store.rs b/veilid-core/src/intf/wasm/protected_store.rs index 8dd3f427..67a4d1da 100644 --- a/veilid-core/src/intf/wasm/protected_store.rs +++ b/veilid-core/src/intf/wasm/protected_store.rs @@ -136,18 +136,22 @@ impl ProtectedStore { } #[instrument(level = "trace", skip(self, value))] - pub async fn save_user_secret_cbor(&self, key: &str, value: &T) -> EyreResult + pub async fn save_user_secret_frozen(&self, key: &str, value: &T) -> EyreResult where - T: Serialize, + T: RkyvSerialize>, { - let v = serde_cbor::to_vec(value).wrap_err("couldn't store as CBOR")?; + let v = to_frozen(value)?; self.save_user_secret(&key, &v).await } #[instrument(level = "trace", skip(self))] - pub async fn load_user_secret_cbor(&self, key: &str) -> EyreResult> + pub async fn load_user_secret_frozen(&self, key: &str) -> EyreResult> where - T: for<'de> Deserialize<'de>, + T: RkyvArchive, + ::Archived: + for<'t> bytecheck::CheckBytes>, + ::Archived: + rkyv::Deserialize, { let out = self.load_user_secret(key).await?; let b = match out { @@ -157,7 +161,7 @@ impl ProtectedStore { } }; - let obj = serde_cbor::from_slice::(&b).wrap_err("failed to deserialize")?; + let obj = from_frozen(&b)?; Ok(Some(obj)) } diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index a12d6018..07554099 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -295,7 +295,7 @@ impl NetworkManager { if let Some(nr) = routing_table.register_node_with_signed_node_info( RoutingDomain::PublicInternet, k, - SignedNodeInfo::with_no_signature(NodeInfo { + SignedDirectNodeInfo::with_no_signature(NodeInfo { network_class: NetworkClass::InboundCapable, // Bootstraps are always inbound capable outbound_protocols: ProtocolTypeSet::only(ProtocolType::UDP), // Bootstraps do not participate in relaying and will not make outbound requests, but will have UDP enabled address_types: AddressTypeSet::all(), // Bootstraps are always IPV4 and IPV6 capable diff --git a/veilid-core/src/routing_table/bucket.rs b/veilid-core/src/routing_table/bucket.rs index 7ffee2f7..3b17c585 100644 --- a/veilid-core/src/routing_table/bucket.rs +++ b/veilid-core/src/routing_table/bucket.rs @@ -8,7 +8,19 @@ pub struct Bucket { } pub(super) type EntriesIter<'a> = alloc::collections::btree_map::Iter<'a, DHTKey, Arc>; -type BucketData = (Vec<(DHTKey, Vec)>, Option); +#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] +struct BucketEntryData { + key: DHTKey, + value: Vec, +} + +#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] +struct BucketData { + entries: Vec, + newest_entry: Option, +} fn state_ordering(state: BucketEntryState) -> usize { match state { @@ -27,29 +39,33 @@ impl Bucket { } } - pub(super) fn load_bucket(&mut self, data: &[u8]) -> EyreResult<()> { - let bucket_data: BucketData = - serde_cbor::from_slice::(data).wrap_err("failed to deserialize bucket")?; + pub(super) fn load_bucket(&mut self, data: Vec) -> EyreResult<()> { + let bucket_data: BucketData = from_rkyv(data)?; - for (k, d) in bucket_data.0 { - let entryinner = serde_cbor::from_slice::(&d) - .wrap_err("failed to deserialize bucket entry")?; + for e in bucket_data.entries { + let entryinner = from_rkyv(e.value).wrap_err("failed to deserialize bucket entry")?; self.entries - .insert(k, Arc::new(BucketEntry::new_with_inner(entryinner))); + .insert(e.key, Arc::new(BucketEntry::new_with_inner(entryinner))); } - self.newest_entry = bucket_data.1; + self.newest_entry = bucket_data.newest_entry; Ok(()) } pub(super) fn save_bucket(&self) -> EyreResult> { - let mut entry_vec = Vec::new(); + let mut entries = Vec::new(); for (k, v) in &self.entries { - let entry_bytes = v.with_mut_inner(|e| serde_cbor::to_vec(e))?; - entry_vec.push((*k, entry_bytes)); + let entry_bytes = v.with_inner(|e| to_rkyv(e))?; + entries.push(BucketEntryData { + key: *k, + value: entry_bytes, + }); } - let bucket_data: BucketData = (entry_vec, self.newest_entry.clone()); - let out = serde_cbor::to_vec(&bucket_data)?; + let bucket_data = BucketData { + entries, + newest_entry: self.newest_entry.clone(), + }; + let out = to_rkyv(&bucket_data)?; Ok(out) } diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 422e030a..fdecd02e 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -64,22 +64,44 @@ pub struct BucketEntryLocalNetwork { node_status: Option, } +/// A range of cryptography versions supported by this entry +#[derive(Debug, Serialize, Deserialize)] +pub struct VersionRange { + /// The minimum cryptography version supported by this entry + min: u8, + /// The maximum cryptography version supported by this entry + max: u8, +} + +/// The data associated with each bucket entry #[derive(Debug, Serialize, Deserialize)] pub struct BucketEntryInner { - min_max_version: Option<(u8, u8)>, + /// The minimum and maximum range of cryptography versions supported by the node, + /// inclusive of the requirements of any relay the node may be using + min_max_version: Option, + /// Whether or not we have updated this peer with our node info since our network + /// and dial info has last changed, for example when our IP address changes updated_since_last_network_change: bool, + /// The last connection descriptors used to contact this node, per protocol type #[serde(skip)] last_connections: BTreeMap, + /// The node info for this entry on the publicinternet routing domain public_internet: BucketEntryPublicInternet, + /// The node info for this entry on the localnetwork routing domain local_network: BucketEntryLocalNetwork, + /// Statistics gathered for the peer peer_stats: PeerStats, + /// The accounting for the latency statistics #[serde(skip)] latency_stats_accounting: LatencyStatsAccounting, + /// The accounting for the transfer statistics #[serde(skip)] transfer_stats_accounting: TransferStatsAccounting, + /// Tracking identifier for NodeRef debugging #[cfg(feature = "tracking")] #[serde(skip)] next_track_id: usize, + /// Backtraces for NodeRef debugging #[cfg(feature = "tracking")] #[serde(skip)] node_ref_tracks: HashMap, @@ -190,12 +212,12 @@ impl BucketEntryInner { // Always allow overwriting invalid/unsigned node if current_sni.has_valid_signature() { // If the timestamp hasn't changed or is less, ignore this update - if signed_node_info.timestamp <= current_sni.timestamp { + if signed_node_info.timestamp() <= current_sni.timestamp() { // If we received a node update with the same timestamp // we can make this node live again, but only if our network has recently changed // which may make nodes that were unreachable now reachable with the same dialinfo if !self.updated_since_last_network_change - && signed_node_info.timestamp == current_sni.timestamp + && signed_node_info.timestamp() == current_sni.timestamp() { // No need to update the signednodeinfo though since the timestamp is the same // Touch the node and let it try to live again @@ -207,11 +229,22 @@ impl BucketEntryInner { } } - // Update the protocol min/max version we have - self.min_max_version = Some(( - signed_node_info.node_info.min_version, - signed_node_info.node_info.max_version, - )); + // Update the protocol min/max version we have to use, to include relay requirements if needed + let mut version_range = VersionRange { + min: signed_node_info.node_info().min_version, + max: signed_node_info.node_info().max_version, + }; + if let Some(relay_info) = signed_node_info.relay_info() { + version_range.min.max_assign(relay_info.min_version); + version_range.max.min_assign(relay_info.max_version); + } + if version_range.min <= version_range.max { + // Can be reached with at least one crypto version + self.min_max_version = Some(version_range); + } else { + // No valid crypto version in range + self.min_max_version = None; + } // Update the signed node info *opt_current_sni = Some(Box::new(signed_node_info)); @@ -238,7 +271,7 @@ impl BucketEntryInner { RoutingDomain::LocalNetwork => &self.local_network.signed_node_info, RoutingDomain::PublicInternet => &self.public_internet.signed_node_info, }; - opt_current_sni.as_ref().map(|s| &s.node_info) + opt_current_sni.as_ref().map(|s| s.node_info()) } pub fn signed_node_info(&self, routing_domain: RoutingDomain) -> Option<&SignedNodeInfo> { @@ -338,11 +371,11 @@ impl BucketEntryInner { out } - pub fn set_min_max_version(&mut self, min_max_version: (u8, u8)) { + pub fn set_min_max_version(&mut self, min_max_version: VersionRange) { self.min_max_version = Some(min_max_version); } - pub fn min_max_version(&self) -> Option<(u8, u8)> { + pub fn min_max_version(&self) -> Option { self.min_max_version } diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 3ab5d8d5..fd1db5e2 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -163,7 +163,7 @@ impl RoutingTable { // Load bucket entries from table db if possible debug!("loading routing table entries"); if let Err(e) = self.load_buckets().await { - log_rtab!(warn "Error loading buckets from storage: {}. Resetting.", e); + log_rtab!(debug "Error loading buckets from storage: {:#?}. Resetting.", e); let mut inner = self.inner.write(); inner.init_buckets(self.clone()); } @@ -173,7 +173,7 @@ impl RoutingTable { let route_spec_store = match RouteSpecStore::load(self.clone()).await { Ok(v) => v, Err(e) => { - log_rtab!(warn "Error loading route spec store: {}. Resetting.", e); + log_rtab!(debug "Error loading route spec store: {:#?}. Resetting.", e); RouteSpecStore::new(self.clone()) } }; @@ -239,7 +239,7 @@ impl RoutingTable { let tdb = table_store.open("routing_table", 1).await?; let bucket_count = bucketvec.len(); let mut dbx = tdb.transact(); - if let Err(e) = dbx.store_cbor(0, b"bucket_count", &bucket_count) { + if let Err(e) = dbx.store_frozen(0, b"bucket_count", &bucket_count) { dbx.rollback(); return Err(e); } @@ -253,14 +253,13 @@ impl RoutingTable { async fn load_buckets(&self) -> EyreResult<()> { // Deserialize all entries - let inner = &mut *self.inner.write(); - let tstore = self.network_manager().table_store(); let tdb = tstore.open("routing_table", 1).await?; - let Some(bucket_count): Option = tdb.load_cbor(0, b"bucket_count")? else { + let Some(bucket_count): Option = tdb.load_rkyv(0, b"bucket_count")? else { log_rtab!(debug "no bucket count in saved routing table"); return Ok(()); }; + let inner = &mut *self.inner.write(); if bucket_count != inner.buckets.len() { // Must have the same number of buckets warn!("bucket count is different, not loading routing table"); @@ -275,8 +274,8 @@ impl RoutingTable { }; bucketdata_vec.push(bucketdata); } - for n in 0..bucket_count { - inner.buckets[n].load_bucket(&bucketdata_vec[n])?; + for (n, bucketdata) in bucketdata_vec.into_iter().enumerate() { + inner.buckets[n].load_bucket(bucketdata)?; } Ok(()) @@ -383,7 +382,7 @@ impl RoutingTable { } /// Return a copy of our node's signednodeinfo - pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedNodeInfo { + pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedDirectNodeInfo { self.inner.read().get_own_signed_node_info(routing_domain) } @@ -526,7 +525,7 @@ impl RoutingTable { &self, routing_domain: RoutingDomain, node_id: DHTKey, - signed_node_info: SignedNodeInfo, + signed_node_info: SignedDirectNodeInfo, allow_invalid: bool, ) -> Option { self.inner.write().register_node_with_signed_node_info( diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 4b660037..6f7f9adf 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -385,9 +385,12 @@ impl NodeRef { out } - pub fn locked<'a>(&self, rti: &'a mut RoutingTableInner) -> NodeRefLocked<'a> { + pub fn locked<'a>(&self, rti: &'a RoutingTableInner) -> NodeRefLocked<'a> { NodeRefLocked::new(rti, self.clone()) } + pub fn locked_mut<'a>(&self, rti: &'a mut RoutingTableInner) -> NodeRefLockedMut<'a> { + NodeRefLockedMut::new(rti, self.clone()) + } } impl NodeRefBase for NodeRef { @@ -480,12 +483,12 @@ impl Drop for NodeRef { /// already locked a RoutingTableInner /// Keeps entry in the routing table until all references are gone pub struct NodeRefLocked<'a> { - inner: Mutex<&'a mut RoutingTableInner>, + inner: Mutex<&'a RoutingTableInner>, nr: NodeRef, } impl<'a> NodeRefLocked<'a> { - pub fn new(inner: &'a mut RoutingTableInner, nr: NodeRef) -> Self { + pub fn new(inner: &'a RoutingTableInner, nr: NodeRef) -> Self { Self { inner: Mutex::new(inner), nr, @@ -510,6 +513,65 @@ impl<'a> NodeRefBase for NodeRefLocked<'a> { self.nr.common.entry.with(inner, f) } + fn operate_mut(&self, _f: F) -> T + where + F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, + { + panic!("need to locked_mut() for this operation") + } +} + +impl<'a> fmt::Display for NodeRefLocked<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.nr) + } +} + +impl<'a> fmt::Debug for NodeRefLocked<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NodeRefLocked") + .field("nr", &self.nr) + .finish() + } +} + +//////////////////////////////////////////////////////////////////////////////////// + +/// Mutable locked reference to a routing table entry +/// For internal use inside the RoutingTable module where you have +/// already locked a RoutingTableInner +/// Keeps entry in the routing table until all references are gone +pub struct NodeRefLockedMut<'a> { + inner: Mutex<&'a mut RoutingTableInner>, + nr: NodeRef, +} + +impl<'a> NodeRefLockedMut<'a> { + pub fn new(inner: &'a mut RoutingTableInner, nr: NodeRef) -> Self { + Self { + inner: Mutex::new(inner), + nr, + } + } +} + +impl<'a> NodeRefBase for NodeRefLockedMut<'a> { + fn common(&self) -> &NodeRefBaseCommon { + &self.nr.common + } + + fn common_mut(&mut self) -> &mut NodeRefBaseCommon { + &mut self.nr.common + } + + fn operate(&self, f: F) -> T + where + F: FnOnce(&RoutingTableInner, &BucketEntryInner) -> T, + { + let inner = &*self.inner.lock(); + self.nr.common.entry.with(inner, f) + } + fn operate_mut(&self, f: F) -> T where F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner) -> T, @@ -519,15 +581,15 @@ impl<'a> NodeRefBase for NodeRefLocked<'a> { } } -impl<'a> fmt::Display for NodeRefLocked<'a> { +impl<'a> fmt::Display for NodeRefLockedMut<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.nr) } } -impl<'a> fmt::Debug for NodeRefLocked<'a> { +impl<'a> fmt::Debug for NodeRefLockedMut<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("NodeRefLocked") + f.debug_struct("NodeRefLockedMut") .field("nr", &self.nr) .finish() } diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 75d4f586..b10d2b76 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -221,11 +221,11 @@ impl RouteSpecStore { ) }; - // Get cbor blob from table store + // Get frozen blob from table store let table_store = routing_table.network_manager().table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; let mut content: RouteSpecStoreContent = - rsstdb.load_cbor(0, b"content")?.unwrap_or_default(); + rsstdb.load_json(0, b"content")?.unwrap_or_default(); // Look up all route hop noderefs since we can't serialize those let mut dead_keys = Vec::new(); @@ -246,7 +246,7 @@ impl RouteSpecStore { // Load secrets from pstore let pstore = routing_table.network_manager().protected_store(); let out: Vec<(DHTKey, DHTKeySecret)> = pstore - .load_user_secret_cbor("RouteSpecStore") + .load_user_secret_rkyv("RouteSpecStore") .await? .unwrap_or_default(); @@ -289,14 +289,14 @@ impl RouteSpecStore { inner.content.clone() }; - // Save all the fields we care about to the cbor blob in table storage + // Save all the fields we care about to the frozen blob in table storage let table_store = self .unlocked_inner .routing_table .network_manager() .table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; - rsstdb.store_cbor(0, b"content", &content)?; + rsstdb.store_json(0, b"content", &content)?; // // Keep secrets in protected store as well let pstore = self @@ -310,7 +310,9 @@ impl RouteSpecStore { out.push((*k, v.secret_key)); } - let _ = pstore.save_user_secret_cbor("RouteSpecStore", &out).await?; // ignore if this previously existed or not + let _ = pstore + .save_user_secret_frozen("RouteSpecStore", &out) + .await?; // ignore if this previously existed or not Ok(()) } diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 43bd7d44..54aeec46 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -101,6 +101,45 @@ impl RoutingDomainDetailCommon { self.network_class.unwrap_or(NetworkClass::Invalid) != NetworkClass::Invalid } + fn make_peer_info(&self, rti: &RoutingTableInner) -> PeerInfo { + let node_info = NodeInfo { + network_class: self.network_class.unwrap_or(NetworkClass::Invalid), + outbound_protocols: self.outbound_protocols, + address_types: self.address_types, + min_version: MIN_CRYPTO_VERSION, + max_version: MAX_CRYPTO_VERSION, + dial_info_detail_list: self.dial_info_details.clone(), + }; + + let relay_peer_info = self + .relay_node + .as_ref() + .and_then(|rn| rn.locked(rti).make_peer_info(self.routing_domain)); + + let signed_node_info = match relay_peer_info { + Some(relay_pi) => SignedNodeInfo::Relayed( + SignedRelayedNodeInfo::with_secret( + NodeId::new(rti.unlocked_inner.node_id), + node_info, + relay_pi.node_id, + relay_pi.signed_node_info, + &rti.unlocked_inner.node_id_secret, + ) + .unwrap(), + ), + None => SignedNodeInfo::Direct( + SignedDirectNodeInfo::with_secret( + NodeId::new(rti.unlocked_inner.node_id), + node_info, + &rti.unlocked_inner.node_id_secret, + ) + .unwrap(), + ), + }; + + PeerInfo::new(NodeId::new(rti.unlocked_inner.node_id), signed_node_info) + } + pub fn with_peer_info(&self, rti: &RoutingTableInner, f: F) -> R where F: FnOnce(&PeerInfo) -> R, @@ -110,7 +149,7 @@ impl RoutingDomainDetailCommon { // Regenerate peer info let pi = PeerInfo::new( NodeId::new(rti.unlocked_inner.node_id), - SignedNodeInfo::with_secret( + SignedDirectNodeInfo::with_secret( NodeInfo { network_class: self.network_class.unwrap_or(NetworkClass::Invalid), outbound_protocols: self.outbound_protocols, @@ -118,10 +157,11 @@ impl RoutingDomainDetailCommon { min_version: MIN_CRYPTO_VERSION, max_version: MAX_CRYPTO_VERSION, dial_info_detail_list: self.dial_info_details.clone(), - relay_peer_info: self - .relay_node - .as_ref() - .and_then(|rn| rn.make_peer_info(self.routing_domain).map(Box::new)), + relay_peer_info: self.relay_node.as_ref().and_then(|rn| { + rn.locked(rti) + .make_peer_info(self.routing_domain) + .map(Box::new) + }), }, NodeId::new(rti.unlocked_inner.node_id), &rti.unlocked_inner.node_id_secret, diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index 6435cd1e..cfd162aa 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -253,7 +253,7 @@ impl RoutingTableInner { } /// Return a copy of our node's signednodeinfo - pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedNodeInfo { + pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedDirectNodeInfo { self.with_routing_domain(routing_domain, |rdd| { rdd.common() .with_peer_info(self, |pi| pi.signed_node_info.clone()) @@ -662,7 +662,7 @@ impl RoutingTableInner { outer_self: RoutingTable, routing_domain: RoutingDomain, node_id: DHTKey, - signed_node_info: SignedNodeInfo, + signed_node_info: SignedDirectNodeInfo, allow_invalid: bool, ) -> Option { // validate signed node info is not something malicious @@ -717,7 +717,8 @@ impl RoutingTableInner { }); if let Some(nr) = &out { // set the most recent node address for connection finding and udp replies - nr.locked(self).set_last_connection(descriptor, timestamp); + nr.locked_mut(self) + .set_last_connection(descriptor, timestamp); } out } diff --git a/veilid-core/src/rpc_processor/coders/mod.rs b/veilid-core/src/rpc_processor/coders/mod.rs index a22e3dfe..3211b338 100644 --- a/veilid-core/src/rpc_processor/coders/mod.rs +++ b/veilid-core/src/rpc_processor/coders/mod.rs @@ -17,7 +17,9 @@ mod public_key; mod sender_info; mod signal_info; mod signature; +mod signed_direct_node_info; mod signed_node_info; +mod signed_relayed_node_info; mod socket_address; mod tunnel; mod value_data; @@ -42,7 +44,9 @@ pub use public_key::*; pub use sender_info::*; pub use signal_info::*; pub use signature::*; +pub use signed_direct_node_info::*; pub use signed_node_info::*; +pub use signed_relayed_node_info::*; pub use socket_address::*; pub use tunnel::*; pub use value_data::*; diff --git a/veilid-core/src/rpc_processor/coders/network_class.rs b/veilid-core/src/rpc_processor/coders/network_class.rs index eaea3d78..88d17fce 100644 --- a/veilid-core/src/rpc_processor/coders/network_class.rs +++ b/veilid-core/src/rpc_processor/coders/network_class.rs @@ -5,7 +5,7 @@ pub fn encode_network_class(network_class: NetworkClass) -> veilid_capnp::Networ NetworkClass::InboundCapable => veilid_capnp::NetworkClass::InboundCapable, NetworkClass::OutboundOnly => veilid_capnp::NetworkClass::OutboundOnly, NetworkClass::WebApp => veilid_capnp::NetworkClass::WebApp, - NetworkClass::Invalid => panic!("invalid network class should not be encoded"), + NetworkClass::Invalid => veilid_capnp::NetworkClass::Invalid, } } @@ -14,5 +14,6 @@ pub fn decode_network_class(network_class: veilid_capnp::NetworkClass) -> Networ veilid_capnp::NetworkClass::InboundCapable => NetworkClass::InboundCapable, veilid_capnp::NetworkClass::OutboundOnly => NetworkClass::OutboundOnly, veilid_capnp::NetworkClass::WebApp => NetworkClass::WebApp, + veilid_capnp::NetworkClass::Invalid => NetworkClass::Invalid, } } diff --git a/veilid-core/src/rpc_processor/coders/node_info.rs b/veilid-core/src/rpc_processor/coders/node_info.rs index 72e7f31e..02f9bd78 100644 --- a/veilid-core/src/rpc_processor/coders/node_info.rs +++ b/veilid-core/src/rpc_processor/coders/node_info.rs @@ -31,18 +31,10 @@ pub fn encode_node_info( encode_dial_info_detail(&node_info.dial_info_detail_list[idx], &mut did_builder)?; } - if let Some(rpi) = &node_info.relay_peer_info { - let mut rpi_builder = builder.reborrow().init_relay_peer_info(); - encode_peer_info(rpi, &mut rpi_builder)?; - } - Ok(()) } -pub fn decode_node_info( - reader: &veilid_capnp::node_info::Reader, - allow_relay_peer_info: bool, -) -> Result { +pub fn decode_node_info(reader: &veilid_capnp::node_info::Reader) -> Result { let network_class = decode_network_class( reader .reborrow() @@ -81,22 +73,6 @@ pub fn decode_node_info( dial_info_detail_list.push(decode_dial_info_detail(&did)?) } - let relay_peer_info = if allow_relay_peer_info { - if reader.has_relay_peer_info() { - Some(Box::new(decode_peer_info( - &reader - .reborrow() - .get_relay_peer_info() - .map_err(RPCError::protocol)?, - false, - )?)) - } else { - None - } - } else { - None - }; - Ok(NodeInfo { network_class, outbound_protocols, @@ -104,6 +80,5 @@ pub fn decode_node_info( min_version, max_version, dial_info_detail_list, - relay_peer_info, }) } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation.rs b/veilid-core/src/rpc_processor/coders/operations/operation.rs index 9d36c192..a33ab29c 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation.rs @@ -120,7 +120,7 @@ impl RPCOperation { let sni_reader = operation_reader .get_sender_node_info() .map_err(RPCError::protocol)?; - let sni = decode_signed_node_info(&sni_reader, sender_node_id, true)?; + let sni = decode_signed_node_info(&sni_reader, sender_node_id)?; Some(sni) } else { None diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs b/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs index 6e561635..71fcba11 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs @@ -47,7 +47,7 @@ impl RPCOperationFindBlockA { .map_err(RPCError::map_internal("too many suppliers"))?, ); for s in suppliers_reader.iter() { - let peer_info = decode_peer_info(&s, true)?; + let peer_info = decode_peer_info(&s)?; suppliers.push(peer_info); } @@ -59,7 +59,7 @@ impl RPCOperationFindBlockA { .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p, true)?; + let peer_info = decode_peer_info(&p)?; peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs b/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs index 852700a0..bfd0ded9 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs @@ -41,7 +41,7 @@ impl RPCOperationFindNodeA { .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p, true)?; + let peer_info = decode_peer_info(&p)?; peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs b/veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs index 9ac5f9f6..bb800511 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_get_value.rs @@ -48,7 +48,7 @@ impl RPCOperationGetValueA { .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p, true)?; + let peer_info = decode_peer_info(&p)?; peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_node_info_update.rs b/veilid-core/src/rpc_processor/coders/operations/operation_node_info_update.rs index a785e6b5..5f077816 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_node_info_update.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_node_info_update.rs @@ -18,7 +18,7 @@ impl RPCOperationNodeInfoUpdate { } let sender_node_id = opt_sender_node_id.unwrap(); let sni_reader = reader.get_signed_node_info().map_err(RPCError::protocol)?; - let signed_node_info = decode_signed_node_info(&sni_reader, sender_node_id, true)?; + let signed_node_info = decode_signed_node_info(&sni_reader, sender_node_id)?; Ok(RPCOperationNodeInfoUpdate { signed_node_info }) } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs b/veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs index af5c5350..18c430d1 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_set_value.rs @@ -53,7 +53,7 @@ impl RPCOperationSetValueA { .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p, true)?; + let peer_info = decode_peer_info(&p)?; peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs b/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs index 0e496417..b094eaaf 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs @@ -49,7 +49,7 @@ impl RPCOperationSupplyBlockA { .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p, true)?; + let peer_info = decode_peer_info(&p)?; peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_watch_value.rs b/veilid-core/src/rpc_processor/coders/operations/operation_watch_value.rs index 48d7d864..cbb08fcb 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_watch_value.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_watch_value.rs @@ -43,7 +43,7 @@ impl RPCOperationWatchValueA { .map_err(RPCError::map_internal("too many peers"))?, ); for p in peers_reader.iter() { - let peer_info = decode_peer_info(&p, true)?; + let peer_info = decode_peer_info(&p)?; peers.push(peer_info); } diff --git a/veilid-core/src/rpc_processor/coders/peer_info.rs b/veilid-core/src/rpc_processor/coders/peer_info.rs index 043fe761..428980f4 100644 --- a/veilid-core/src/rpc_processor/coders/peer_info.rs +++ b/veilid-core/src/rpc_processor/coders/peer_info.rs @@ -14,10 +14,7 @@ pub fn encode_peer_info( Ok(()) } -pub fn decode_peer_info( - reader: &veilid_capnp::peer_info::Reader, - allow_relay_peer_info: bool, -) -> Result { +pub fn decode_peer_info(reader: &veilid_capnp::peer_info::Reader) -> Result { let nid_reader = reader .reborrow() .get_node_id() @@ -27,8 +24,7 @@ pub fn decode_peer_info( .get_signed_node_info() .map_err(RPCError::protocol)?; let node_id = NodeId::new(decode_public_key(&nid_reader)); - let signed_node_info = - decode_signed_node_info(&sni_reader, &node_id.key, allow_relay_peer_info)?; + let signed_node_info = decode_signed_node_info(&sni_reader, &node_id.key)?; Ok(PeerInfo { node_id, diff --git a/veilid-core/src/rpc_processor/coders/private_safety_route.rs b/veilid-core/src/rpc_processor/coders/private_safety_route.rs index 4ac99409..31a8362b 100644 --- a/veilid-core/src/rpc_processor/coders/private_safety_route.rs +++ b/veilid-core/src/rpc_processor/coders/private_safety_route.rs @@ -77,7 +77,7 @@ pub fn decode_route_hop(reader: &veilid_capnp::route_hop::Reader) -> Result { let pi_reader = pi.map_err(RPCError::protocol)?; RouteNode::PeerInfo( - decode_peer_info(&pi_reader, true) + decode_peer_info(&pi_reader) .map_err(RPCError::map_protocol("invalid peer info in route hop"))?, ) } diff --git a/veilid-core/src/rpc_processor/coders/signal_info.rs b/veilid-core/src/rpc_processor/coders/signal_info.rs index d07ff7d4..7a272973 100644 --- a/veilid-core/src/rpc_processor/coders/signal_info.rs +++ b/veilid-core/src/rpc_processor/coders/signal_info.rs @@ -53,7 +53,7 @@ pub fn decode_signal_info( let pi_reader = r.get_peer_info().map_err(RPCError::map_protocol( "invalid peer info in hole punch signal info", ))?; - let peer_info = decode_peer_info(&pi_reader, true)?; + let peer_info = decode_peer_info(&pi_reader)?; SignalInfo::HolePunch { receipt, peer_info } } @@ -69,7 +69,7 @@ pub fn decode_signal_info( let pi_reader = r.get_peer_info().map_err(RPCError::map_protocol( "invalid peer info in reverse connect signal info", ))?; - let peer_info = decode_peer_info(&pi_reader, true)?; + let peer_info = decode_peer_info(&pi_reader)?; SignalInfo::ReverseConnect { receipt, peer_info } } diff --git a/veilid-core/src/rpc_processor/coders/signed_direct_node_info.rs b/veilid-core/src/rpc_processor/coders/signed_direct_node_info.rs new file mode 100644 index 00000000..1e9768f4 --- /dev/null +++ b/veilid-core/src/rpc_processor/coders/signed_direct_node_info.rs @@ -0,0 +1,43 @@ +use crate::*; +use rpc_processor::*; + +pub fn encode_signed_direct_node_info( + signed_direct_node_info: &SignedDirectNodeInfo, + builder: &mut veilid_capnp::signed_direct_node_info::Builder, +) -> Result<(), RPCError> { + // + let mut ni_builder = builder.reborrow().init_node_info(); + encode_node_info(&signed_direct_node_info.node_info, &mut ni_builder)?; + + builder + .reborrow() + .set_timestamp(signed_direct_node_info.timestamp); + + let mut sig_builder = builder.reborrow().init_signature(); + encode_signature(&signed_direct_node_info.signature, &mut sig_builder); + + Ok(()) +} + +pub fn decode_signed_direct_node_info( + reader: &veilid_capnp::signed_direct_node_info::Reader, + node_id: &DHTKey, +) -> Result { + let ni_reader = reader + .reborrow() + .get_node_info() + .map_err(RPCError::protocol)?; + let node_info = decode_node_info(&ni_reader)?; + + let sig_reader = reader + .reborrow() + .get_signature() + .map_err(RPCError::protocol)?; + + let timestamp = reader.reborrow().get_timestamp(); + + let signature = decode_signature(&sig_reader); + + SignedDirectNodeInfo::new(NodeId::new(*node_id), node_info, timestamp, signature) + .map_err(RPCError::protocol) +} diff --git a/veilid-core/src/rpc_processor/coders/signed_node_info.rs b/veilid-core/src/rpc_processor/coders/signed_node_info.rs index 11c9d51c..64ae9c80 100644 --- a/veilid-core/src/rpc_processor/coders/signed_node_info.rs +++ b/veilid-core/src/rpc_processor/coders/signed_node_info.rs @@ -5,14 +5,16 @@ pub fn encode_signed_node_info( signed_node_info: &SignedNodeInfo, builder: &mut veilid_capnp::signed_node_info::Builder, ) -> Result<(), RPCError> { - // - let mut ni_builder = builder.reborrow().init_node_info(); - encode_node_info(&signed_node_info.node_info, &mut ni_builder)?; - - let mut sig_builder = builder.reborrow().init_signature(); - encode_signature(&signed_node_info.signature, &mut sig_builder); - - builder.reborrow().set_timestamp(signed_node_info.timestamp); + match signed_node_info { + SignedNodeInfo::Direct(d) => { + let mut d_builder = builder.reborrow().init_direct(); + encode_signed_direct_node_info(d, &mut d_builder)?; + } + SignedNodeInfo::Relayed(r) => { + let mut r_builder = builder.reborrow().init_relayed(); + encode_signed_relayed_node_info(r, &mut r_builder)?; + } + } Ok(()) } @@ -20,22 +22,20 @@ pub fn encode_signed_node_info( pub fn decode_signed_node_info( reader: &veilid_capnp::signed_node_info::Reader, node_id: &DHTKey, - allow_relay_peer_info: bool, ) -> Result { - let ni_reader = reader - .reborrow() - .get_node_info() - .map_err(RPCError::protocol)?; - let node_info = decode_node_info(&ni_reader, allow_relay_peer_info)?; - - let sig_reader = reader - .reborrow() - .get_signature() - .map_err(RPCError::protocol)?; - let signature = decode_signature(&sig_reader); - - let timestamp = reader.reborrow().get_timestamp(); - - SignedNodeInfo::new(node_info, NodeId::new(*node_id), signature, timestamp) - .map_err(RPCError::protocol) + match reader + .which() + .map_err(RPCError::map_internal("invalid signal operation"))? + { + veilid_capnp::signed_node_info::Direct(d) => { + let d_reader = d.map_err(RPCError::protocol)?; + let sdni = decode_signed_direct_node_info(&d_reader, node_id)?; + Ok(SignedNodeInfo::Direct(sdni)) + } + veilid_capnp::signed_node_info::Relayed(r) => { + let r_reader = r.map_err(RPCError::protocol)?; + let srni = decode_signed_relayed_node_info(&r_reader, node_id)?; + Ok(SignedNodeInfo::Relayed(srni)) + } + } } diff --git a/veilid-core/src/rpc_processor/coders/signed_relayed_node_info.rs b/veilid-core/src/rpc_processor/coders/signed_relayed_node_info.rs new file mode 100644 index 00000000..530f8254 --- /dev/null +++ b/veilid-core/src/rpc_processor/coders/signed_relayed_node_info.rs @@ -0,0 +1,67 @@ +use crate::*; +use rpc_processor::*; + +pub fn encode_signed_relayed_node_info( + signed_relayed_node_info: &SignedRelayedNodeInfo, + builder: &mut veilid_capnp::signed_relayed_node_info::Builder, +) -> Result<(), RPCError> { + // + let mut ni_builder = builder.reborrow().init_node_info(); + encode_node_info(&signed_relayed_node_info.node_info, &mut ni_builder)?; + + let mut rid_builder = builder.reborrow().init_relay_id(); + encode_public_key(&signed_relayed_node_info.relay_id.key, &mut rid_builder)?; + + let mut ri_builder = builder.reborrow().init_relay_info(); + encode_signed_direct_node_info(&signed_relayed_node_info.relay_info, &mut ri_builder)?; + + builder + .reborrow() + .set_timestamp(signed_relayed_node_info.timestamp); + + let mut sig_builder = builder.reborrow().init_signature(); + encode_signature(&signed_relayed_node_info.signature, &mut sig_builder); + + Ok(()) +} + +pub fn decode_signed_relayed_node_info( + reader: &veilid_capnp::signed_relayed_node_info::Reader, + node_id: &DHTKey, +) -> Result { + let ni_reader = reader + .reborrow() + .get_node_info() + .map_err(RPCError::protocol)?; + let node_info = decode_node_info(&ni_reader)?; + + let rid_reader = reader + .reborrow() + .get_relay_id() + .map_err(RPCError::protocol)?; + let relay_id = decode_public_key(&rid_reader); + + let ri_reader = reader + .reborrow() + .get_relay_info() + .map_err(RPCError::protocol)?; + let relay_info = decode_signed_direct_node_info(&ri_reader, &relay_id)?; + + let sig_reader = reader + .reborrow() + .get_signature() + .map_err(RPCError::protocol)?; + let timestamp = reader.reborrow().get_timestamp(); + + let signature = decode_signature(&sig_reader); + + SignedRelayedNodeInfo::new( + NodeId::new(*node_id), + node_info, + NodeId::new(relay_id), + relay_info, + timestamp, + signature, + ) + .map_err(RPCError::protocol) +} diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index f2186b05..4d0a6f85 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -619,7 +619,7 @@ impl RPCProcessor { // routing table caching when it is okay to do so // This is only done in the PublicInternet routing domain because // as far as we can tell this is the only domain that will really benefit - fn get_sender_signed_node_info(&self, dest: &Destination) -> Option { + fn get_sender_signed_node_info(&self, dest: &Destination) -> Option { // Don't do this if the sender is to remain private // Otherwise we would be attaching the original sender's identity to the final destination, // thus defeating the purpose of the safety route entirely :P @@ -682,7 +682,7 @@ impl RPCProcessor { let op_id = operation.op_id(); // Log rpc send - debug!(target: "rpc_message", dir = "send", kind = "question", op_id, desc = operation.kind().desc(), ?dest); + trace!(target: "rpc_message", dir = "send", kind = "question", op_id, desc = operation.kind().desc(), ?dest); // Produce rendered operation let RenderedOperation { @@ -745,7 +745,7 @@ impl RPCProcessor { let operation = RPCOperation::new_statement(statement, opt_sender_info); // Log rpc send - debug!(target: "rpc_message", dir = "send", kind = "statement", op_id = operation.op_id(), desc = operation.kind().desc(), ?dest); + trace!(target: "rpc_message", dir = "send", kind = "statement", op_id = operation.op_id(), desc = operation.kind().desc(), ?dest); // Produce rendered operation let RenderedOperation { @@ -865,7 +865,7 @@ impl RPCProcessor { let operation = RPCOperation::new_answer(&request.operation, answer, opt_sender_info); // Log rpc send - debug!(target: "rpc_message", dir = "send", kind = "answer", op_id = operation.op_id(), desc = operation.kind().desc(), ?dest); + trace!(target: "rpc_message", dir = "send", kind = "answer", op_id = operation.op_id(), desc = operation.kind().desc(), ?dest); // Produce rendered operation let RenderedOperation { @@ -997,7 +997,7 @@ impl RPCProcessor { }; // Log rpc receive - debug!(target: "rpc_message", dir = "recv", kind, op_id = msg.operation.op_id(), desc = msg.operation.kind().desc(), header = ?msg.header); + trace!(target: "rpc_message", dir = "recv", kind, op_id = msg.operation.op_id(), desc = msg.operation.kind().desc(), header = ?msg.header); // Process specific message kind match msg.operation.kind() { diff --git a/veilid-core/src/tests/common/test_table_store.rs b/veilid-core/src/tests/common/test_table_store.rs index f10f322e..5e3478a3 100644 --- a/veilid-core/src/tests/common/test_table_store.rs +++ b/veilid-core/src/tests/common/test_table_store.rs @@ -122,21 +122,21 @@ pub async fn test_store_delete_load(ts: TableStore) { assert_eq!(db.load(2, b"baz").unwrap(), Some(b"QWERTY".to_vec())); } -pub async fn test_cbor(ts: TableStore) { - trace!("test_cbor"); +pub async fn test_frozen(ts: TableStore) { + trace!("test_frozen"); let _ = ts.delete("test"); let db = ts.open("test", 3).await.expect("should have opened"); let (dht_key, _) = generate_secret(); - assert!(db.store_cbor(0, b"asdf", &dht_key).is_ok()); + assert!(db.store_rkyv(0, b"asdf", &dht_key).is_ok()); - assert_eq!(db.load_cbor::(0, b"qwer").unwrap(), None); + assert_eq!(db.load_rkyv::(0, b"qwer").unwrap(), None); - let d = match db.load_cbor::(0, b"asdf") { + let d = match db.load_rkyv::(0, b"asdf") { Ok(x) => x, Err(e) => { - panic!("couldn't decode cbor: {}", e); + panic!("couldn't decode: {}", e); } }; assert_eq!(d, Some(dht_key), "keys should be equal"); @@ -147,8 +147,8 @@ pub async fn test_cbor(ts: TableStore) { ); assert!( - db.load_cbor::(1, b"foo").is_err(), - "should fail to load cbor" + db.load_rkyv::(1, b"foo").is_err(), + "should fail to unfreeze" ); } @@ -157,7 +157,7 @@ pub async fn test_all() { let ts = api.table_store().unwrap(); test_delete_open_delete(ts.clone()).await; test_store_delete_load(ts.clone()).await; - test_cbor(ts.clone()).await; + test_frozen(ts.clone()).await; let _ = ts.delete("test").await; diff --git a/veilid-core/src/tests/common/test_veilid_core.rs b/veilid-core/src/tests/common/test_veilid_core.rs index e84828a0..bf8d6915 100644 --- a/veilid-core/src/tests/common/test_veilid_core.rs +++ b/veilid-core/src/tests/common/test_veilid_core.rs @@ -43,7 +43,78 @@ pub async fn test_attach_detach() { api.shutdown().await; } +pub async fn test_signed_node_info() { + info!("--- test_signed_node_info ---"); + + let (update_callback, config_callback) = setup_veilid_core(); + let api = api_startup(update_callback, config_callback) + .await + .expect("startup failed"); + + // Test direct + let node_info = NodeInfo { + network_class: NetworkClass::InboundCapable, + outbound_protocols: ProtocolTypeSet::all(), + address_types: AddressTypeSet::all(), + min_version: 0, + max_version: 0, + dial_info_detail_list: vec![DialInfoDetail { + class: DialInfoClass::Mapped, + dial_info: DialInfo::udp(SocketAddress::default()), + }], + }; + + let (pkey, skey) = generate_secret(); + + let sni = + SignedDirectNodeInfo::with_secret(NodeId::new(pkey.clone()), node_info.clone(), &skey) + .unwrap(); + let _ = SignedDirectNodeInfo::new( + NodeId::new(pkey), + node_info.clone(), + sni.timestamp, + sni.signature, + ) + .unwrap(); + + // Test relayed + let node_info2 = NodeInfo { + network_class: NetworkClass::OutboundOnly, + outbound_protocols: ProtocolTypeSet::all(), + address_types: AddressTypeSet::all(), + min_version: 0, + max_version: 0, + dial_info_detail_list: vec![DialInfoDetail { + class: DialInfoClass::Blocked, + dial_info: DialInfo::udp(SocketAddress::default()), + }], + }; + + let (pkey2, skey2) = generate_secret(); + + let sni2 = SignedRelayedNodeInfo::with_secret( + NodeId::new(pkey2.clone()), + node_info2.clone(), + NodeId::new(pkey.clone()), + sni.clone(), + &skey2, + ) + .unwrap(); + let _ = SignedRelayedNodeInfo::new( + NodeId::new(pkey2), + node_info2, + NodeId::new(pkey), + sni, + sni2.timestamp, + sni2.signature, + ) + .unwrap(); + + api.shutdown().await; +} + pub async fn test_all() { test_startup_shutdown().await; test_attach_detach().await; + test_signed_node_info().await; } diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 9bc78d7d..9405db24 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -4,7 +4,6 @@ use super::*; use data_encoding::BASE64URL_NOPAD; use routing_table::*; -use rpc_processor::*; #[derive(Default, Debug)] struct DebugCache { diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index b28bc49b..9f522aba 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -30,7 +30,7 @@ pub use routing_table::{NodeRef, NodeRefBase, RoutingTable}; use core::fmt; use core_context::{api_shutdown, VeilidCoreContext}; use enumset::*; -use rpc_processor::RPCProcessor; +use rpc_processor::*; use serde::*; use xx::*; @@ -60,7 +60,21 @@ macro_rules! apibail_parse { }; } -#[derive(ThisError, Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +#[derive( + ThisError, + Clone, + Debug, + PartialOrd, + PartialEq, + Eq, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] #[serde(tag = "kind")] pub enum VeilidAPIError { #[error("Not initialized")] @@ -159,7 +173,21 @@ impl VeilidAPIError { ///////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Copy, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Copy, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum VeilidLogLevel { Error = 1, Warn, @@ -220,14 +248,20 @@ impl fmt::Display for VeilidLogLevel { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidLog { pub log_level: VeilidLogLevel, pub message: String, pub backtrace: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidAppMessage { /// Some(sender) if the message was sent directly, None if received via a private/safety route #[serde(with = "opt_json_as_string")] @@ -237,7 +271,10 @@ pub struct VeilidAppMessage { pub message: Vec, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidAppCall { /// Some(sender) if the request was sent directly, None if received via a private/safety route #[serde(with = "opt_json_as_string")] @@ -250,19 +287,28 @@ pub struct VeilidAppCall { pub id: u64, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidStateAttachment { pub state: AttachmentState, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct PeerTableData { pub node_id: DHTKey, pub peer_address: PeerAddress, pub peer_stats: PeerStats, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidStateNetwork { pub started: bool, #[serde(with = "json_as_string")] @@ -272,7 +318,8 @@ pub struct VeilidStateNetwork { pub peers: Vec, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(u8), derive(CheckBytes))] #[serde(tag = "kind")] pub enum VeilidUpdate { Log(VeilidLog), @@ -283,7 +330,8 @@ pub enum VeilidUpdate { Shutdown, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidState { pub attachment: VeilidStateAttachment, pub network: VeilidStateNetwork, @@ -291,7 +339,21 @@ pub struct VeilidState { ///////////////////////////////////////////////////////////////////////////////////////////////////// /// -#[derive(Clone, Debug, Default, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +#[derive( + Clone, + Debug, + Default, + PartialOrd, + PartialEq, + Eq, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct NodeId { pub key: DHTKey, } @@ -315,7 +377,21 @@ impl FromStr for NodeId { } } -#[derive(Clone, Debug, Default, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +#[derive( + Clone, + Debug, + Default, + PartialOrd, + PartialEq, + Eq, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct ValueKey { pub key: DHTKey, pub subkey: Option, @@ -336,7 +412,21 @@ impl ValueKey { } } -#[derive(Clone, Debug, Default, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +#[derive( + Clone, + Debug, + Default, + PartialOrd, + PartialEq, + Eq, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct ValueData { pub data: Vec, pub seq: u32, @@ -354,7 +444,21 @@ impl ValueData { } } -#[derive(Clone, Debug, Default, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +#[derive( + Clone, + Debug, + Default, + PartialOrd, + PartialEq, + Eq, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct BlockId { pub key: DHTKey, } @@ -367,7 +471,22 @@ impl BlockId { ///////////////////////////////////////////////////////////////////////////////////////////////////// // Keep member order appropriate for sorting < preference -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)] +#[derive( + Copy, + Clone, + Debug, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum DialInfoClass { Direct = 0, // D = Directly reachable with public IP and no firewall, with statically configured port Mapped = 1, // M = Directly reachable with via portmap behind any NAT or firewalled with dynamically negotiated port @@ -401,7 +520,22 @@ impl DialInfoClass { } // Ordering here matters, >= is used to check strength of sequencing requirement -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Copy, + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum Sequencing { NoPreference, PreferOrdered, @@ -409,14 +543,44 @@ pub enum Sequencing { } // Ordering here matters, >= is used to check strength of stability requirement -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Copy, + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum Stability { LowLatency, Reliable, } /// The choice of safety route to include in compiled routes -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Copy, + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum SafetySelection { /// Don't use a safety route, only specify the sequencing preference Unsafe(Sequencing), @@ -425,7 +589,22 @@ pub enum SafetySelection { } /// Options for safety routes (sender privacy) -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Copy, + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct SafetySpec { /// preferred safety route if it still exists pub preferred_route: Option, @@ -438,7 +617,21 @@ pub struct SafetySpec { } // Keep member order appropriate for sorting < preference -#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize, Hash)] +#[derive( + Debug, + Clone, + PartialEq, + PartialOrd, + Ord, + Eq, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct DialInfoDetail { pub class: DialInfoClass, pub dial_info: DialInfo, @@ -468,7 +661,22 @@ impl DialInfoDetail { > = None:: core::cmp::Ordering>; } -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)] +#[derive( + Copy, + Clone, + Debug, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum NetworkClass { InboundCapable = 0, // I = Inbound capable without relay, may require signal OutboundOnly = 1, // O = Outbound only, inbound relay required except with reverse connect signal @@ -493,7 +701,10 @@ impl NetworkClass { /// is returned by the StatusA call /// PublicInternet RoutingDomain Status -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive( + Clone, Debug, Default, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct PublicInternetNodeStatus { pub will_route: bool, pub will_tunnel: bool, @@ -502,13 +713,17 @@ pub struct PublicInternetNodeStatus { pub will_validate_dial_info: bool, } -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive( + Clone, Debug, Default, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct LocalNetworkNodeStatus { pub will_relay: bool, pub will_validate_dial_info: bool, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum NodeStatus { PublicInternet(PublicInternetNodeStatus), LocalNetwork(LocalNetworkNodeStatus), @@ -547,181 +762,57 @@ impl NodeStatus { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct NodeInfo { pub network_class: NetworkClass, + #[with(RkyvEnumSet)] pub outbound_protocols: ProtocolTypeSet, + #[with(RkyvEnumSet)] pub address_types: AddressTypeSet, pub min_version: u8, pub max_version: u8, pub dial_info_detail_list: Vec, - pub relay_peer_info: Option>, -} - -impl NodeInfo { - pub fn first_filtered_dial_info_detail( - &self, - sort: Option, - filter: F, - ) -> Option - where - S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, - F: Fn(&DialInfoDetail) -> bool, - { - if let Some(sort) = sort { - let mut dids = self.dial_info_detail_list.clone(); - dids.sort_by(sort); - for did in dids { - if filter(&did) { - return Some(did); - } - } - } else { - for did in &self.dial_info_detail_list { - if filter(did) { - return Some(did.clone()); - } - } - }; - None - } - - pub fn all_filtered_dial_info_details( - &self, - sort: Option, - filter: F, - ) -> Vec - where - S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, - F: Fn(&DialInfoDetail) -> bool, - { - let mut dial_info_detail_list = Vec::new(); - - if let Some(sort) = sort { - let mut dids = self.dial_info_detail_list.clone(); - dids.sort_by(sort); - for did in dids { - if filter(&did) { - dial_info_detail_list.push(did); - } - } - } else { - for did in &self.dial_info_detail_list { - if filter(did) { - dial_info_detail_list.push(did.clone()); - } - } - }; - dial_info_detail_list - } - - pub fn has_any_dial_info(&self) -> bool { - !self.dial_info_detail_list.is_empty() - || !self - .relay_peer_info - .as_ref() - .map(|rpi| rpi.signed_node_info.node_info.has_direct_dial_info()) - .unwrap_or_default() - } - - pub fn has_sequencing_matched_dial_info(&self, sequencing: Sequencing) -> bool { - // Check our dial info - for did in &self.dial_info_detail_list { - match sequencing { - Sequencing::NoPreference | Sequencing::PreferOrdered => return true, - Sequencing::EnsureOrdered => { - if did.dial_info.protocol_type().is_connection_oriented() { - return true; - } - } - } - } - // Check our relay if we have one - return self - .relay_peer_info - .as_ref() - .map(|rpi| { - let relay_ni = &rpi.signed_node_info.node_info; - for did in &relay_ni.dial_info_detail_list { - match sequencing { - Sequencing::NoPreference | Sequencing::PreferOrdered => return true, - Sequencing::EnsureOrdered => { - if did.dial_info.protocol_type().is_connection_oriented() { - return true; - } - } - } - } - false - }) - .unwrap_or_default(); - } - - pub fn has_direct_dial_info(&self) -> bool { - !self.dial_info_detail_list.is_empty() - } - - // Is some relay required either for signal or inbound relay or outbound relay? - pub fn requires_relay(&self) -> bool { - match self.network_class { - NetworkClass::InboundCapable => { - for did in &self.dial_info_detail_list { - if did.class.requires_relay() { - return true; - } - } - } - NetworkClass::OutboundOnly => { - return true; - } - NetworkClass::WebApp => { - return true; - } - NetworkClass::Invalid => {} - } - false - } - - // Can this node assist with signalling? Yes but only if it doesn't require signalling, itself. - pub fn can_signal(&self) -> bool { - // Must be inbound capable - if !matches!(self.network_class, NetworkClass::InboundCapable) { - return false; - } - // Do any of our dial info require signalling? if so, we can't offer signalling - for did in &self.dial_info_detail_list { - if did.class.requires_signal() { - return false; - } - } - true - } - - // Can this node relay be an inbound relay? - pub fn can_inbound_relay(&self) -> bool { - // For now this is the same - self.can_signal() - } - - // Is this node capable of validating dial info - pub fn can_validate_dial_info(&self) -> bool { - // For now this is the same - self.can_signal() - } } #[allow(clippy::derive_hash_xor_eq)] -#[derive(Debug, PartialOrd, Ord, Hash, Serialize, Deserialize, EnumSetType)] +#[derive( + Debug, + PartialOrd, + Ord, + Hash, + EnumSetType, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[enumset(repr = "u8")] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum Direction { Inbound, Outbound, } pub type DirectionSet = EnumSet; -#[allow(clippy::derive_hash_xor_eq)] -#[derive(Debug, PartialOrd, Ord, Hash, Serialize, Deserialize, EnumSetType)] // Keep member order appropriate for sorting < preference // Must match DialInfo order +#[allow(clippy::derive_hash_xor_eq)] +#[derive( + Debug, + PartialOrd, + Ord, + Hash, + EnumSetType, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[enumset(repr = "u8")] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum LowLevelProtocolType { UDP, TCP, @@ -734,10 +825,23 @@ impl LowLevelProtocolType { } pub type LowLevelProtocolTypeSet = EnumSet; -#[allow(clippy::derive_hash_xor_eq)] -#[derive(Debug, PartialOrd, Ord, Hash, Serialize, Deserialize, EnumSetType)] // Keep member order appropriate for sorting < preference // Must match DialInfo order +#[allow(clippy::derive_hash_xor_eq)] +#[derive( + Debug, + PartialOrd, + Ord, + Hash, + EnumSetType, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[enumset(repr = "u8")] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum ProtocolType { UDP, TCP, @@ -794,10 +898,24 @@ impl ProtocolType { ProtocolType::TCP | ProtocolType::WS | ProtocolType::WSS } } + pub type ProtocolTypeSet = EnumSet; #[allow(clippy::derive_hash_xor_eq)] -#[derive(Debug, PartialOrd, Ord, Hash, Serialize, Deserialize, EnumSetType)] +#[derive( + Debug, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + EnumSetType, +)] +#[enumset(repr = "u8")] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum AddressType { IPV4, IPV6, @@ -806,7 +924,20 @@ pub type AddressTypeSet = EnumSet; // Routing domain here is listed in order of preference, keep in order #[allow(clippy::derive_hash_xor_eq)] -#[derive(Debug, Ord, PartialOrd, Hash, Serialize, Deserialize, EnumSetType)] +#[derive( + Debug, + Ord, + PartialOrd, + Hash, + EnumSetType, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[enumset(repr = "u8")] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum RoutingDomain { LocalNetwork = 0, PublicInternet = 1, @@ -822,7 +953,22 @@ impl RoutingDomain { } pub type RoutingDomainSet = EnumSet; -#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)] +#[derive( + Copy, + Clone, + Debug, + PartialEq, + PartialOrd, + Ord, + Eq, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum Address { IPV4(Ipv4Addr), IPV6(Ipv6Addr), @@ -937,8 +1083,22 @@ impl FromStr for Address { } #[derive( - Copy, Default, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize, + Copy, + Default, + Clone, + Debug, + PartialEq, + PartialOrd, + Ord, + Eq, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, )] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct SocketAddress { address: Address, port: u16, @@ -994,9 +1154,24 @@ impl FromStr for SocketAddress { ////////////////////////////////////////////////////////////////// -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive( + Copy, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct DialInfoFilter { + #[with(RkyvEnumSet)] pub protocol_type_set: ProtocolTypeSet, + #[with(RkyvEnumSet)] pub address_type_set: AddressTypeSet, } @@ -1063,32 +1238,106 @@ pub trait MatchesDialInfoFilter { fn matches_filter(&self, filter: &DialInfoFilter) -> bool; } -#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize, Hash)] +#[derive( + Clone, + Default, + Debug, + PartialEq, + PartialOrd, + Ord, + Eq, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct DialInfoUDP { pub socket_address: SocketAddress, } -#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize, Hash)] +#[derive( + Clone, + Default, + Debug, + PartialEq, + PartialOrd, + Ord, + Eq, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct DialInfoTCP { pub socket_address: SocketAddress, } -#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize, Hash)] +#[derive( + Clone, + Default, + Debug, + PartialEq, + PartialOrd, + Ord, + Eq, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct DialInfoWS { pub socket_address: SocketAddress, pub request: String, } -#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize, Hash)] +#[derive( + Clone, + Default, + Debug, + PartialEq, + PartialOrd, + Ord, + Eq, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct DialInfoWSS { pub socket_address: SocketAddress, pub request: String, } -#[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize, Hash)] -#[serde(tag = "kind")] // Keep member order appropriate for sorting < preference // Must match ProtocolType order +#[derive( + Clone, + Debug, + PartialEq, + PartialOrd, + Ord, + Eq, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] +#[serde(tag = "kind")] pub enum DialInfo { UDP(DialInfoUDP), TCP(DialInfoTCP), @@ -1601,57 +1850,63 @@ impl MatchesDialInfoFilter for DialInfo { ////////////////////////////////////////////////////////////////////////// // Signed NodeInfo that can be passed around amongst peers and verifiable -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SignedNodeInfo { +#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] +pub struct SignedDirectNodeInfo { pub node_info: NodeInfo, - pub signature: DHTSignature, pub timestamp: u64, + pub signature: DHTSignature, } -impl SignedNodeInfo { +impl SignedDirectNodeInfo { pub fn new( - node_info: NodeInfo, node_id: NodeId, - signature: DHTSignature, + node_info: NodeInfo, timestamp: u64, + signature: DHTSignature, ) -> Result { - let mut node_info_bytes = serde_cbor::to_vec(&node_info) - .map_err(|e| VeilidAPIError::parse_error("failed to encode node info as cbor", e))?; - let mut timestamp_bytes = serde_cbor::to_vec(×tamp) - .map_err(|e| VeilidAPIError::parse_error("failed to encode timestamp as cbor", e))?; - - node_info_bytes.append(&mut timestamp_bytes); - + let node_info_bytes = Self::make_signature_bytes(&node_info, timestamp)?; verify(&node_id.key, &node_info_bytes, &signature)?; Ok(Self { node_info, - signature, timestamp, + signature, }) } pub fn with_secret( - node_info: NodeInfo, node_id: NodeId, + node_info: NodeInfo, secret: &DHTKeySecret, ) -> Result { let timestamp = intf::get_timestamp(); - - let mut node_info_bytes = serde_cbor::to_vec(&node_info) - .map_err(|e| VeilidAPIError::parse_error("failed to encode node info as cbor", e))?; - let mut timestamp_bytes = serde_cbor::to_vec(×tamp) - .map_err(|e| VeilidAPIError::parse_error("failed to encode timestamp as cbor", e))?; - - node_info_bytes.append(&mut timestamp_bytes); - + let node_info_bytes = Self::make_signature_bytes(&node_info, timestamp)?; let signature = sign(&node_id.key, secret, &node_info_bytes)?; Ok(Self { node_info, - signature, timestamp, + signature, }) } + fn make_signature_bytes( + node_info: &NodeInfo, + timestamp: u64, + ) -> Result, VeilidAPIError> { + let mut node_info_bytes = Vec::new(); + + // Add nodeinfo to signature + let mut ni_msg = ::capnp::message::Builder::new_default(); + let mut ni_builder = ni_msg.init_root::(); + encode_node_info(node_info, &mut ni_builder).map_err(VeilidAPIError::internal)?; + node_info_bytes.append(&mut builder_to_vec(ni_msg).map_err(VeilidAPIError::internal)?); + + // Add timestamp to signature + node_info_bytes.append(&mut timestamp.to_le_bytes().to_vec()); + + Ok(node_info_bytes) + } + pub fn with_no_signature(node_info: NodeInfo) -> Self { Self { node_info, @@ -1665,7 +1920,136 @@ impl SignedNodeInfo { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +/// Signed NodeInfo with a relay that can be passed around amongst peers and verifiable +#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] +pub struct SignedRelayedNodeInfo { + pub node_info: NodeInfo, + pub relay_id: NodeId, + pub relay_info: SignedDirectNodeInfo, + pub timestamp: u64, + pub signature: DHTSignature, +} + +impl SignedRelayedNodeInfo { + pub fn new( + node_id: NodeId, + node_info: NodeInfo, + relay_id: NodeId, + relay_info: SignedDirectNodeInfo, + timestamp: u64, + signature: DHTSignature, + ) -> Result { + let node_info_bytes = + Self::make_signature_bytes(&node_info, &relay_id, &relay_info, timestamp)?; + verify(&node_id.key, &node_info_bytes, &signature)?; + Ok(Self { + node_info, + relay_id, + relay_info, + signature, + timestamp, + }) + } + + pub fn with_secret( + node_id: NodeId, + node_info: NodeInfo, + relay_id: NodeId, + relay_info: SignedDirectNodeInfo, + secret: &DHTKeySecret, + ) -> Result { + let timestamp = intf::get_timestamp(); + let node_info_bytes = + Self::make_signature_bytes(&node_info, &relay_id, &relay_info, timestamp)?; + let signature = sign(&node_id.key, secret, &node_info_bytes)?; + Ok(Self { + node_info, + relay_id, + relay_info, + signature, + timestamp, + }) + } + + fn make_signature_bytes( + node_info: &NodeInfo, + relay_id: &NodeId, + relay_info: &SignedDirectNodeInfo, + timestamp: u64, + ) -> Result, VeilidAPIError> { + let mut sig_bytes = Vec::new(); + + // Add nodeinfo to signature + let mut ni_msg = ::capnp::message::Builder::new_default(); + let mut ni_builder = ni_msg.init_root::(); + encode_node_info(node_info, &mut ni_builder).map_err(VeilidAPIError::internal)?; + sig_bytes.append(&mut builder_to_vec(ni_msg).map_err(VeilidAPIError::internal)?); + + // Add relay id to signature + let mut rid_msg = ::capnp::message::Builder::new_default(); + let mut rid_builder = rid_msg.init_root::(); + encode_public_key(&relay_id.key, &mut rid_builder).map_err(VeilidAPIError::internal)?; + sig_bytes.append(&mut builder_to_vec(rid_msg).map_err(VeilidAPIError::internal)?); + + // Add relay info to signature + let mut ri_msg = ::capnp::message::Builder::new_default(); + let mut ri_builder = ri_msg.init_root::(); + encode_signed_direct_node_info(relay_info, &mut ri_builder) + .map_err(VeilidAPIError::internal)?; + sig_bytes.append(&mut builder_to_vec(ri_msg).map_err(VeilidAPIError::internal)?); + + // Add timestamp to signature + sig_bytes.append(&mut timestamp.to_le_bytes().to_vec()); + + Ok(sig_bytes) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(u8), derive(CheckBytes))] +pub enum SignedNodeInfo { + Direct(SignedDirectNodeInfo), + Relayed(SignedRelayedNodeInfo), +} + +impl SignedNodeInfo { + pub fn has_valid_signature(&self) -> bool { + match self { + SignedNodeInfo::Direct(d) => d.has_valid_signature(), + SignedNodeInfo::Relayed(r) => true, + } + } + + pub fn timestamp(&self) -> u64 { + match self { + SignedNodeInfo::Direct(d) => d.timestamp, + SignedNodeInfo::Relayed(r) => r.timestamp, + } + } + + pub fn node_info(&self) -> &NodeInfo { + match self { + SignedNodeInfo::Direct(d) => &d.node_info, + SignedNodeInfo::Relayed(r) => &r.node_info, + } + } + pub fn relay_id(&self) -> Option { + match self { + SignedNodeInfo::Direct(d) => None, + SignedNodeInfo::Relayed(r) => Some(r.relay_id.clone()), + } + } + pub fn relay_info(&self) -> Option<&NodeInfo> { + match self { + SignedNodeInfo::Direct(d) => None, + SignedNodeInfo::Relayed(r) => Some(&r.relay_info.node_info), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct PeerInfo { pub node_id: NodeId, pub signed_node_info: SignedNodeInfo, @@ -1680,7 +2064,177 @@ impl PeerInfo { } } -#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)] +impl PeerInfo { + /* + pub fn first_filtered_dial_info_detail( + &self, + sort: Option, + filter: F, + ) -> Option + where + S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, + F: Fn(&DialInfoDetail) -> bool, + { + if let Some(sort) = sort { + let mut dids = self.dial_info_detail_list.clone(); + dids.sort_by(sort); + for did in dids { + if filter(&did) { + return Some(did); + } + } + } else { + for did in &self.dial_info_detail_list { + if filter(did) { + return Some(did.clone()); + } + } + }; + None + } + + pub fn all_filtered_dial_info_details( + &self, + sort: Option, + filter: F, + ) -> Vec + where + S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, + F: Fn(&DialInfoDetail) -> bool, + { + let mut dial_info_detail_list = Vec::new(); + + if let Some(sort) = sort { + let mut dids = self.dial_info_detail_list.clone(); + dids.sort_by(sort); + for did in dids { + if filter(&did) { + dial_info_detail_list.push(did); + } + } + } else { + for did in &self.dial_info_detail_list { + if filter(did) { + dial_info_detail_list.push(did.clone()); + } + } + }; + dial_info_detail_list + } + + pub fn has_any_dial_info(&self) -> bool { + !self.dial_info_detail_list.is_empty() + || !self + .relay_peer_info + .as_ref() + .map(|rpi| rpi.signed_node_info.node_info.has_direct_dial_info()) + .unwrap_or_default() + } + + pub fn has_sequencing_matched_dial_info(&self, sequencing: Sequencing) -> bool { + // Check our dial info + for did in &self.dial_info_detail_list { + match sequencing { + Sequencing::NoPreference | Sequencing::PreferOrdered => return true, + Sequencing::EnsureOrdered => { + if did.dial_info.protocol_type().is_connection_oriented() { + return true; + } + } + } + } + // Check our relay if we have one + return self + .relay_peer_info + .as_ref() + .map(|rpi| { + let relay_ni = &rpi.signed_node_info.node_info; + for did in &relay_ni.dial_info_detail_list { + match sequencing { + Sequencing::NoPreference | Sequencing::PreferOrdered => return true, + Sequencing::EnsureOrdered => { + if did.dial_info.protocol_type().is_connection_oriented() { + return true; + } + } + } + } + false + }) + .unwrap_or_default(); + } + + pub fn has_direct_dial_info(&self) -> bool { + !self.dial_info_detail_list.is_empty() + } + + // Is some relay required either for signal or inbound relay or outbound relay? + pub fn requires_relay(&self) -> bool { + match self.network_class { + NetworkClass::InboundCapable => { + for did in &self.dial_info_detail_list { + if did.class.requires_relay() { + return true; + } + } + } + NetworkClass::OutboundOnly => { + return true; + } + NetworkClass::WebApp => { + return true; + } + NetworkClass::Invalid => {} + } + false + } + + // Can this node assist with signalling? Yes but only if it doesn't require signalling, itself. + pub fn can_signal(&self) -> bool { + // Must be inbound capable + if !matches!(self.network_class, NetworkClass::InboundCapable) { + return false; + } + // Do any of our dial info require signalling? if so, we can't offer signalling + for did in &self.dial_info_detail_list { + if did.class.requires_signal() { + return false; + } + } + true + } + + // Can this node relay be an inbound relay? + pub fn can_inbound_relay(&self) -> bool { + // For now this is the same + self.can_signal() + } + + // Is this node capable of validating dial info + pub fn can_validate_dial_info(&self) -> bool { + // For now this is the same + self.can_signal() + } + + */ +} + +#[derive( + Copy, + Clone, + Debug, + PartialEq, + PartialOrd, + Eq, + Ord, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct PeerAddress { protocol_type: ProtocolType, #[serde(with = "json_as_string")] @@ -1719,7 +2273,22 @@ impl PeerAddress { /// If the medium does not allow local addresses, None should have been used or 'new_no_local' /// If we are specifying only a port, then the socket's 'local_address()' should have been used, since an /// established connection is always from a real address to another real address. -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Copy, + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct ConnectionDescriptor { remote: PeerAddress, local: Option, @@ -1778,7 +2347,21 @@ impl MatchesDialInfoFilter for ConnectionDescriptor { ////////////////////////////////////////////////////////////////////////// -#[derive(Clone, Debug, Default, Eq, PartialEq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + PartialOrd, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct NodeDialInfo { pub node_id: NodeId, pub dial_info: DialInfo, @@ -1813,7 +2396,19 @@ impl FromStr for NodeDialInfo { } } -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Clone, + Debug, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct LatencyStats { #[serde(with = "json_as_string")] pub fastest: u64, // fastest latency in the ROLLING_LATENCIES_SIZE last latencies @@ -1823,7 +2418,19 @@ pub struct LatencyStats { pub slowest: u64, // slowest latency in the ROLLING_LATENCIES_SIZE last latencies } -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Clone, + Debug, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct TransferStats { #[serde(with = "json_as_string")] pub total: u64, // total amount transferred ever @@ -1835,13 +2442,37 @@ pub struct TransferStats { pub minimum: u64, // minimum rate over the ROLLING_TRANSFERS_SIZE last amounts } -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Clone, + Debug, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct TransferStatsDownUp { pub down: TransferStats, pub up: TransferStats, } -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Clone, + Debug, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct RPCStats { pub messages_sent: u32, // number of rpcs that have been sent in the total_time range pub messages_rcvd: u32, // number of rpcs that have been received in the total_time range @@ -1856,7 +2487,19 @@ pub struct RPCStats { pub failed_to_send: u32, // number of messages that have failed to send since we last successfully sent one } -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Clone, + Debug, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct PeerStats { #[serde(with = "json_as_string")] pub time_added: u64, // when the peer was added to the routing table @@ -1870,7 +2513,8 @@ pub type ValueChangeCallback = ///////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum SignalInfo { HolePunch { // UDP Hole Punch Request @@ -1887,13 +2531,41 @@ pub enum SignalInfo { } ///////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +#[derive( + Copy, + Clone, + Debug, + PartialOrd, + PartialEq, + Eq, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum TunnelMode { Raw, Turn, } -#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +#[derive( + Copy, + Clone, + Debug, + PartialOrd, + PartialEq, + Eq, + Ord, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] +#[archive_attr(repr(u8), derive(CheckBytes))] pub enum TunnelError { BadId, // Tunnel ID was rejected NoEndpoint, // Endpoint was unreachable @@ -1903,7 +2575,8 @@ pub enum TunnelError { pub type TunnelId = u64; -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct TunnelEndpoint { pub mode: TunnelMode, pub description: String, // XXX: TODO @@ -1918,7 +2591,10 @@ impl Default for TunnelEndpoint { } } -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive( + Clone, Debug, Default, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct FullTunnel { pub id: TunnelId, pub timeout: u64, @@ -1926,7 +2602,10 @@ pub struct FullTunnel { pub remote: TunnelEndpoint, } -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive( + Clone, Debug, Default, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct PartialTunnel { pub id: TunnelId, pub timeout: u64, diff --git a/veilid-core/src/veilid_api/serialize_helpers.rs b/veilid-core/src/veilid_api/serialize_helpers.rs index 3f843e4b..bb8fb045 100644 --- a/veilid-core/src/veilid_api/serialize_helpers.rs +++ b/veilid-core/src/veilid_api/serialize_helpers.rs @@ -1,5 +1,9 @@ use super::*; +pub use bytecheck::CheckBytes; use core::fmt::Debug; +pub use rkyv::Archive as RkyvArchive; +pub use rkyv::Deserialize as RkyvDeserialize; +pub use rkyv::Serialize as RkyvSerialize; // XXX: Don't trace these functions as they are used in the transfer of API logs, which will recurse! @@ -128,3 +132,79 @@ pub mod arc_serialize { Ok(Arc::new(T::deserialize(d)?)) } } + +pub fn to_rkyv(v: &T) -> EyreResult> +where + T: RkyvSerialize>, +{ + Ok(rkyv::to_bytes::(v) + .wrap_err("failed to freeze object")? + .to_vec()) +} + +pub fn from_rkyv(v: Vec) -> EyreResult +where + T: RkyvArchive, + ::Archived: + for<'t> bytecheck::CheckBytes>, + ::Archived: + rkyv::Deserialize, +{ + match rkyv::from_bytes::(&v) { + Ok(v) => Ok(v), + Err(e) => { + bail!("failed to deserialize frozen object: {}", e); + } + } +} + +pub struct RkyvEnumSet; + +impl rkyv::with::ArchiveWith> for RkyvEnumSet +where + T: EnumSetType + EnumSetTypeWithRepr, + ::Repr: rkyv::Archive, +{ + type Archived = rkyv::Archived<::Repr>; + type Resolver = rkyv::Resolver<::Repr>; + + #[inline] + unsafe fn resolve_with( + field: &EnumSet, + pos: usize, + resolver: Self::Resolver, + out: *mut Self::Archived, + ) { + let r = field.as_repr(); + r.resolve(pos, resolver, out); + } +} + +impl rkyv::with::SerializeWith, S> for RkyvEnumSet +where + S: rkyv::Fallible + ?Sized, + T: EnumSetType + EnumSetTypeWithRepr, + ::Repr: rkyv::Serialize, +{ + fn serialize_with(field: &EnumSet, serializer: &mut S) -> Result { + let r = field.as_repr(); + r.serialize(serializer) + } +} + +impl + rkyv::with::DeserializeWith::Repr>, EnumSet, D> + for RkyvEnumSet +where + D: rkyv::Fallible + ?Sized, + T: EnumSetType + EnumSetTypeWithRepr, + ::Repr: rkyv::Archive, + rkyv::Archived<::Repr>: rkyv::Deserialize, D>, +{ + fn deserialize_with( + field: &rkyv::Archived<::Repr>, + deserializer: &mut D, + ) -> Result, D::Error> { + Ok(field.deserialize(deserializer)?.into()) + } +} From 592c83d83a24a784339013e3da3126fefaea08bd Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 9 Nov 2022 22:27:37 -0500 Subject: [PATCH 46/67] checkpoint --- veilid-core/src/network_manager/mod.rs | 12 +- veilid-core/src/routing_table/mod.rs | 34 +-- veilid-core/src/routing_table/node_ref.rs | 44 ++- .../src/routing_table/route_spec_store.rs | 63 ++-- .../src/routing_table/routing_domains.rs | 109 ++++--- .../src/routing_table/routing_table_inner.rs | 56 ++-- veilid-core/src/rpc_processor/mod.rs | 12 +- .../src/rpc_processor/rpc_find_node.rs | 5 +- .../src/rpc_processor/rpc_node_info_update.rs | 5 +- veilid-core/src/veilid_api/mod.rs | 284 +++++++++--------- 10 files changed, 299 insertions(+), 325 deletions(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 21d0f5eb..37ec14ef 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1210,18 +1210,16 @@ impl NetworkManager { }; // Node A is our own node - let node_a = routing_table.get_own_node_info(routing_domain); - let node_a_id = routing_table.node_id(); + let peer_a = routing_table.get_own_peer_info(routing_domain); // Node B is the target node - let node_b = match target_node_ref.node_info(routing_domain) { + let peer_b = match target_node_ref.make_peer_info(routing_domain) { Some(ni) => ni, None => { log_net!("no node info for node {:?}", target_node_ref); return Ok(NodeContactMethod::Unreachable); } }; - let node_b_id = target_node_ref.node_id(); // Dial info filter comes from the target node ref let dial_info_filter = target_node_ref.dial_info_filter(); @@ -1229,10 +1227,8 @@ impl NetworkManager { let cm = routing_table.get_contact_method( routing_domain, - &node_a_id, - &node_a, - &node_b_id, - &node_b, + &peer_a, + &peer_b, dial_info_filter, sequencing, ); diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index fd1db5e2..6f585960 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -348,24 +348,30 @@ impl RoutingTable { .node_info_is_valid_in_routing_domain(routing_domain, node_info) } + pub fn signed_node_info_is_valid_in_routing_domain( + &self, + routing_domain: RoutingDomain, + signed_node_info: &SignedNodeInfo, + ) -> bool { + self.inner + .read() + .signed_node_info_is_valid_in_routing_domain(routing_domain, signed_node_info) + } + /// Look up the best way for two nodes to reach each other over a specific routing domain #[instrument(level = "trace", skip(self), ret)] pub fn get_contact_method( &self, routing_domain: RoutingDomain, - node_a_id: &DHTKey, - node_a: &NodeInfo, - node_b_id: &DHTKey, - node_b: &NodeInfo, + peer_a: &PeerInfo, + peer_b: &PeerInfo, dial_info_filter: DialInfoFilter, sequencing: Sequencing, ) -> ContactMethod { self.inner.read().get_contact_method( routing_domain, - node_a_id, - node_a, - node_b_id, - node_b, + peer_a, + peer_b, dial_info_filter, sequencing, ) @@ -381,16 +387,6 @@ impl RoutingTable { self.inner.read().get_own_peer_info(routing_domain) } - /// Return a copy of our node's signednodeinfo - pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedDirectNodeInfo { - self.inner.read().get_own_signed_node_info(routing_domain) - } - - /// Return a copy of our node's nodeinfo - pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo { - self.inner.read().get_own_node_info(routing_domain) - } - /// If we have a valid network class in this routing domain, then our 'NodeInfo' is valid pub fn has_valid_own_node_info(&self, routing_domain: RoutingDomain) -> bool { self.inner.read().has_valid_own_node_info(routing_domain) @@ -525,7 +521,7 @@ impl RoutingTable { &self, routing_domain: RoutingDomain, node_id: DHTKey, - signed_node_info: SignedDirectNodeInfo, + signed_node_info: SignedNodeInfo, allow_invalid: bool, ) -> Option { self.inner.write().register_node_with_signed_node_info( diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 6f7f9adf..ae77c4df 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -116,10 +116,10 @@ pub trait NodeRefBase: Sized { e.update_node_status(node_status); }); } - fn min_max_version(&self) -> Option<(u8, u8)> { + fn min_max_version(&self) -> Option { self.operate(|_rti, e| e.min_max_version()) } - fn set_min_max_version(&self, min_max_version: (u8, u8)) { + fn set_min_max_version(&self, min_max_version: VersionRange) { self.operate_mut(|_rti, e| e.set_min_max_version(min_max_version)) } fn state(&self, cur_ts: u64) -> BucketEntryState { @@ -170,26 +170,24 @@ pub trait NodeRefBase: Sized { } fn relay(&self, routing_domain: RoutingDomain) -> Option { self.operate_mut(|rti, e| { - let opt_target_rpi = e - .node_info(routing_domain) - .map(|n| n.relay_peer_info.as_ref().map(|pi| pi.as_ref().clone())) - .flatten(); - opt_target_rpi.and_then(|t| { - // If relay is ourselves, then return None, because we can't relay through ourselves - // and to contact this node we should have had an existing inbound connection - if t.node_id.key == rti.unlocked_inner.node_id { - return None; - } + e.signed_node_info(routing_domain) + .and_then(|n| n.relay_peer_info()) + .and_then(|t| { + // If relay is ourselves, then return None, because we can't relay through ourselves + // and to contact this node we should have had an existing inbound connection + if t.node_id.key == rti.unlocked_inner.node_id { + return None; + } - // Register relay node and return noderef - rti.register_node_with_signed_node_info( - self.routing_table(), - routing_domain, - t.node_id.key, - t.signed_node_info, - false, - ) - }) + // Register relay node and return noderef + rti.register_node_with_signed_node_info( + self.routing_table(), + routing_domain, + t.node_id.key, + t.signed_node_info, + false, + ) + }) }) } @@ -301,8 +299,8 @@ pub trait NodeRefBase: Sized { fn has_any_dial_info(&self) -> bool { self.operate(|_rti, e| { for rtd in RoutingDomain::all() { - if let Some(ni) = e.node_info(rtd) { - if ni.has_any_dial_info() { + if let Some(sni) = e.signed_node_info(rtd) { + if sni.has_any_dial_info() { return true; } } diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index b10d2b76..a03c0614 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -114,10 +114,10 @@ fn route_hops_to_hop_cache(hops: &[DHTKey]) -> Vec { } /// get the hop cache key for a particular route permutation -fn route_permutation_to_hop_cache(nodes: &[(DHTKey, NodeInfo)], perm: &[usize]) -> Vec { +fn route_permutation_to_hop_cache(nodes: &[PeerInfo], perm: &[usize]) -> Vec { let mut cache: Vec = Vec::with_capacity(perm.len() * DHT_KEY_LENGTH); for n in perm { - cache.extend_from_slice(&nodes[*n].0.bytes) + cache.extend_from_slice(&nodes[*n].node_id.key.bytes) } cache } @@ -422,12 +422,12 @@ impl RouteSpecStore { // Exclude nodes with no publicinternet nodeinfo, or incompatible nodeinfo or node status won't route v.with(rti, move |_rti, e| { - let node_info_ok = if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) - { - ni.has_sequencing_matched_dial_info(sequencing) - } else { - false - }; + let node_info_ok = + if let Some(sni) = e.signed_node_info(RoutingDomain::PublicInternet) { + sni.has_sequencing_matched_dial_info(sequencing) + } else { + false + }; let node_status_ok = if let Some(ns) = e.node_status(RoutingDomain::PublicInternet) { ns.will_route() @@ -495,20 +495,15 @@ impl RouteSpecStore { }); cmpout }; - let transform = |rti: &RoutingTableInner, - k: DHTKey, - v: Option>| - -> (DHTKey, NodeInfo) { - // Return the key and the nodeinfo for that key - ( - k, + let transform = + |rti: &RoutingTableInner, k: DHTKey, v: Option>| -> PeerInfo { + // Return the peerinfo for that key v.unwrap().with(rti, |_rti, e| { - e.node_info(RoutingDomain::PublicInternet.into()) + e.make_peer_info(k, RoutingDomain::PublicInternet.into()) .unwrap() .clone() - }), - ) - }; + }) + }; // Pull the whole routing table in sorted order let node_count = rti.get_entry_count( @@ -536,18 +531,15 @@ impl RouteSpecStore { // Ensure this route is viable by checking that each node can contact the next one if directions.contains(Direction::Outbound) { - let our_node_info = rti.get_own_node_info(RoutingDomain::PublicInternet); - let our_node_id = rti.node_id(); - let mut previous_node = &(our_node_id, our_node_info); + let our_peer_info = rti.get_own_peer_info(RoutingDomain::PublicInternet); + let mut previous_node = &our_peer_info; let mut reachable = true; for n in permutation { let current_node = nodes.get(*n).unwrap(); let cm = rti.get_contact_method( RoutingDomain::PublicInternet, - &previous_node.0, - &previous_node.1, - ¤t_node.0, - ¤t_node.1, + previous_node, + current_node, DialInfoFilter::all(), sequencing, ); @@ -562,18 +554,15 @@ impl RouteSpecStore { } } if directions.contains(Direction::Inbound) { - let our_node_info = rti.get_own_node_info(RoutingDomain::PublicInternet); - let our_node_id = rti.node_id(); - let mut next_node = &(our_node_id, our_node_info); + let our_peer_info = rti.get_own_peer_info(RoutingDomain::PublicInternet); + let mut next_node = &our_peer_info; let mut reachable = true; for n in permutation.iter().rev() { let current_node = nodes.get(*n).unwrap(); let cm = rti.get_contact_method( RoutingDomain::PublicInternet, - &next_node.0, - &next_node.1, - ¤t_node.0, - ¤t_node.1, + next_node, + current_node, DialInfoFilter::all(), sequencing, ); @@ -609,11 +598,11 @@ impl RouteSpecStore { } // Got a unique route, lets build the detail, register it, and return it - let hops = route_nodes.iter().map(|v| nodes[*v].0).collect(); - let hop_node_refs = route_nodes + let hops: Vec = route_nodes.iter().map(|v| nodes[*v].node_id.key).collect(); + let hop_node_refs = hops .iter() - .map(|v| { - rti.lookup_node_ref(self.unlocked_inner.routing_table.clone(), nodes[*v].0) + .map(|k| { + rti.lookup_node_ref(self.unlocked_inner.routing_table.clone(), *k) .unwrap() }) .collect(); diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 54aeec46..d52365c8 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -111,18 +111,31 @@ impl RoutingDomainDetailCommon { dial_info_detail_list: self.dial_info_details.clone(), }; - let relay_peer_info = self + let relay_info = self .relay_node .as_ref() - .and_then(|rn| rn.locked(rti).make_peer_info(self.routing_domain)); + .and_then(|rn| { + let opt_relay_pi = rn.locked(rti).make_peer_info(self.routing_domain); + if let Some(relay_pi) = opt_relay_pi { + match relay_pi.signed_node_info { + SignedNodeInfo::Direct(d) => Some((relay_pi.node_id, d)), + SignedNodeInfo::Relayed(_) => { + warn!("relay node should not have a relay itself! if this happens, a relay updated its signed node info and became a relay, which should cause the relay to be dropped"); + None + }, + } + } else { + None + } + }); - let signed_node_info = match relay_peer_info { - Some(relay_pi) => SignedNodeInfo::Relayed( + let signed_node_info = match relay_info { + Some((relay_id, relay_sdni)) => SignedNodeInfo::Relayed( SignedRelayedNodeInfo::with_secret( NodeId::new(rti.unlocked_inner.node_id), node_info, - relay_pi.node_id, - relay_pi.signed_node_info, + relay_id, + relay_sdni, &rti.unlocked_inner.node_id_secret, ) .unwrap(), @@ -133,7 +146,7 @@ impl RoutingDomainDetailCommon { node_info, &rti.unlocked_inner.node_id_secret, ) - .unwrap(), + .unwrap() ), }; @@ -147,27 +160,8 @@ impl RoutingDomainDetailCommon { let mut cpi = self.cached_peer_info.lock(); if cpi.is_none() { // Regenerate peer info - let pi = PeerInfo::new( - NodeId::new(rti.unlocked_inner.node_id), - SignedDirectNodeInfo::with_secret( - NodeInfo { - network_class: self.network_class.unwrap_or(NetworkClass::Invalid), - outbound_protocols: self.outbound_protocols, - address_types: self.address_types, - min_version: MIN_CRYPTO_VERSION, - max_version: MAX_CRYPTO_VERSION, - dial_info_detail_list: self.dial_info_details.clone(), - relay_peer_info: self.relay_node.as_ref().and_then(|rn| { - rn.locked(rti) - .make_peer_info(self.routing_domain) - .map(Box::new) - }), - }, - NodeId::new(rti.unlocked_inner.node_id), - &rti.unlocked_inner.node_id_secret, - ) - .unwrap(), - ); + let pi = self.make_peer_info(rti); + // Cache the peer info *cpi = Some(pi); } @@ -204,10 +198,8 @@ pub trait RoutingDomainDetail { fn get_contact_method( &self, rti: &RoutingTableInner, - node_a_id: &DHTKey, - node_a: &NodeInfo, - node_b_id: &DHTKey, - node_b: &NodeInfo, + peer_a: &PeerInfo, + peer_b: &PeerInfo, dial_info_filter: DialInfoFilter, sequencing: Sequencing, ) -> ContactMethod; @@ -280,13 +272,15 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { fn get_contact_method( &self, _rti: &RoutingTableInner, - node_a_id: &DHTKey, - node_a: &NodeInfo, - node_b_id: &DHTKey, - node_b: &NodeInfo, + peer_a: &PeerInfo, + peer_b: &PeerInfo, dial_info_filter: DialInfoFilter, sequencing: Sequencing, ) -> ContactMethod { + // Get the nodeinfos for convenience + let node_a = peer_a.signed_node_info.node_info(); + let node_b = peer_b.signed_node_info.node_info(); + // Get the best match dial info for node B if we have it if let Some(target_did) = first_filtered_dial_info_detail(node_a, node_b, &dial_info_filter, sequencing) @@ -298,17 +292,18 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { } // Get the target's inbound relay, it must have one or it is not reachable - if let Some(inbound_relay) = &node_b.relay_peer_info { + if let Some(node_b_relay) = peer_b.signed_node_info.relay_info() { + let node_b_relay_id = peer_b.signed_node_info.relay_id().unwrap(); // Note that relay_peer_info could be node_a, in which case a connection already exists // and we shouldn't have even gotten here - if inbound_relay.node_id.key == *node_a_id { + if node_b_relay_id.key == peer_a.node_id.key { return ContactMethod::Existing; } // Can node A reach the inbound relay directly? if first_filtered_dial_info_detail( node_a, - &inbound_relay.signed_node_info.node_info, + node_b_relay, &dial_info_filter, sequencing, ) @@ -332,8 +327,8 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { // Can we receive a direct reverse connection? if !reverse_did.class.requires_signal() { return ContactMethod::SignalReverse( - inbound_relay.node_id.key, - *node_b_id, + node_b_relay_id.key, + peer_b.node_id.key, ); } } @@ -364,8 +359,8 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { { // The target and ourselves have a udp dialinfo that they can reach return ContactMethod::SignalHolePunch( - inbound_relay.node_id.key, - *node_b_id, + node_b_relay_id.key, + peer_a.node_id.key, ); } } @@ -373,28 +368,30 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { // Otherwise we have to inbound relay } - return ContactMethod::InboundRelay(inbound_relay.node_id.key); + return ContactMethod::InboundRelay(node_b_relay_id.key); } } } // If the node B has no direct dial info, it needs to have an inbound relay - else if let Some(inbound_relay) = &node_b.relay_peer_info { + else if let Some(node_b_relay) = peer_b.signed_node_info.relay_info() { + let node_b_relay_id = peer_b.signed_node_info.relay_id().unwrap(); + // Can we reach the full relay? if first_filtered_dial_info_detail( node_a, - &inbound_relay.signed_node_info.node_info, + &node_b_relay, &dial_info_filter, sequencing, ) .is_some() { - return ContactMethod::InboundRelay(inbound_relay.node_id.key); + return ContactMethod::InboundRelay(node_b_relay_id.key); } } // If node A can't reach the node by other means, it may need to use its own relay - if let Some(outbound_relay) = &node_a.relay_peer_info { - return ContactMethod::OutboundRelay(outbound_relay.node_id.key); + if let Some(node_a_relay_id) = peer_a.signed_node_info.relay_id() { + return ContactMethod::OutboundRelay(node_a_relay_id.key); } ContactMethod::Unreachable @@ -450,20 +447,18 @@ impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { fn get_contact_method( &self, _rti: &RoutingTableInner, - _node_a_id: &DHTKey, - node_a: &NodeInfo, - _node_b_id: &DHTKey, - node_b: &NodeInfo, + peer_a: &PeerInfo, + peer_b: &PeerInfo, dial_info_filter: DialInfoFilter, sequencing: Sequencing, ) -> ContactMethod { // Scope the filter down to protocols node A can do outbound let dial_info_filter = dial_info_filter.filtered( &DialInfoFilter::all() - .with_address_type_set(node_a.address_types) - .with_protocol_type_set(node_a.outbound_protocols), + .with_address_type_set(peer_a.signed_node_info.node_info().address_types) + .with_protocol_type_set(peer_a.signed_node_info.node_info().outbound_protocols), ); - + // Get first filtered dialinfo let (sort, dial_info_filter) = match sequencing { Sequencing::NoPreference => (None, dial_info_filter), @@ -485,7 +480,7 @@ impl RoutingDomainDetail for LocalNetworkRoutingDomainDetail { let filter = |did: &DialInfoDetail| did.matches_filter(&dial_info_filter); - let opt_target_did = node_b.first_filtered_dial_info_detail(sort, filter); + let opt_target_did = peer_b.signed_node_info.node_info().first_filtered_dial_info_detail(sort, filter); if let Some(target_did) = opt_target_did { return ContactMethod::Direct(target_did.dial_info); } diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index cfd162aa..f37672fe 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -191,9 +191,20 @@ impl RoutingTableInner { return false; } } + true + } + + pub fn signed_node_info_is_valid_in_routing_domain( + &self, + routing_domain: RoutingDomain, + signed_node_info: &SignedNodeInfo, + ) -> bool { + if !self.node_info_is_valid_in_routing_domain(routing_domain, signed_node_info.node_info()) + { + return false; + } // Ensure the relay is also valid in this routing domain if it is provided - if let Some(relay_peer_info) = node_info.relay_peer_info.as_ref() { - let relay_ni = &relay_peer_info.signed_node_info.node_info; + if let Some(relay_ni) = signed_node_info.relay_info() { if !self.node_info_is_valid_in_routing_domain(routing_domain, relay_ni) { return false; } @@ -205,23 +216,13 @@ impl RoutingTableInner { pub fn get_contact_method( &self, routing_domain: RoutingDomain, - node_a_id: &DHTKey, - node_a: &NodeInfo, - node_b_id: &DHTKey, - node_b: &NodeInfo, + peer_a: &PeerInfo, + peer_b: &PeerInfo, dial_info_filter: DialInfoFilter, sequencing: Sequencing, ) -> ContactMethod { self.with_routing_domain(routing_domain, |rdd| { - rdd.get_contact_method( - self, - node_a_id, - node_a, - node_b_id, - node_b, - dial_info_filter, - sequencing, - ) + rdd.get_contact_method(self, peer_a, peer_b, dial_info_filter, sequencing) }) } @@ -252,22 +253,6 @@ impl RoutingTableInner { }) } - /// Return a copy of our node's signednodeinfo - pub fn get_own_signed_node_info(&self, routing_domain: RoutingDomain) -> SignedDirectNodeInfo { - self.with_routing_domain(routing_domain, |rdd| { - rdd.common() - .with_peer_info(self, |pi| pi.signed_node_info.clone()) - }) - } - - /// Return a copy of our node's nodeinfo - pub fn get_own_node_info(&self, routing_domain: RoutingDomain) -> NodeInfo { - self.with_routing_domain(routing_domain, |rdd| { - rdd.common() - .with_peer_info(self, |pi| pi.signed_node_info.node_info.clone()) - }) - } - /// Return our currently registered network class pub fn has_valid_own_node_info(&self, routing_domain: RoutingDomain) -> bool { self.with_routing_domain(routing_domain, |rdd| rdd.common().has_valid_own_node_info()) @@ -662,7 +647,7 @@ impl RoutingTableInner { outer_self: RoutingTable, routing_domain: RoutingDomain, node_id: DHTKey, - signed_node_info: SignedDirectNodeInfo, + signed_node_info: SignedNodeInfo, allow_invalid: bool, ) -> Option { // validate signed node info is not something malicious @@ -670,8 +655,8 @@ impl RoutingTableInner { log_rtab!(debug "can't register own node id in routing table"); return None; } - if let Some(rpi) = &signed_node_info.node_info.relay_peer_info { - if rpi.node_id.key == node_id { + if let Some(relay_id) = signed_node_info.relay_id() { + if relay_id.key == node_id { log_rtab!(debug "node can not be its own relay"); return None; } @@ -683,8 +668,7 @@ impl RoutingTableInner { return None; } // verify signed node info is valid in this routing domain - if !self - .node_info_is_valid_in_routing_domain(routing_domain, &signed_node_info.node_info) + if !self.signed_node_info_is_valid_in_routing_domain(routing_domain, &signed_node_info) { log_rtab!(debug "signed node info for {} not valid in the {:?} routing domain", node_id, routing_domain); return None; diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 4d0a6f85..8b3ccc6a 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -336,10 +336,14 @@ impl RPCProcessor { ////////////////////////////////////////////////////////////////////// - /// Determine if a NodeInfo can be placed into the specified routing domain - fn filter_node_info(&self, routing_domain: RoutingDomain, node_info: &NodeInfo) -> bool { + /// Determine if a SignedNodeInfo can be placed into the specified routing domain + fn filter_node_info( + &self, + routing_domain: RoutingDomain, + signed_node_info: &SignedNodeInfo, + ) -> bool { let routing_table = self.routing_table(); - routing_table.node_info_is_valid_in_routing_domain(routing_domain, &node_info) + routing_table.signed_node_info_is_valid_in_routing_domain(routing_domain, &signed_node_info) } ////////////////////////////////////////////////////////////////////// @@ -619,7 +623,7 @@ impl RPCProcessor { // routing table caching when it is okay to do so // This is only done in the PublicInternet routing domain because // as far as we can tell this is the only domain that will really benefit - fn get_sender_signed_node_info(&self, dest: &Destination) -> Option { + fn get_sender_signed_node_info(&self, dest: &Destination) -> Option { // Don't do this if the sender is to remain private // Otherwise we would be attaching the original sender's identity to the final destination, // thus defeating the purpose of the safety route entirely :P diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index 42d7c4d1..219d7f9b 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -50,10 +50,7 @@ impl RPCProcessor { // Verify peers are in the correct peer scope for peer_info in &find_node_a.peers { - if !self.filter_node_info( - RoutingDomain::PublicInternet, - &peer_info.signed_node_info.node_info, - ) { + if !self.filter_node_info(RoutingDomain::PublicInternet, &peer_info.signed_node_info) { return Err(RPCError::invalid_format( "find_node response has invalid peer scope", )); diff --git a/veilid-core/src/rpc_processor/rpc_node_info_update.rs b/veilid-core/src/rpc_processor/rpc_node_info_update.rs index c5ec9f86..b264baeb 100644 --- a/veilid-core/src/rpc_processor/rpc_node_info_update.rs +++ b/veilid-core/src/rpc_processor/rpc_node_info_update.rs @@ -11,7 +11,8 @@ impl RPCProcessor { // Get the signed node info for the desired routing domain to send update with let signed_node_info = self .routing_table() - .get_own_signed_node_info(routing_domain); + .get_own_peer_info(routing_domain) + .signed_node_info; let node_info_update = RPCOperationNodeInfoUpdate { signed_node_info }; let statement = RPCStatement::new(RPCStatementDetail::NodeInfoUpdate(node_info_update)); @@ -50,7 +51,7 @@ impl RPCProcessor { }; // Update our routing table with signed node info - if !self.filter_node_info(routing_domain, &node_info_update.signed_node_info.node_info) { + if !self.filter_node_info(routing_domain, &node_info_update.signed_node_info) { log_rpc!(debug "node info doesn't belong in {:?} routing domain: {}", routing_domain, sender_node_id); return Ok(()); } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 9f522aba..caa0435c 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -775,6 +775,68 @@ pub struct NodeInfo { pub dial_info_detail_list: Vec, } +impl NodeInfo { + pub fn first_filtered_dial_info_detail( + &self, + sort: Option, + filter: F, + ) -> Option + where + S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, + F: Fn(&DialInfoDetail) -> bool, + { + if let Some(sort) = sort { + let mut dids = self.dial_info_detail_list.clone(); + dids.sort_by(sort); + for did in dids { + if filter(&did) { + return Some(did); + } + } + } else { + for did in &self.dial_info_detail_list { + if filter(did) { + return Some(did.clone()); + } + } + }; + None + } + + pub fn all_filtered_dial_info_details( + &self, + sort: Option, + filter: F, + ) -> Vec + where + S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, + F: Fn(&DialInfoDetail) -> bool, + { + let mut dial_info_detail_list = Vec::new(); + + if let Some(sort) = sort { + let mut dids = self.dial_info_detail_list.clone(); + dids.sort_by(sort); + for did in dids { + if filter(&did) { + dial_info_detail_list.push(did); + } + } + } else { + for did in &self.dial_info_detail_list { + if filter(did) { + dial_info_detail_list.push(did.clone()); + } + } + }; + dial_info_detail_list + } + + pub fn has_direct_dial_info(&self) -> bool { + !self.dial_info_detail_list.is_empty() + } +} + #[allow(clippy::derive_hash_xor_eq)] #[derive( Debug, @@ -2046,6 +2108,53 @@ impl SignedNodeInfo { SignedNodeInfo::Relayed(r) => Some(&r.relay_info.node_info), } } + pub fn relay_peer_info(&self) -> Option { + match self { + SignedNodeInfo::Direct(d) => None, + SignedNodeInfo::Relayed(r) => Some(PeerInfo::new( + r.relay_id.clone(), + SignedNodeInfo::Direct(r.relay_info.clone()), + )), + } + } + pub fn has_any_dial_info(&self) -> bool { + self.node_info().has_direct_dial_info() + || self + .relay_info() + .map(|relay_ni| relay_ni.has_direct_dial_info()) + .unwrap_or_default() + } + + pub fn has_sequencing_matched_dial_info(&self, sequencing: Sequencing) -> bool { + // Check our dial info + for did in &self.node_info().dial_info_detail_list { + match sequencing { + Sequencing::NoPreference | Sequencing::PreferOrdered => return true, + Sequencing::EnsureOrdered => { + if did.dial_info.protocol_type().is_connection_oriented() { + return true; + } + } + } + } + // Check our relay if we have one + return self + .relay_info() + .map(|relay_ni| { + for did in &relay_ni.dial_info_detail_list { + match sequencing { + Sequencing::NoPreference | Sequencing::PreferOrdered => return true, + Sequencing::EnsureOrdered => { + if did.dial_info.protocol_type().is_connection_oriented() { + return true; + } + } + } + } + false + }) + .unwrap_or_default(); + } } #[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] @@ -2066,157 +2175,62 @@ impl PeerInfo { impl PeerInfo { /* - pub fn first_filtered_dial_info_detail( - &self, - sort: Option, - filter: F, - ) -> Option - where - S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, - F: Fn(&DialInfoDetail) -> bool, - { - if let Some(sort) = sort { - let mut dids = self.dial_info_detail_list.clone(); - dids.sort_by(sort); - for did in dids { - if filter(&did) { - return Some(did); - } - } - } else { - for did in &self.dial_info_detail_list { - if filter(did) { - return Some(did.clone()); - } - } - }; - None - } - pub fn all_filtered_dial_info_details( - &self, - sort: Option, - filter: F, - ) -> Vec - where - S: Fn(&DialInfoDetail, &DialInfoDetail) -> std::cmp::Ordering, - F: Fn(&DialInfoDetail) -> bool, - { - let mut dial_info_detail_list = Vec::new(); + xxx move these back to NodeInfo - if let Some(sort) = sort { - let mut dids = self.dial_info_detail_list.clone(); - dids.sort_by(sort); - for did in dids { - if filter(&did) { - dial_info_detail_list.push(did); - } - } - } else { - for did in &self.dial_info_detail_list { - if filter(did) { - dial_info_detail_list.push(did.clone()); - } - } - }; - dial_info_detail_list - } - - pub fn has_any_dial_info(&self) -> bool { - !self.dial_info_detail_list.is_empty() - || !self - .relay_peer_info - .as_ref() - .map(|rpi| rpi.signed_node_info.node_info.has_direct_dial_info()) - .unwrap_or_default() - } - - pub fn has_sequencing_matched_dial_info(&self, sequencing: Sequencing) -> bool { - // Check our dial info - for did in &self.dial_info_detail_list { - match sequencing { - Sequencing::NoPreference | Sequencing::PreferOrdered => return true, - Sequencing::EnsureOrdered => { - if did.dial_info.protocol_type().is_connection_oriented() { - return true; - } - } - } + pub fn has_direct_dial_info(&self) -> bool { + !self.dial_info_detail_list.is_empty() } - // Check our relay if we have one - return self - .relay_peer_info - .as_ref() - .map(|rpi| { - let relay_ni = &rpi.signed_node_info.node_info; - for did in &relay_ni.dial_info_detail_list { - match sequencing { - Sequencing::NoPreference | Sequencing::PreferOrdered => return true, - Sequencing::EnsureOrdered => { - if did.dial_info.protocol_type().is_connection_oriented() { - return true; - } + + // Is some relay required either for signal or inbound relay or outbound relay? + pub fn requires_relay(&self) -> bool { + match self.network_class { + NetworkClass::InboundCapable => { + for did in &self.dial_info_detail_list { + if did.class.requires_relay() { + return true; } } } - false - }) - .unwrap_or_default(); - } - - pub fn has_direct_dial_info(&self) -> bool { - !self.dial_info_detail_list.is_empty() - } - - // Is some relay required either for signal or inbound relay or outbound relay? - pub fn requires_relay(&self) -> bool { - match self.network_class { - NetworkClass::InboundCapable => { - for did in &self.dial_info_detail_list { - if did.class.requires_relay() { - return true; - } + NetworkClass::OutboundOnly => { + return true; } + NetworkClass::WebApp => { + return true; + } + NetworkClass::Invalid => {} } - NetworkClass::OutboundOnly => { - return true; - } - NetworkClass::WebApp => { - return true; - } - NetworkClass::Invalid => {} + false } - false - } - // Can this node assist with signalling? Yes but only if it doesn't require signalling, itself. - pub fn can_signal(&self) -> bool { - // Must be inbound capable - if !matches!(self.network_class, NetworkClass::InboundCapable) { - return false; - } - // Do any of our dial info require signalling? if so, we can't offer signalling - for did in &self.dial_info_detail_list { - if did.class.requires_signal() { + // Can this node assist with signalling? Yes but only if it doesn't require signalling, itself. + pub fn can_signal(&self) -> bool { + // Must be inbound capable + if !matches!(self.network_class, NetworkClass::InboundCapable) { return false; } + // Do any of our dial info require signalling? if so, we can't offer signalling + for did in &self.dial_info_detail_list { + if did.class.requires_signal() { + return false; + } + } + true } - true - } - // Can this node relay be an inbound relay? - pub fn can_inbound_relay(&self) -> bool { - // For now this is the same - self.can_signal() - } + // Can this node relay be an inbound relay? + pub fn can_inbound_relay(&self) -> bool { + // For now this is the same + self.can_signal() + } - // Is this node capable of validating dial info - pub fn can_validate_dial_info(&self) -> bool { - // For now this is the same - self.can_signal() - } + // Is this node capable of validating dial info + pub fn can_validate_dial_info(&self) -> bool { + // For now this is the same + self.can_signal() + } - */ + */ } #[derive( From 9c2a7488f17d543975d9b0911b6c107b9abd2e34 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 10 Nov 2022 21:53:45 -0500 Subject: [PATCH 47/67] bug fixes --- veilid-core/src/crypto/envelope.rs | 8 +- veilid-core/src/crypto/key.rs | 2 +- veilid-core/src/intf/table_db.rs | 3 +- veilid-core/src/network_manager/mod.rs | 36 ++--- .../native/network_class_discovery.rs | 4 +- veilid-core/src/network_manager/tasks.rs | 14 +- veilid-core/src/routing_table/bucket_entry.rs | 30 +++-- veilid-core/src/routing_table/debug.rs | 12 +- veilid-core/src/routing_table/mod.rs | 6 +- .../src/routing_table/route_spec_store.rs | 55 ++++---- veilid-core/src/rpc_processor/mod.rs | 14 +- veilid-core/src/veilid_api/mod.rs | 123 ++++++++---------- .../src/veilid_api/serialize_helpers.rs | 5 +- 13 files changed, 166 insertions(+), 146 deletions(-) diff --git a/veilid-core/src/crypto/envelope.rs b/veilid-core/src/crypto/envelope.rs index 3a3c75e4..03cb0cf1 100644 --- a/veilid-core/src/crypto/envelope.rs +++ b/veilid-core/src/crypto/envelope.rs @@ -4,6 +4,7 @@ use super::*; use crate::xx::*; use crate::*; use core::convert::TryInto; +use crate::routing_table::VersionRange; // #[repr(C, packed)] // struct EnvelopeHeader { @@ -271,8 +272,11 @@ impl Envelope { self.version } - pub fn get_min_max_version(&self) -> (u8, u8) { - (self.min_version, self.max_version) + pub fn get_min_max_version(&self) -> VersionRange { + VersionRange { + min: self.min_version, + max: self.max_version, + } } pub fn get_timestamp(&self) -> u64 { diff --git a/veilid-core/src/crypto/key.rs b/veilid-core/src/crypto/key.rs index 79ae26e5..ad484a65 100644 --- a/veilid-core/src/crypto/key.rs +++ b/veilid-core/src/crypto/key.rs @@ -39,7 +39,7 @@ pub const DHT_SIGNATURE_LENGTH_ENCODED: usize = 86; macro_rules! byte_array_type { ($name:ident, $size:expr) => { #[derive(Clone, Copy, RkyvArchive, RkyvSerialize, RkyvDeserialize)] - #[archive_attr(repr(C), derive(CheckBytes))] + #[archive_attr(repr(C), derive(CheckBytes, Hash, Eq, PartialEq, PartialOrd, Ord))] pub struct $name { pub bytes: [u8; $size], pub valid: bool, diff --git a/veilid-core/src/intf/table_db.rs b/veilid-core/src/intf/table_db.rs index ddf6cf72..6064d051 100644 --- a/veilid-core/src/intf/table_db.rs +++ b/veilid-core/src/intf/table_db.rs @@ -1,6 +1,5 @@ use crate::xx::*; use crate::*; -use serde::{Deserialize, Serialize}; cfg_if! { if #[cfg(target_arch = "wasm32")] { @@ -223,7 +222,7 @@ impl<'a> TableDBTransaction<'a> { /// Store a key in rkyv format with a value in a column in the TableDB pub fn store_json(&mut self, col: u32, key: &[u8], value: &T) -> EyreResult<()> where - T: Serialize, + T: serde::Serialize, { let v = serde_json::to_vec(value)?; self.dbt.as_mut().unwrap().put(col, key, v.as_slice()); diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 37ec14ef..6c87aaf1 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -646,15 +646,16 @@ impl NetworkManager { /// Get our node's capabilities in the PublicInternet routing domain fn generate_public_internet_node_status(&self) -> PublicInternetNodeStatus { - let node_info = self + let own_peer_info = self .routing_table() - .get_own_node_info(RoutingDomain::PublicInternet); + .get_own_peer_info(RoutingDomain::PublicInternet); + let own_node_info = own_peer_info.signed_node_info.node_info(); - let will_route = node_info.can_inbound_relay(); // xxx: eventually this may have more criteria added - let will_tunnel = node_info.can_inbound_relay(); // xxx: we may want to restrict by battery life and network bandwidth at some point - let will_signal = node_info.can_signal(); - let will_relay = node_info.can_inbound_relay(); - let will_validate_dial_info = node_info.can_validate_dial_info(); + let will_route = own_node_info.can_inbound_relay(); // xxx: eventually this may have more criteria added + let will_tunnel = own_node_info.can_inbound_relay(); // xxx: we may want to restrict by battery life and network bandwidth at some point + let will_signal = own_node_info.can_signal(); + let will_relay = own_node_info.can_inbound_relay(); + let will_validate_dial_info = own_node_info.can_validate_dial_info(); PublicInternetNodeStatus { will_route, @@ -666,12 +667,14 @@ impl NetworkManager { } /// Get our node's capabilities in the LocalNetwork routing domain fn generate_local_network_node_status(&self) -> LocalNetworkNodeStatus { - let node_info = self + let own_peer_info = self .routing_table() - .get_own_node_info(RoutingDomain::LocalNetwork); + .get_own_peer_info(RoutingDomain::LocalNetwork); - let will_relay = node_info.can_inbound_relay(); - let will_validate_dial_info = node_info.can_validate_dial_info(); + let own_node_info = own_peer_info.signed_node_info.node_info(); + + let will_relay = own_node_info.can_inbound_relay(); + let will_validate_dial_info = own_node_info.can_validate_dial_info(); LocalNetworkNodeStatus { will_relay, @@ -960,17 +963,18 @@ impl NetworkManager { } // Get node's min/max version and see if we can send to it // and if so, get the max version we can use - let version = if let Some((node_min, node_max)) = node_ref.min_max_version() { + let version = if let Some(min_max_version) = node_ref.min_max_version() { #[allow(clippy::absurd_extreme_comparisons)] - if node_min > MAX_CRYPTO_VERSION || node_max < MIN_CRYPTO_VERSION { + if min_max_version.min > MAX_CRYPTO_VERSION || min_max_version.max < MIN_CRYPTO_VERSION + { bail!( "can't talk to this node {} because version is unsupported: ({},{})", via_node_id, - node_min, - node_max + min_max_version.min, + min_max_version.max ); } - cmp::min(node_max, MAX_CRYPTO_VERSION) + cmp::min(min_max_version.max, MAX_CRYPTO_VERSION) } else { MAX_CRYPTO_VERSION }; diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/network_class_discovery.rs index 2b8b8aca..7a2daa08 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/network_class_discovery.rs @@ -129,8 +129,8 @@ impl DiscoveryContext { move |rti: &RoutingTableInner, _k: DHTKey, v: Option>| { let v = v.unwrap(); v.with(rti, |_rti, e| { - if let Some(n) = e.node_info(RoutingDomain::PublicInternet) { - n.relay_peer_info.is_none() + if let Some(n) = e.signed_node_info(RoutingDomain::PublicInternet) { + n.relay_id().is_none() } else { false } diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index 07554099..6ec80996 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -295,15 +295,14 @@ impl NetworkManager { if let Some(nr) = routing_table.register_node_with_signed_node_info( RoutingDomain::PublicInternet, k, - SignedDirectNodeInfo::with_no_signature(NodeInfo { + SignedNodeInfo::Direct(SignedDirectNodeInfo::with_no_signature(NodeInfo { network_class: NetworkClass::InboundCapable, // Bootstraps are always inbound capable outbound_protocols: ProtocolTypeSet::only(ProtocolType::UDP), // Bootstraps do not participate in relaying and will not make outbound requests, but will have UDP enabled address_types: AddressTypeSet::all(), // Bootstraps are always IPV4 and IPV6 capable min_version: v.min_version, // Minimum crypto version specified in txt record max_version: v.max_version, // Maximum crypto version specified in txt record dial_info_detail_list: v.dial_info_details, // Dial info is as specified in the bootstrap list - relay_peer_info: None, // Bootstraps never require a relay themselves - }), + })), true, ) { // Add this our futures to process in parallel @@ -524,7 +523,8 @@ impl NetworkManager { ) -> EyreResult<()> { // Get our node's current node info and network class and do the right thing let routing_table = self.routing_table(); - let node_info = routing_table.get_own_node_info(RoutingDomain::PublicInternet); + let own_peer_info = routing_table.get_own_peer_info(RoutingDomain::PublicInternet); + let own_node_info = own_peer_info.signed_node_info.node_info(); let network_class = routing_table.get_network_class(RoutingDomain::PublicInternet); // Get routing domain editor @@ -541,7 +541,7 @@ impl NetworkManager { info!("Relay node died, dropping relay {}", relay_node); editor.clear_relay_node(); false - } else if !node_info.requires_relay() { + } else if !own_node_info.requires_relay() { info!( "Relay node no longer required, dropping relay {}", relay_node @@ -557,7 +557,7 @@ impl NetworkManager { }; // Do we need a relay? - if !has_relay && node_info.requires_relay() { + if !has_relay && own_node_info.requires_relay() { // Do we want an outbound relay? let mut got_outbound_relay = false; if network_class.outbound_wants_relay() { @@ -604,7 +604,7 @@ impl NetworkManager { ) -> EyreResult<()> { // Get our node's current node info and network class and do the right thing let routing_table = self.routing_table(); - let node_info = routing_table.get_own_node_info(RoutingDomain::PublicInternet); + let own_peer_info = routing_table.get_own_peer_info(RoutingDomain::PublicInternet); let network_class = routing_table.get_network_class(RoutingDomain::PublicInternet); // Get routing domain editor diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index fdecd02e..11032215 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -1,6 +1,6 @@ use super::*; use core::sync::atomic::{AtomicU32, Ordering}; -use serde::{Deserialize, Serialize}; +use rkyv::with::Skip; /// Reliable pings are done with increased spacing between pings @@ -40,10 +40,11 @@ pub enum BucketEntryState { } #[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)] -struct LastConnectionKey(ProtocolType, AddressType); +pub struct LastConnectionKey(ProtocolType, AddressType); /// Bucket entry information specific to the LocalNetwork RoutingDomain -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct BucketEntryPublicInternet { /// The PublicInternet node info signed_node_info: Option>, @@ -54,7 +55,8 @@ pub struct BucketEntryPublicInternet { } /// Bucket entry information specific to the LocalNetwork RoutingDomain -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct BucketEntryLocalNetwork { /// The LocalNetwork node info signed_node_info: Option>, @@ -65,16 +67,18 @@ pub struct BucketEntryLocalNetwork { } /// A range of cryptography versions supported by this entry -#[derive(Debug, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct VersionRange { /// The minimum cryptography version supported by this entry - min: u8, + pub min: u8, /// The maximum cryptography version supported by this entry - max: u8, + pub max: u8, } /// The data associated with each bucket entry -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct BucketEntryInner { /// The minimum and maximum range of cryptography versions supported by the node, /// inclusive of the requirements of any relay the node may be using @@ -83,7 +87,7 @@ pub struct BucketEntryInner { /// and dial info has last changed, for example when our IP address changes updated_since_last_network_change: bool, /// The last connection descriptors used to contact this node, per protocol type - #[serde(skip)] + #[with(Skip)] last_connections: BTreeMap, /// The node info for this entry on the publicinternet routing domain public_internet: BucketEntryPublicInternet, @@ -92,18 +96,18 @@ pub struct BucketEntryInner { /// Statistics gathered for the peer peer_stats: PeerStats, /// The accounting for the latency statistics - #[serde(skip)] + #[with(Skip)] latency_stats_accounting: LatencyStatsAccounting, /// The accounting for the transfer statistics - #[serde(skip)] + #[with(Skip)] transfer_stats_accounting: TransferStatsAccounting, /// Tracking identifier for NodeRef debugging #[cfg(feature = "tracking")] - #[serde(skip)] + #[with(Skip)] next_track_id: usize, /// Backtraces for NodeRef debugging #[cfg(feature = "tracking")] - #[serde(skip)] + #[with(Skip)] node_ref_tracks: HashMap, } diff --git a/veilid-core/src/routing_table/debug.rs b/veilid-core/src/routing_table/debug.rs index 43af8b78..3170fae6 100644 --- a/veilid-core/src/routing_table/debug.rs +++ b/veilid-core/src/routing_table/debug.rs @@ -1,7 +1,7 @@ use super::*; impl RoutingTable { - pub fn debug_info_nodeinfo(&self) -> String { + pub(crate) fn debug_info_nodeinfo(&self) -> String { let mut out = String::new(); let inner = self.inner.read(); out += "Routing Table Info:\n"; @@ -23,7 +23,7 @@ impl RoutingTable { out } - pub async fn debug_info_txtrecord(&self) -> String { + pub(crate) async fn debug_info_txtrecord(&self) -> String { let mut out = String::new(); let gdis = self.dial_info_details(RoutingDomain::PublicInternet); @@ -71,7 +71,7 @@ impl RoutingTable { out } - pub fn debug_info_dialinfo(&self) -> String { + pub(crate) fn debug_info_dialinfo(&self) -> String { let ldis = self.dial_info_details(RoutingDomain::LocalNetwork); let gdis = self.dial_info_details(RoutingDomain::PublicInternet); let mut out = String::new(); @@ -100,7 +100,7 @@ impl RoutingTable { out } - pub fn debug_info_entries(&self, limit: usize, min_state: BucketEntryState) -> String { + pub(crate) fn debug_info_entries(&self, limit: usize, min_state: BucketEntryState) -> String { let inner = self.inner.read(); let inner = &*inner; let cur_ts = intf::get_timestamp(); @@ -148,7 +148,7 @@ impl RoutingTable { out } - pub fn debug_info_entry(&self, node_id: DHTKey) -> String { + pub(crate) fn debug_info_entry(&self, node_id: DHTKey) -> String { let mut out = String::new(); out += &format!("Entry {:?}:\n", node_id); if let Some(nr) = self.lookup_node_ref(node_id) { @@ -160,7 +160,7 @@ impl RoutingTable { out } - pub fn debug_info_buckets(&self, min_state: BucketEntryState) -> String { + pub(crate) fn debug_info_buckets(&self, min_state: BucketEntryState) -> String { let inner = self.inner.read(); let inner = &*inner; let cur_ts = intf::get_timestamp(); diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 6f585960..0fcb7304 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -239,7 +239,7 @@ impl RoutingTable { let tdb = table_store.open("routing_table", 1).await?; let bucket_count = bucketvec.len(); let mut dbx = tdb.transact(); - if let Err(e) = dbx.store_frozen(0, b"bucket_count", &bucket_count) { + if let Err(e) = dbx.store_rkyv(0, b"bucket_count", &bucket_count) { dbx.rollback(); return Err(e); } @@ -845,8 +845,8 @@ impl RoutingTable { } // node can not be its own relay - if let Some(rpi) = &p.signed_node_info.node_info.relay_peer_info { - if rpi.node_id == p.node_id { + if let Some(rid) = &p.signed_node_info.relay_id() { + if rid.key == p.node_id.key { continue; } } diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index a03c0614..e1230214 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -1,6 +1,6 @@ use super::*; use crate::veilid_api::*; -use serde::*; +use rkyv::with::Skip; /// Compiled route (safety route + private route) #[derive(Clone, Debug)] @@ -13,32 +13,40 @@ pub struct CompiledRoute { pub first_hop: NodeRef, } -#[derive(Clone, Debug, Serialize, Deserialize)] -struct RouteSpecDetail { +#[derive(Clone, Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] +pub struct KeyPair { + key: DHTKey, + secret: DHTKeySecret, +} + +#[derive(Clone, Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] +pub struct RouteSpecDetail { /// Secret key - #[serde(skip)] + #[with(Skip)] pub secret_key: DHTKeySecret, /// Route hops pub hops: Vec, /// Route noderefs - #[serde(skip)] + #[with(Skip)] hop_node_refs: Vec, /// Transfers up and down transfer_stats_down_up: TransferStatsDownUp, /// Latency stats latency_stats: LatencyStats, /// Accounting mechanism for this route's RPC latency - #[serde(skip)] + #[with(Skip)] latency_stats_accounting: LatencyStatsAccounting, /// Accounting mechanism for the bandwidth across this route - #[serde(skip)] + #[with(Skip)] transfer_stats_accounting: TransferStatsAccounting, /// Published private route, do not reuse for ephemeral routes /// Not serialized because all routes should be re-published when restarting - #[serde(skip)] + #[with(Skip)] published: bool, // Can optimize the rendering of this route, using node ids only instead of full peer info - #[serde(skip)] + #[with(Skip)] reachable: bool, /// Timestamp of when the route was created created_ts: u64, @@ -47,6 +55,7 @@ struct RouteSpecDetail { /// Timestamp of when the route was last used for anything last_used_ts: Option, /// Directions this route is guaranteed to work in + #[with(RkyvEnumSet)] directions: DirectionSet, /// Stability preference (prefer reliable nodes over faster) pub stability: Stability, @@ -55,7 +64,8 @@ struct RouteSpecDetail { } /// The core representation of the RouteSpecStore that can be serialized -#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[archive_attr(repr(C), derive(CheckBytes))] pub struct RouteSpecStoreContent { /// All of the routes we have allocated so far details: HashMap, @@ -225,7 +235,7 @@ impl RouteSpecStore { let table_store = routing_table.network_manager().table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; let mut content: RouteSpecStoreContent = - rsstdb.load_json(0, b"content")?.unwrap_or_default(); + rsstdb.load_rkyv(0, b"content")?.unwrap_or_default(); // Look up all route hop noderefs since we can't serialize those let mut dead_keys = Vec::new(); @@ -245,17 +255,17 @@ impl RouteSpecStore { // Load secrets from pstore let pstore = routing_table.network_manager().protected_store(); - let out: Vec<(DHTKey, DHTKeySecret)> = pstore + let out: Vec = pstore .load_user_secret_rkyv("RouteSpecStore") .await? .unwrap_or_default(); let mut dead_keys = Vec::new(); - for (k, v) in out { - if let Some(rsd) = content.details.get_mut(&k) { - rsd.secret_key = v; + for KeyPair { key, secret } in out { + if let Some(rsd) = content.details.get_mut(&key) { + rsd.secret_key = secret; } else { - dead_keys.push(k); + dead_keys.push(key); } } for k in dead_keys { @@ -296,7 +306,7 @@ impl RouteSpecStore { .network_manager() .table_store(); let rsstdb = table_store.open("RouteSpecStore", 1).await?; - rsstdb.store_json(0, b"content", &content)?; + rsstdb.store_rkyv(0, b"content", &content)?; // // Keep secrets in protected store as well let pstore = self @@ -305,14 +315,15 @@ impl RouteSpecStore { .network_manager() .protected_store(); - let mut out: Vec<(DHTKey, DHTKeySecret)> = Vec::with_capacity(content.details.len()); + let mut out: Vec = Vec::with_capacity(content.details.len()); for (k, v) in &content.details { - out.push((*k, v.secret_key)); + out.push(KeyPair { + key: *k, + secret: v.secret_key, + }); } - let _ = pstore - .save_user_secret_frozen("RouteSpecStore", &out) - .await?; // ignore if this previously existed or not + let _ = pstore.save_user_secret_rkyv("RouteSpecStore", &out).await?; // ignore if this previously existed or not Ok(()) } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 8b3ccc6a..d74527a4 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -648,7 +648,11 @@ impl RPCProcessor { if target.has_seen_our_node_info(RoutingDomain::PublicInternet) { return None; } - Some(routing_table.get_own_signed_node_info(RoutingDomain::PublicInternet)) + Some( + routing_table + .get_own_peer_info(RoutingDomain::PublicInternet) + .signed_node_info, + ) } Destination::Relay { relay: _, @@ -659,7 +663,11 @@ impl RPCProcessor { if target.has_seen_our_node_info(RoutingDomain::PublicInternet) { return None; } - Some(routing_table.get_own_signed_node_info(RoutingDomain::PublicInternet)) + Some( + routing_table + .get_own_peer_info(RoutingDomain::PublicInternet) + .signed_node_info, + ) } else { None } @@ -929,7 +937,7 @@ impl RPCProcessor { let mut opt_sender_nr: Option = None; if let Some(sender_node_info) = operation.sender_node_info() { // Sender NodeInfo was specified, update our routing table with it - if !self.filter_node_info(routing_domain, &sender_node_info.node_info) { + if !self.filter_node_info(routing_domain, &sender_node_info) { return Err(RPCError::invalid_format( "sender signednodeinfo has invalid peer scope", )); diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index caa0435c..93fb47ce 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -832,9 +832,58 @@ impl NodeInfo { dial_info_detail_list } - pub fn has_direct_dial_info(&self) -> bool { + /// Does this node has some dial info + pub fn has_dial_info(&self) -> bool { !self.dial_info_detail_list.is_empty() } + + /// Is some relay required either for signal or inbound relay or outbound relay? + pub fn requires_relay(&self) -> bool { + match self.network_class { + NetworkClass::InboundCapable => { + for did in &self.dial_info_detail_list { + if did.class.requires_relay() { + return true; + } + } + } + NetworkClass::OutboundOnly => { + return true; + } + NetworkClass::WebApp => { + return true; + } + NetworkClass::Invalid => {} + } + false + } + + /// Can this node assist with signalling? Yes but only if it doesn't require signalling, itself. + pub fn can_signal(&self) -> bool { + // Must be inbound capable + if !matches!(self.network_class, NetworkClass::InboundCapable) { + return false; + } + // Do any of our dial info require signalling? if so, we can't offer signalling + for did in &self.dial_info_detail_list { + if did.class.requires_signal() { + return false; + } + } + true + } + + /// Can this node relay be an inbound relay? + pub fn can_inbound_relay(&self) -> bool { + // For now this is the same + self.can_signal() + } + + /// Is this node capable of validating dial info + pub fn can_validate_dial_info(&self) -> bool { + // For now this is the same + self.can_signal() + } } #[allow(clippy::derive_hash_xor_eq)] @@ -2079,7 +2128,7 @@ impl SignedNodeInfo { pub fn has_valid_signature(&self) -> bool { match self { SignedNodeInfo::Direct(d) => d.has_valid_signature(), - SignedNodeInfo::Relayed(r) => true, + SignedNodeInfo::Relayed(_) => true, } } @@ -2098,19 +2147,19 @@ impl SignedNodeInfo { } pub fn relay_id(&self) -> Option { match self { - SignedNodeInfo::Direct(d) => None, + SignedNodeInfo::Direct(_) => None, SignedNodeInfo::Relayed(r) => Some(r.relay_id.clone()), } } pub fn relay_info(&self) -> Option<&NodeInfo> { match self { - SignedNodeInfo::Direct(d) => None, + SignedNodeInfo::Direct(_) => None, SignedNodeInfo::Relayed(r) => Some(&r.relay_info.node_info), } } pub fn relay_peer_info(&self) -> Option { match self { - SignedNodeInfo::Direct(d) => None, + SignedNodeInfo::Direct(_) => None, SignedNodeInfo::Relayed(r) => Some(PeerInfo::new( r.relay_id.clone(), SignedNodeInfo::Direct(r.relay_info.clone()), @@ -2118,10 +2167,10 @@ impl SignedNodeInfo { } } pub fn has_any_dial_info(&self) -> bool { - self.node_info().has_direct_dial_info() + self.node_info().has_dial_info() || self .relay_info() - .map(|relay_ni| relay_ni.has_direct_dial_info()) + .map(|relay_ni| relay_ni.has_dial_info()) .unwrap_or_default() } @@ -2173,66 +2222,6 @@ impl PeerInfo { } } -impl PeerInfo { - /* - - xxx move these back to NodeInfo - - pub fn has_direct_dial_info(&self) -> bool { - !self.dial_info_detail_list.is_empty() - } - - // Is some relay required either for signal or inbound relay or outbound relay? - pub fn requires_relay(&self) -> bool { - match self.network_class { - NetworkClass::InboundCapable => { - for did in &self.dial_info_detail_list { - if did.class.requires_relay() { - return true; - } - } - } - NetworkClass::OutboundOnly => { - return true; - } - NetworkClass::WebApp => { - return true; - } - NetworkClass::Invalid => {} - } - false - } - - // Can this node assist with signalling? Yes but only if it doesn't require signalling, itself. - pub fn can_signal(&self) -> bool { - // Must be inbound capable - if !matches!(self.network_class, NetworkClass::InboundCapable) { - return false; - } - // Do any of our dial info require signalling? if so, we can't offer signalling - for did in &self.dial_info_detail_list { - if did.class.requires_signal() { - return false; - } - } - true - } - - // Can this node relay be an inbound relay? - pub fn can_inbound_relay(&self) -> bool { - // For now this is the same - self.can_signal() - } - - // Is this node capable of validating dial info - pub fn can_validate_dial_info(&self) -> bool { - // For now this is the same - self.can_signal() - } - - */ -} - #[derive( Copy, Clone, diff --git a/veilid-core/src/veilid_api/serialize_helpers.rs b/veilid-core/src/veilid_api/serialize_helpers.rs index bb8fb045..74e990f9 100644 --- a/veilid-core/src/veilid_api/serialize_helpers.rs +++ b/veilid-core/src/veilid_api/serialize_helpers.rs @@ -199,12 +199,13 @@ where D: rkyv::Fallible + ?Sized, T: EnumSetType + EnumSetTypeWithRepr, ::Repr: rkyv::Archive, - rkyv::Archived<::Repr>: rkyv::Deserialize, D>, + rkyv::Archived<::Repr>: + rkyv::Deserialize<::Repr, D>, { fn deserialize_with( field: &rkyv::Archived<::Repr>, deserializer: &mut D, ) -> Result, D::Error> { - Ok(field.deserialize(deserializer)?.into()) + Ok(EnumSet::::from_repr(field.deserialize(deserializer)?)) } } From fbe5d807e6683e3cc2fee67075bbf05a4a3c749b Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 10 Nov 2022 22:11:57 -0500 Subject: [PATCH 48/67] unit tests pass --- veilid-core/src/attachment_manager.rs | 1 + veilid-core/src/crypto/key.rs | 1 + veilid-core/src/intf/native/protected_store.rs | 3 ++- veilid-core/src/intf/table_db.rs | 3 ++- veilid-core/src/routing_table/bucket.rs | 1 + veilid-core/src/routing_table/bucket_entry.rs | 4 +++- veilid-core/src/routing_table/route_spec_store.rs | 4 +++- veilid-core/src/veilid_api/mod.rs | 1 + veilid-core/src/veilid_api/serialize_helpers.rs | 6 +++--- 9 files changed, 17 insertions(+), 7 deletions(-) diff --git a/veilid-core/src/attachment_manager.rs b/veilid-core/src/attachment_manager.rs index 8afafcdd..48447e3f 100644 --- a/veilid-core/src/attachment_manager.rs +++ b/veilid-core/src/attachment_manager.rs @@ -6,6 +6,7 @@ use crate::xx::*; use crate::*; use core::convert::TryFrom; use core::fmt; +use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; use serde::*; state_machine! { diff --git a/veilid-core/src/crypto/key.rs b/veilid-core/src/crypto/key.rs index ad484a65..7da8c9ad 100644 --- a/veilid-core/src/crypto/key.rs +++ b/veilid-core/src/crypto/key.rs @@ -12,6 +12,7 @@ use digest::generic_array::typenum::U64; use digest::{Digest, Output}; use ed25519_dalek::{Keypair, PublicKey, Signature}; use generic_array::GenericArray; +use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; ////////////////////////////////////////////////////////////////////// diff --git a/veilid-core/src/intf/native/protected_store.rs b/veilid-core/src/intf/native/protected_store.rs index a784189e..a3c35f68 100644 --- a/veilid-core/src/intf/native/protected_store.rs +++ b/veilid-core/src/intf/native/protected_store.rs @@ -2,6 +2,7 @@ use crate::xx::*; use crate::*; use data_encoding::BASE64URL_NOPAD; use keyring_manager::*; +use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; use std::path::Path; pub struct ProtectedStoreInner { @@ -167,7 +168,7 @@ impl ProtectedStore { ::Archived: for<'t> bytecheck::CheckBytes>, ::Archived: - rkyv::Deserialize, + RkyvDeserialize, { let out = self.load_user_secret(key).await?; let b = match out { diff --git a/veilid-core/src/intf/table_db.rs b/veilid-core/src/intf/table_db.rs index 6064d051..6e080247 100644 --- a/veilid-core/src/intf/table_db.rs +++ b/veilid-core/src/intf/table_db.rs @@ -1,5 +1,6 @@ use crate::xx::*; use crate::*; +use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; cfg_if! { if #[cfg(target_arch = "wasm32")] { @@ -123,7 +124,7 @@ impl TableDB { ::Archived: for<'t> bytecheck::CheckBytes>, ::Archived: - rkyv::Deserialize, + RkyvDeserialize, { let db = &self.inner.lock().database; let out = db.get(col, key).wrap_err("failed to get key")?; diff --git a/veilid-core/src/routing_table/bucket.rs b/veilid-core/src/routing_table/bucket.rs index 3b17c585..b1e3199c 100644 --- a/veilid-core/src/routing_table/bucket.rs +++ b/veilid-core/src/routing_table/bucket.rs @@ -1,5 +1,6 @@ use super::*; use core::sync::atomic::Ordering; +use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; pub struct Bucket { routing_table: RoutingTable, diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 11032215..2ee36d1c 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -1,6 +1,8 @@ use super::*; use core::sync::atomic::{AtomicU32, Ordering}; -use rkyv::with::Skip; +use rkyv::{ + with::Skip, Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, +}; /// Reliable pings are done with increased spacing between pings diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index e1230214..0d1f271d 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -1,6 +1,8 @@ use super::*; use crate::veilid_api::*; -use rkyv::with::Skip; +use rkyv::{ + with::Skip, Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, +}; /// Compiled route (safety route + private route) #[derive(Clone, Debug)] diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 93fb47ce..4df910ec 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -30,6 +30,7 @@ pub use routing_table::{NodeRef, NodeRefBase, RoutingTable}; use core::fmt; use core_context::{api_shutdown, VeilidCoreContext}; use enumset::*; +use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; use rpc_processor::*; use serde::*; use xx::*; diff --git a/veilid-core/src/veilid_api/serialize_helpers.rs b/veilid-core/src/veilid_api/serialize_helpers.rs index 74e990f9..680e3a48 100644 --- a/veilid-core/src/veilid_api/serialize_helpers.rs +++ b/veilid-core/src/veilid_api/serialize_helpers.rs @@ -1,9 +1,9 @@ use super::*; pub use bytecheck::CheckBytes; use core::fmt::Debug; -pub use rkyv::Archive as RkyvArchive; -pub use rkyv::Deserialize as RkyvDeserialize; -pub use rkyv::Serialize as RkyvSerialize; +use rkyv::Archive as RkyvArchive; +use rkyv::Deserialize as RkyvDeserialize; +use rkyv::Serialize as RkyvSerialize; // XXX: Don't trace these functions as they are used in the transfer of API logs, which will recurse! From baf1353fd247ccbc590b2b9312b5464032db4f19 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 11 Nov 2022 18:00:11 -0500 Subject: [PATCH 49/67] test --- Cargo.lock | 153 +++++++++++++++++--------------- veilid-server/src/client_api.rs | 2 +- 2 files changed, 83 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ed60b92..654352fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,9 +51,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464b3811b747f8f7ebc8849c9c728c39f6ac98a055edad93baf9eb330e3f8f9d" +checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107" dependencies = [ "cfg-if 1.0.0", "getrandom 0.2.8", @@ -182,22 +182,22 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" dependencies = [ - "concurrent-queue", + "concurrent-queue 1.2.4", "event-listener", "futures-core", ] [[package]] name = "async-executor" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +checksum = "17adb73da160dfb475c183343c8cccd80721ea5a605d3eb57125f0a7b7a92d0b" dependencies = [ + "async-lock", "async-task", - "concurrent-queue", + "concurrent-queue 2.0.0", "fastrand", "futures-lite", - "once_cell", "slab", ] @@ -224,7 +224,7 @@ checksum = "e8121296a9f05be7f34aa4196b1747243b3b62e048bb7906f644f3fbfc490cf7" dependencies = [ "async-lock", "autocfg", - "concurrent-queue", + "concurrent-queue 1.2.4", "futures-lite", "libc", "log", @@ -719,15 +719,15 @@ checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" [[package]] name = "capnp" -version = "0.14.10" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf2d33bdc66ac282c34d34ecc9f8d8f1d82d1540994c5f8ed1c36dd0f95aaf3" +checksum = "0e850735c543306805e2ba8ee0a9632b0f62bb05872a8be2e2674e9903a1c048" [[package]] name = "capnp-futures" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9821801cc6f199a9d9c3c793504e800c797b536d2befddaffb15144e40a6e63a" +checksum = "4660c91469f0222724da5b6f5fed9605b466c5dd55e1fbbc3aabd1034637ccd0" dependencies = [ "capnp", "futures", @@ -735,9 +735,9 @@ dependencies = [ [[package]] name = "capnp-rpc" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4f17f96f68f2c1168ed7105d9e5cb4a095a5bef3578aee0f9c0644b85ca95e" +checksum = "96b9b0311f48c3fba1cca45dcddb6f922ca544c9d9d5151f1096afd8a9d7bc31" dependencies = [ "capnp", "capnp-futures", @@ -746,9 +746,9 @@ dependencies = [ [[package]] name = "capnpc" -version = "0.14.9" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdc9f1dc84666d4ff007b1a16c8f97db80764a624625979be05d869bcff43aaa" +checksum = "0ed9e0059e1c97ed65eb246d90fb843a16f8da5eb31ec4a560acb31a825923f9" dependencies = [ "capnp", ] @@ -761,9 +761,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.74" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581f5dba903aac52ea3feb5ec4810848460ee833876f1f9b0fdeab1f19091574" +checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f" [[package]] name = "cesu8" @@ -975,6 +975,15 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "concurrent-queue" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "config" version = "0.13.2" @@ -1266,7 +1275,7 @@ checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" name = "cursive" version = "0.20.0" dependencies = [ - "ahash 0.8.1", + "ahash 0.8.2", "async-std", "cfg-if 1.0.0", "crossbeam-channel", @@ -1291,7 +1300,7 @@ dependencies = [ "flexi_logger", "lazy_static", "log", - "time 0.3.16", + "time 0.3.17", "unicode-width", ] @@ -1311,7 +1320,7 @@ dependencies = [ name = "cursive_core" version = "0.3.5" dependencies = [ - "ahash 0.8.1", + "ahash 0.8.2", "async-std", "crossbeam-channel", "enum-map", @@ -1320,7 +1329,7 @@ dependencies = [ "log", "num", "owning_ref", - "time 0.3.16", + "time 0.3.17", "tokio 1.21.2", "toml", "unicode-segmentation", @@ -1365,9 +1374,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" +checksum = "97abf9f0eca9e52b7f81b945524e76710e6cb2366aead23b7d4fbf72e281f888" dependencies = [ "cc", "cxxbridge-flags", @@ -1377,9 +1386,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" +checksum = "7cc32cc5fea1d894b77d269ddb9f192110069a8a9c1f1d441195fba90553dea3" dependencies = [ "cc", "codespan-reporting", @@ -1392,15 +1401,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" +checksum = "8ca220e4794c934dc6b1207c3b42856ad4c302f2df1712e9f8d2eec5afaacf1f" [[package]] name = "cxxbridge-macro" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" +checksum = "b846f081361125bfc8dc9d3940c84e1fd83ba54bbca7b17cd29483c828be0704" dependencies = [ "proc-macro2", "quote", @@ -1666,9 +1675,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" dependencies = [ "log", "regex", @@ -1827,7 +1836,7 @@ dependencies = [ "regex", "rustversion", "thiserror", - "time 0.3.16", + "time 0.3.17", ] [[package]] @@ -2269,9 +2278,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.22" +version = "0.14.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abfba89e19b959ca163c7752ba59d737c1ceea53a5d31a149c805446fc958064" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" dependencies = [ "bytes 1.2.1", "futures-channel", @@ -2490,9 +2499,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" [[package]] name = "itertools" @@ -2710,9 +2719,9 @@ checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" [[package]] name = "libloading" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ "cfg-if 1.0.0", "winapi 0.3.9", @@ -3270,9 +3279,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ "hermit-abi", "libc", @@ -3441,9 +3450,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.3.1" +version = "6.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" +checksum = "7b5bf27447411e9ee3ff51186bf7a08e16c341efdde93f4d823e8844429bed7e" [[package]] name = "overload" @@ -3572,9 +3581,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc7bc69c062e492337d74d59b120c274fd3d261b6bf6d3207d499b4b379c41a" +checksum = "a528564cc62c19a7acac4d81e01f39e53e25e17b934878f4c6d25cc2836e62f8" dependencies = [ "thiserror", "ucd-trie", @@ -3582,9 +3591,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b75706b9642ebcb34dab3bc7750f811609a0eb1dd8b88c2d15bf628c1c65b2" +checksum = "d5fd9bc6500181952d34bd0b2b0163a54d794227b498be0b7afa7698d0a7b18f" dependencies = [ "pest", "pest_generator", @@ -3592,9 +3601,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f9272122f5979a6511a749af9db9bfc810393f63119970d7085fed1c4ea0db" +checksum = "d2610d5ac5156217b4ff8e46ddcef7cdf44b273da2ac5bca2ecbfa86a330e7c4" dependencies = [ "pest", "pest_meta", @@ -3605,9 +3614,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8717927f9b79515e565a64fe46c38b8cd0427e64c40680b14a7365ab09ac8d" +checksum = "824749bf7e21dd66b36fbe26b3f45c713879cccd4a009a917ab8e045ca8246fe" dependencies = [ "once_cell", "pest", @@ -3753,9 +3762,9 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" @@ -3835,9 +3844,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399c3c31cdec40583bb68f0b18403400d01ec4289c383aa047560439952c4dd7" +checksum = "a0841812012b2d4a6145fae9a6af1534873c32aa67fff26bd09f8fa42c83f95a" dependencies = [ "bytes 1.2.1", "prost-derive", @@ -3845,9 +3854,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb" +checksum = "1d8b442418ea0822409d9e7d047cbf1e7e9e1760b172bf9982cf29d517c93511" dependencies = [ "bytes 1.2.1", "heck", @@ -3856,18 +3865,20 @@ dependencies = [ "log", "multimap", "petgraph", + "prettyplease", "prost", "prost-types", "regex", + "syn", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.11.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" +checksum = "164ae68b6587001ca506d3bf7f1000bfa248d0e1217b618108fba4ec1d0cc306" dependencies = [ "anyhow", "itertools", @@ -3878,9 +3889,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dfaa718ad76a44b3415e6c4d53b17c8f99160dcb3a99b10470fce8ad43f6e3e" +checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a" dependencies = [ "bytes 1.2.1", "prost", @@ -4059,9 +4070,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", @@ -4079,9 +4090,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -4682,7 +4693,7 @@ checksum = "48dfff04aade74dd495b007c831cd6f4e0cee19c344dd9dc0884c0289b70a786" dependencies = [ "log", "termcolor", - "time 0.3.16", + "time 0.3.17", ] [[package]] @@ -4898,9 +4909,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fab5c8b9980850e06d92ddbe3ab839c062c801f3927c0fb8abd6fc8e918fbca" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "itoa", "libc", @@ -4918,9 +4929,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bb801831d812c562ae7d2bfb531f26e66e4e1f6b17307ba4149c5064710e5b" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" dependencies = [ "time-core", ] @@ -5191,7 +5202,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.16", + "time 0.3.17", "tracing-subscriber", ] diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index 6207cb5f..700ea79e 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -364,7 +364,7 @@ impl ClientApi { fn send_request_to_all_clients(self: Rc, request: F) where F: Fn(u64, &mut RegistrationHandle) -> Option<::capnp::capability::RemotePromise>, - T: capnp::traits::Pipelined + for<'a> capnp::traits::Owned<'a> + 'static + Unpin, + T: capnp::traits::Pipelined + capnp::traits::Owned + 'static + Unpin, { // Send status update to each registered client let registration_map = self.inner.borrow().registration_map.clone(); From 1c93b6e8cbb158213b4c842022391a8c85d0b871 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 12 Nov 2022 12:10:38 -0500 Subject: [PATCH 50/67] rkyv issue --- veilid-core/src/crypto/envelope.rs | 14 +- veilid-core/src/crypto/key.rs | 145 +++++------------- veilid-core/src/crypto/mod.rs | 10 +- veilid-core/src/crypto/receipt.rs | 6 - veilid-core/src/crypto/tests/test_dht_key.rs | 2 - veilid-core/src/routing_table/mod.rs | 4 +- .../src/rpc_processor/coders/block_id.rs | 3 - .../src/rpc_processor/coders/public_key.rs | 3 - .../src/rpc_processor/coders/signature.rs | 4 - .../coders/signed_direct_node_info.rs | 5 +- .../src/tests/common/test_veilid_config.rs | 8 +- .../src/tests/common/test_veilid_core.rs | 2 +- veilid-core/src/veilid_api/mod.rs | 11 +- veilid-core/src/veilid_config.rs | 42 +++-- veilid-flutter/lib/veilid.dart | 4 +- veilid-server/src/cmdline.rs | 4 +- veilid-server/src/settings.rs | 15 +- 17 files changed, 86 insertions(+), 196 deletions(-) diff --git a/veilid-core/src/crypto/envelope.rs b/veilid-core/src/crypto/envelope.rs index 03cb0cf1..32a02447 100644 --- a/veilid-core/src/crypto/envelope.rs +++ b/veilid-core/src/crypto/envelope.rs @@ -1,10 +1,10 @@ #![allow(dead_code)] #![allow(clippy::absurd_extreme_comparisons)] use super::*; +use crate::routing_table::VersionRange; use crate::xx::*; use crate::*; use core::convert::TryInto; -use crate::routing_table::VersionRange; // #[repr(C, packed)] // struct EnvelopeHeader { @@ -59,9 +59,6 @@ impl Envelope { sender_id: DHTKey, recipient_id: DHTKey, ) -> Self { - assert!(sender_id.valid); - assert!(recipient_id.valid); - assert!(version >= MIN_CRYPTO_VERSION); assert!(version <= MAX_CRYPTO_VERSION); Self { @@ -206,15 +203,6 @@ impl Envelope { body: &[u8], node_id_secret: &DHTKeySecret, ) -> Result, VeilidAPIError> { - // Ensure sender node id is valid - if !self.sender_id.valid { - return Err(VeilidAPIError::generic("sender id is invalid")); - } - // Ensure recipient node id is valid - if !self.recipient_id.valid { - return Err(VeilidAPIError::generic("recipient id is invalid")); - } - // Ensure body isn't too long let envelope_size: usize = body.len() + MIN_ENVELOPE_SIZE; if envelope_size > MAX_ENVELOPE_SIZE { diff --git a/veilid-core/src/crypto/key.rs b/veilid-core/src/crypto/key.rs index 7da8c9ad..d2e5f568 100644 --- a/veilid-core/src/crypto/key.rs +++ b/veilid-core/src/crypto/key.rs @@ -2,10 +2,10 @@ use crate::veilid_rng::*; use crate::xx::*; use crate::*; -use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; +use core::cmp::{Eq, Ord, PartialEq, PartialOrd}; use core::convert::{TryFrom, TryInto}; use core::fmt; -use core::hash::{Hash, Hasher}; +use core::hash::Hash; use data_encoding::BASE64URL_NOPAD; use digest::generic_array::typenum::U64; @@ -39,11 +39,29 @@ pub const DHT_SIGNATURE_LENGTH_ENCODED: usize = 86; macro_rules! byte_array_type { ($name:ident, $size:expr) => { - #[derive(Clone, Copy, RkyvArchive, RkyvSerialize, RkyvDeserialize)] + #[derive( + Clone, + Copy, + Hash, + Eq, + PartialEq, + PartialOrd, + Ord, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + )] #[archive_attr(repr(C), derive(CheckBytes, Hash, Eq, PartialEq, PartialOrd, Ord))] pub struct $name { pub bytes: [u8; $size], - pub valid: bool, + } + + impl Default for $name { + fn default() -> Self { + Self { + bytes: [0u8; $size], + } + } } impl serde::Serialize for $name { @@ -51,12 +69,7 @@ macro_rules! byte_array_type { where S: serde::Serializer, { - let s: String; - if self.valid { - s = self.encode(); - } else { - s = "".to_owned(); - } + let s = self.encode(); serde::Serialize::serialize(&s, serializer) } } @@ -76,28 +89,19 @@ macro_rules! byte_array_type { impl $name { pub fn new(bytes: [u8; $size]) -> Self { - Self { bytes, valid: true } + Self { bytes } } pub fn try_from_vec(v: Vec) -> Result { - let mut this = Self { - bytes: [0u8; $size], - valid: true, - }; - - if v.len() != $size { - apibail_generic!(format!( - "Expected a Vec of length {} but it was {}", - $size, - v.len() - )); - } - - for n in 0..v.len() { - this.bytes[n] = v[n]; - } - - Ok(this) + let vl = v.len(); + Ok(Self { + bytes: v.try_into().map_err(|_| { + VeilidAPIError::generic(format!( + "Expected a Vec of length {} but it was {}", + $size, vl + )) + })?, + }) } pub fn bit(&self, index: usize) -> bool { @@ -143,7 +147,6 @@ macro_rules! byte_array_type { } pub fn encode(&self) -> String { - assert!(self.valid); BASE64URL_NOPAD.encode(&self.bytes) } @@ -169,63 +172,7 @@ macro_rules! byte_array_type { } } } - impl PartialOrd for $name { - fn partial_cmp(&self, other: &$name) -> Option { - Some(self.cmp(other)) - } - } - impl Ord for $name { - fn cmp(&self, other: &$name) -> Ordering { - if !self.valid && !other.valid { - return Ordering::Equal; - } - if !self.valid && other.valid { - return Ordering::Less; - } - if self.valid && !other.valid { - return Ordering::Greater; - } - for n in 0..$size { - if self.bytes[n] < other.bytes[n] { - return Ordering::Less; - } - if self.bytes[n] > other.bytes[n] { - return Ordering::Greater; - } - } - Ordering::Equal - } - } - impl PartialEq<$name> for $name { - fn eq(&self, other: &$name) -> bool { - if self.valid != other.valid { - return false; - } - for n in 0..$size { - if self.bytes[n] != other.bytes[n] { - return false; - } - } - true - } - } - impl Eq for $name {} - impl Hash for $name { - fn hash(&self, state: &mut H) { - self.valid.hash(state); - if self.valid { - self.bytes.hash(state); - } - } - } - impl Default for $name { - fn default() -> Self { - let mut this = $name::new([0u8; $size]); - this.valid = false; - this - } - } impl fmt::Display for $name { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", String::from(self)) @@ -235,24 +182,13 @@ macro_rules! byte_array_type { impl fmt::Debug for $name { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, concat!(stringify!($name), "("))?; - write!( - f, - "{}", - if self.valid { - self.encode() - } else { - "".to_owned() - } - )?; + write!(f, "{}", self.encode())?; write!(f, ")") } } impl From<&$name> for String { fn from(value: &$name) -> Self { - if !value.valid { - return "".to_owned(); - } let mut s = String::new(); for n in 0..($size / 8) { let b: [u8; 8] = value.bytes[n * 8..(n + 1) * 8].try_into().unwrap(); @@ -280,10 +216,7 @@ macro_rules! byte_array_type { apibail_generic!(concat!(stringify!($name), " is incorrect length")); } match hex::decode_to_slice(value, &mut out.bytes) { - Ok(_) => { - out.valid = true; - Ok(out) - } + Ok(_) => Ok(out), Err(err) => Err(VeilidAPIError::generic(err)), } } @@ -381,9 +314,6 @@ pub fn sign( dht_key_secret: &DHTKeySecret, data: &[u8], ) -> Result { - assert!(dht_key.valid); - assert!(dht_key_secret.valid); - let mut kpb: [u8; DHT_KEY_SECRET_LENGTH + DHT_KEY_LENGTH] = [0u8; DHT_KEY_SECRET_LENGTH + DHT_KEY_LENGTH]; @@ -408,8 +338,6 @@ pub fn verify( data: &[u8], signature: &DHTSignature, ) -> Result<(), VeilidAPIError> { - assert!(dht_key.valid); - assert!(signature.valid); let pk = PublicKey::from_bytes(&dht_key.bytes) .map_err(|e| VeilidAPIError::parse_error("Public key is invalid", e))?; let sig = Signature::from_bytes(&signature.bytes) @@ -428,7 +356,6 @@ pub fn generate_hash(data: &[u8]) -> DHTKey { } pub fn validate_hash(data: &[u8], dht_key: &DHTKey) -> bool { - assert!(dht_key.valid); let bytes = *blake3::hash(data).as_bytes(); bytes == dht_key.bytes @@ -446,8 +373,6 @@ pub fn validate_key(dht_key: &DHTKey, dht_key_secret: &DHTKeySecret) -> bool { } pub fn distance(key1: &DHTKey, key2: &DHTKey) -> DHTKeyDistance { - assert!(key1.valid); - assert!(key2.valid); let mut bytes = [0u8; DHT_KEY_LENGTH]; for (n, byte) in bytes.iter_mut().enumerate() { diff --git a/veilid-core/src/crypto/mod.rs b/veilid-core/src/crypto/mod.rs index 4184c54b..ebede803 100644 --- a/veilid-core/src/crypto/mod.rs +++ b/veilid-core/src/crypto/mod.rs @@ -112,15 +112,15 @@ impl Crypto { let (table_store, node_id) = { let mut inner = self.inner.lock(); let c = self.config.get(); - inner.node_id = c.network.node_id; - inner.node_id_secret = c.network.node_id_secret; + inner.node_id = c.network.node_id.unwrap(); + inner.node_id_secret = c.network.node_id_secret.unwrap(); (inner.table_store.clone(), c.network.node_id) }; // load caches if they are valid for this node id let mut db = table_store.open("crypto_caches", 1).await?; let caches_valid = match db.load(0, b"node_id")? { - Some(v) => v.as_slice() == node_id.bytes, + Some(v) => v.as_slice() == node_id.unwrap().bytes, None => false, }; if caches_valid { @@ -132,7 +132,7 @@ impl Crypto { drop(db); table_store.delete("crypto_caches").await?; db = table_store.open("crypto_caches", 1).await?; - db.store(0, b"node_id", &node_id.bytes)?; + db.store(0, b"node_id", &node_id.unwrap().bytes)?; } // Schedule flushing @@ -220,8 +220,6 @@ impl Crypto { // These are safe to use regardless of initialization status pub fn compute_dh(key: &DHTKey, secret: &DHTKeySecret) -> Result { - assert!(key.valid); - assert!(secret.valid); let pk_ed = ed::PublicKey::from_bytes(&key.bytes).map_err(VeilidAPIError::internal)?; let pk_xd = Self::ed25519_to_x25519_pk(&pk_ed)?; let sk_ed = ed::SecretKey::from_bytes(&secret.bytes).map_err(VeilidAPIError::internal)?; diff --git a/veilid-core/src/crypto/receipt.rs b/veilid-core/src/crypto/receipt.rs index b4990b4e..d67fea51 100644 --- a/veilid-core/src/crypto/receipt.rs +++ b/veilid-core/src/crypto/receipt.rs @@ -58,7 +58,6 @@ impl Receipt { sender_id: DHTKey, extra_data: D, ) -> Result { - assert!(sender_id.valid); if extra_data.as_ref().len() > MAX_EXTRA_DATA_SIZE { return Err(VeilidAPIError::parse_error( "extra data too large for receipt", @@ -151,11 +150,6 @@ impl Receipt { } pub fn to_signed_data(&self, secret: &DHTKeySecret) -> Result, VeilidAPIError> { - // Ensure sender node id is valid - if !self.sender_id.valid { - return Err(VeilidAPIError::internal("sender id is invalid")); - } - // Ensure extra data isn't too long let receipt_size: usize = self.extra_data.len() + MIN_RECEIPT_SIZE; if receipt_size > MAX_RECEIPT_SIZE { diff --git a/veilid-core/src/crypto/tests/test_dht_key.rs b/veilid-core/src/crypto/tests/test_dht_key.rs index 0d6eee4e..3b6ded61 100644 --- a/veilid-core/src/crypto/tests/test_dht_key.rs +++ b/veilid-core/src/crypto/tests/test_dht_key.rs @@ -88,9 +88,7 @@ pub async fn test_key_conversions() { // Test default key let (dht_key, dht_key_secret) = (key::DHTKey::default(), key::DHTKeySecret::default()); assert_eq!(dht_key.bytes, EMPTY_KEY); - assert!(!dht_key.valid); assert_eq!(dht_key_secret.bytes, EMPTY_KEY_SECRET); - assert!(!dht_key_secret.valid); let dht_key_string = String::from(&dht_key); trace!("dht_key_string: {:?}", dht_key_string); let dht_key_string2 = String::from(&dht_key); diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 0fcb7304..2336396c 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -81,8 +81,8 @@ impl RoutingTable { RoutingTableUnlockedInner { config: config.clone(), network_manager, - node_id: c.network.node_id, - node_id_secret: c.network.node_id_secret, + node_id: c.network.node_id.unwrap(), + node_id_secret: c.network.node_id_secret.unwrap(), kick_queue: Mutex::new(BTreeSet::default()), rolling_transfers_task: TickTask::new(ROLLING_TRANSFERS_INTERVAL_SECS), kick_buckets_task: TickTask::new(1), diff --git a/veilid-core/src/rpc_processor/coders/block_id.rs b/veilid-core/src/rpc_processor/coders/block_id.rs index 1d23f19b..c42481b6 100644 --- a/veilid-core/src/rpc_processor/coders/block_id.rs +++ b/veilid-core/src/rpc_processor/coders/block_id.rs @@ -22,9 +22,6 @@ pub fn encode_block_id( key: &DHTKey, builder: &mut veilid_capnp::b_l_a_k_e3_hash::Builder, ) -> Result<(), RPCError> { - if !key.valid { - return Err(RPCError::protocol("invalid key")); - } builder.set_u0(u64::from_be_bytes( key.bytes[0..8].try_into().map_err(RPCError::internal)?, )); diff --git a/veilid-core/src/rpc_processor/coders/public_key.rs b/veilid-core/src/rpc_processor/coders/public_key.rs index f7eb530a..1bec140b 100644 --- a/veilid-core/src/rpc_processor/coders/public_key.rs +++ b/veilid-core/src/rpc_processor/coders/public_key.rs @@ -22,9 +22,6 @@ pub fn encode_public_key( key: &DHTKey, builder: &mut veilid_capnp::curve25519_public_key::Builder, ) -> Result<(), RPCError> { - if !key.valid { - return Err(RPCError::protocol("invalid key")); - } builder.set_u0(u64::from_be_bytes( key.bytes[0..8] .try_into() diff --git a/veilid-core/src/rpc_processor/coders/signature.rs b/veilid-core/src/rpc_processor/coders/signature.rs index e41eb9a9..956fb74a 100644 --- a/veilid-core/src/rpc_processor/coders/signature.rs +++ b/veilid-core/src/rpc_processor/coders/signature.rs @@ -5,10 +5,6 @@ pub fn encode_signature( sig: &DHTSignature, builder: &mut veilid_capnp::ed25519_signature::Builder, ) { - if !sig.valid { - panic!("don't encode invalid signatures"); - } - let sig = &sig.bytes; builder.set_u0(u64::from_be_bytes( diff --git a/veilid-core/src/rpc_processor/coders/signed_direct_node_info.rs b/veilid-core/src/rpc_processor/coders/signed_direct_node_info.rs index 1e9768f4..a04d3445 100644 --- a/veilid-core/src/rpc_processor/coders/signed_direct_node_info.rs +++ b/veilid-core/src/rpc_processor/coders/signed_direct_node_info.rs @@ -14,7 +14,10 @@ pub fn encode_signed_direct_node_info( .set_timestamp(signed_direct_node_info.timestamp); let mut sig_builder = builder.reborrow().init_signature(); - encode_signature(&signed_direct_node_info.signature, &mut sig_builder); + let Some(signature) = &signed_direct_node_info.signature else { + return Err(RPCError::internal("Should not encode SignedDirectNodeInfo without signature!")); + }; + encode_signature(signature, &mut sig_builder); Ok(()) } diff --git a/veilid-core/src/tests/common/test_veilid_config.rs b/veilid-core/src/tests/common/test_veilid_config.rs index 423d93fa..7304c45d 100644 --- a/veilid-core/src/tests/common/test_veilid_config.rs +++ b/veilid-core/src/tests/common/test_veilid_config.rs @@ -193,8 +193,8 @@ fn config_callback(key: String) -> ConfigCallbackReturn { "network.client_whitelist_timeout_ms" => Ok(Box::new(300_000u32)), "network.reverse_connection_receipt_time_ms" => Ok(Box::new(5_000u32)), "network.hole_punch_receipt_time_ms" => Ok(Box::new(5_000u32)), - "network.node_id" => Ok(Box::new(DHTKey::default())), - "network.node_id_secret" => Ok(Box::new(DHTKeySecret::default())), + "network.node_id" => Ok(Box::new(Option::::None)), + "network.node_id_secret" => Ok(Box::new(Option::::None)), "network.bootstrap" => Ok(Box::new(Vec::::new())), "network.bootstrap_nodes" => Ok(Box::new(Vec::::new())), "network.routing_table.limit_over_attached" => Ok(Box::new(64u32)), @@ -318,8 +318,8 @@ pub async fn test_config() { assert_eq!(inner.network.client_whitelist_timeout_ms, 300_000u32); assert_eq!(inner.network.reverse_connection_receipt_time_ms, 5_000u32); assert_eq!(inner.network.hole_punch_receipt_time_ms, 5_000u32); - assert!(!inner.network.node_id.valid); - assert!(!inner.network.node_id_secret.valid); + assert!(inner.network.node_id.is_none()); + assert!(inner.network.node_id_secret.is_none()); assert_eq!(inner.network.bootstrap, Vec::::new()); assert_eq!(inner.network.bootstrap_nodes, Vec::::new()); assert_eq!(inner.network.rpc.concurrency, 2u32); diff --git a/veilid-core/src/tests/common/test_veilid_core.rs b/veilid-core/src/tests/common/test_veilid_core.rs index bf8d6915..ed31f2a6 100644 --- a/veilid-core/src/tests/common/test_veilid_core.rs +++ b/veilid-core/src/tests/common/test_veilid_core.rs @@ -73,7 +73,7 @@ pub async fn test_signed_node_info() { NodeId::new(pkey), node_info.clone(), sni.timestamp, - sni.signature, + sni.signature.unwrap(), ) .unwrap(); diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 4df910ec..bf2ac7b6 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -360,7 +360,6 @@ pub struct NodeId { } impl NodeId { pub fn new(key: DHTKey) -> Self { - assert!(key.valid); Self { key } } } @@ -1967,7 +1966,7 @@ impl MatchesDialInfoFilter for DialInfo { pub struct SignedDirectNodeInfo { pub node_info: NodeInfo, pub timestamp: u64, - pub signature: DHTSignature, + pub signature: Option, } impl SignedDirectNodeInfo { @@ -1982,7 +1981,7 @@ impl SignedDirectNodeInfo { Ok(Self { node_info, timestamp, - signature, + signature: Some(signature), }) } @@ -1997,7 +1996,7 @@ impl SignedDirectNodeInfo { Ok(Self { node_info, timestamp, - signature, + signature: Some(signature), }) } @@ -2022,13 +2021,13 @@ impl SignedDirectNodeInfo { pub fn with_no_signature(node_info: NodeInfo) -> Self { Self { node_info, - signature: DHTSignature::default(), + signature: None, timestamp: intf::get_timestamp(), } } pub fn has_valid_signature(&self) -> bool { - self.signature.valid + self.signature.is_some() } } diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index d1287864..87ea38fb 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -217,8 +217,8 @@ pub struct VeilidConfigNetwork { pub client_whitelist_timeout_ms: u32, pub reverse_connection_receipt_time_ms: u32, pub hole_punch_receipt_time_ms: u32, - pub node_id: DHTKey, - pub node_id_secret: DHTKeySecret, + pub node_id: Option, + pub node_id_secret: Option, pub bootstrap: Vec, pub bootstrap_nodes: Vec, pub routing_table: VeilidConfigRoutingTable, @@ -675,7 +675,7 @@ impl VeilidConfig { let mut node_id = self.inner.read().network.node_id; let mut node_id_secret = self.inner.read().network.node_id_secret; // See if node id was previously stored in the protected store - if !node_id.valid { + if node_id.is_none() { debug!("pulling node id from storage"); if let Some(s) = protected_store .load_user_secret_string("node_id") @@ -683,14 +683,14 @@ impl VeilidConfig { .map_err(VeilidAPIError::internal)? { debug!("node id found in storage"); - node_id = DHTKey::try_decode(s.as_str()).map_err(VeilidAPIError::internal)? + node_id = Some(DHTKey::try_decode(s.as_str()).map_err(VeilidAPIError::internal)?); } else { debug!("node id not found in storage"); } } // See if node id secret was previously stored in the protected store - if !node_id_secret.valid { + if node_id_secret.is_none() { debug!("pulling node id secret from storage"); if let Some(s) = protected_store .load_user_secret_string("node_id_secret") @@ -699,27 +699,25 @@ impl VeilidConfig { { debug!("node id secret found in storage"); node_id_secret = - DHTKeySecret::try_decode(s.as_str()).map_err(VeilidAPIError::internal)? + Some(DHTKeySecret::try_decode(s.as_str()).map_err(VeilidAPIError::internal)?); } else { debug!("node id secret not found in storage"); } } // If we have a node id from storage, check it - if node_id.valid && node_id_secret.valid { - // Validate node id - if !crypto::validate_key(&node_id, &node_id_secret) { - apibail_generic!("node id secret and node id key don't match"); - } - } - - // If we still don't have a valid node id, generate one - if !node_id.valid || !node_id_secret.valid { - debug!("generating new node id"); - let (i, s) = generate_secret(); - node_id = i; - node_id_secret = s; - } + let (node_id, node_id_secret) = + if let (Some(node_id), Some(node_id_secret)) = (node_id, node_id_secret) { + // Validate node id + if !crypto::validate_key(&node_id, &node_id_secret) { + apibail_generic!("node id secret and node id key don't match"); + } + (node_id, node_id_secret) + } else { + // If we still don't have a valid node id, generate one + debug!("generating new node id"); + generate_secret() + }; info!("Node Id is {}", node_id.encode()); // info!("Node Id Secret is {}", node_id_secret.encode()); @@ -733,8 +731,8 @@ impl VeilidConfig { .await .map_err(VeilidAPIError::internal)?; - self.inner.write().network.node_id = node_id; - self.inner.write().network.node_id_secret = node_id_secret; + self.inner.write().network.node_id = Some(node_id); + self.inner.write().network.node_id_secret = Some(node_id_secret); trace!("init_node_id complete"); diff --git a/veilid-flutter/lib/veilid.dart b/veilid-flutter/lib/veilid.dart index 70f41a68..52ff4a78 100644 --- a/veilid-flutter/lib/veilid.dart +++ b/veilid-flutter/lib/veilid.dart @@ -737,8 +737,8 @@ class VeilidConfigNetwork { int clientWhitelistTimeoutMs; int reverseConnectionReceiptTimeMs; int holePunchReceiptTimeMs; - String nodeId; - String nodeIdSecret; + String? nodeId; + String? nodeIdSecret; List bootstrap; List bootstrapNodes; VeilidConfigRoutingTable routingTable; diff --git a/veilid-server/src/cmdline.rs b/veilid-server/src/cmdline.rs index 58364971..ee40f09a 100644 --- a/veilid-server/src/cmdline.rs +++ b/veilid-server/src/cmdline.rs @@ -256,8 +256,8 @@ pub fn process_command_line() -> EyreResult<(Settings, ArgMatches)> { let s = DHTKeySecret::try_decode(&buffer)?; (k, s) }; - settingsrw.core.network.node_id = k; - settingsrw.core.network.node_id_secret = s; + settingsrw.core.network.node_id = Some(k); + settingsrw.core.network.node_id_secret = Some(s); } if matches.occurrences_of("bootstrap") != 0 { diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index 1ada4a7c..d02d7d77 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -67,8 +67,8 @@ core: client_whitelist_timeout_ms: 300000 reverse_connection_receipt_time_ms: 5000 hole_punch_receipt_time_ms: 5000 - node_id: '' - node_id_secret: '' + node_id: null + node_id_secret: null bootstrap: ['bootstrap.dev.veilid.net'] bootstrap_nodes: [] routing_table: @@ -588,8 +588,8 @@ pub struct Network { pub client_whitelist_timeout_ms: u32, pub reverse_connection_receipt_time_ms: u32, pub hole_punch_receipt_time_ms: u32, - pub node_id: veilid_core::DHTKey, - pub node_id_secret: veilid_core::DHTKeySecret, + pub node_id: Option, + pub node_id_secret: Option, pub bootstrap: Vec, pub bootstrap_nodes: Vec, pub routing_table: RoutingTable, @@ -1484,11 +1484,8 @@ mod tests { assert_eq!(s.core.network.client_whitelist_timeout_ms, 300_000u32); assert_eq!(s.core.network.reverse_connection_receipt_time_ms, 5_000u32); assert_eq!(s.core.network.hole_punch_receipt_time_ms, 5_000u32); - assert_eq!(s.core.network.node_id, veilid_core::DHTKey::default()); - assert_eq!( - s.core.network.node_id_secret, - veilid_core::DHTKeySecret::default() - ); + assert_eq!(s.core.network.node_id, None); + assert_eq!(s.core.network.node_id_secret, None); // assert_eq!( s.core.network.bootstrap, From 53b52848b55477ce6c8b7dccfd699daa72e3e818 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 12 Nov 2022 15:15:06 -0500 Subject: [PATCH 51/67] temporary fix --- Cargo.lock | 10 ++++------ veilid-core/Cargo.toml | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 654352fc..61e52043 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4105,9 +4105,9 @@ dependencies = [ [[package]] name = "rend" -version = "0.3.6" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79af64b4b6362ffba04eef3a4e10829718a4896dac19daa741851c86781edf95" +checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab" dependencies = [ "bytecheck", ] @@ -4140,8 +4140,7 @@ dependencies = [ [[package]] name = "rkyv" version = "0.7.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cec2b3485b07d96ddfd3134767b8a447b45ea4eb91448d0a35180ec0ffd5ed15" +source = "git+https://github.com/crioux/rkyv.git?branch=issue_326#2f19cfac9f31a15e2fe74ad362eec7b011dc35b9" dependencies = [ "bytecheck", "hashbrown", @@ -4154,8 +4153,7 @@ dependencies = [ [[package]] name = "rkyv_derive" version = "0.7.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eaedadc88b53e36dd32d940ed21ae4d850d5916f2581526921f553a72ac34c4" +source = "git+https://github.com/crioux/rkyv.git?branch=issue_326#2f19cfac9f31a15e2fe74ad362eec7b011dc35b9" dependencies = [ "proc-macro2", "quote", diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 905ccad1..e0ec79c8 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -60,7 +60,8 @@ async-std-resolver = { version = "^0", optional = true } trust-dns-resolver = { version = "^0", optional = true } keyvaluedb = { path = "../external/keyvaluedb/keyvaluedb" } serde_bytes = { version = "^0" } -rkyv = { version = "^0", default_features = false, features = ["std", "alloc", "strict", "size_64", "archive_le", "validation"] } +#rkyv = { version = "^0", default_features = false, features = ["std", "alloc", "strict", "size_64", "validation"] } +rkyv = { git = "https://github.com/crioux/rkyv.git", branch = "issue_326", default_features = false, features = ["std", "alloc", "strict", "size_64", "validation"] } bytecheck = "^0" # Dependencies for native builds only From 94d1598ce112bfb0262125bfae2a27467be1fa89 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 12 Nov 2022 21:29:43 -0500 Subject: [PATCH 52/67] remove node_dial_info --- veilid-core/proto/veilid.capnp | 5 -- veilid-core/src/crypto/key.rs | 6 +-- veilid-core/src/network_manager/tasks.rs | 18 ++++--- .../routing_table/routing_domain_editor.rs | 9 ++-- veilid-core/src/rpc_processor/coders/mod.rs | 2 - .../rpc_processor/coders/node_dial_info.rs | 29 ----------- veilid-core/src/veilid_api/mod.rs | 49 ------------------- veilid-server/src/settings.rs | 25 +++++++--- 8 files changed, 35 insertions(+), 108 deletions(-) delete mode 100644 veilid-core/src/rpc_processor/coders/node_dial_info.rs diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index c3f00bcf..3525da9f 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -102,11 +102,6 @@ struct DialInfo { } } -struct NodeDialInfo { - nodeId @0 :NodeID; # node id - dialInfo @1 :DialInfo; # how to get to the node -} - # Signals ############################## diff --git a/veilid-core/src/crypto/key.rs b/veilid-core/src/crypto/key.rs index d2e5f568..90f57297 100644 --- a/veilid-core/src/crypto/key.rs +++ b/veilid-core/src/crypto/key.rs @@ -150,10 +150,10 @@ macro_rules! byte_array_type { BASE64URL_NOPAD.encode(&self.bytes) } - pub fn try_decode(input: &str) -> Result { + pub fn try_decode>(input: S) -> Result { let mut bytes = [0u8; $size]; - let res = BASE64URL_NOPAD.decode_len(input.len()); + let res = BASE64URL_NOPAD.decode_len(input.as_ref().len()); match res { Ok(v) => { if v != $size { @@ -165,7 +165,7 @@ macro_rules! byte_array_type { } } - let res = BASE64URL_NOPAD.decode_mut(input.as_bytes(), &mut bytes); + let res = BASE64URL_NOPAD.decode_mut(input.as_ref().as_bytes(), &mut bytes); match res { Ok(_) => Ok(Self::new(bytes)), Err(_) => apibail_generic!("Failed to decode"), diff --git a/veilid-core/src/network_manager/tasks.rs b/veilid-core/src/network_manager/tasks.rs index 6ec80996..c9834645 100644 --- a/veilid-core/src/network_manager/tasks.rs +++ b/veilid-core/src/network_manager/tasks.rs @@ -256,14 +256,18 @@ impl NetworkManager { let mut bsmap = BootstrapRecordMap::new(); let mut bootstrap_node_dial_infos = Vec::new(); for b in bootstrap_nodes { - let ndis = NodeDialInfo::from_str(b.as_str()) - .wrap_err("Invalid node dial info in bootstrap entry")?; - bootstrap_node_dial_infos.push(ndis); + let (id_str, di_str) = b + .split_once('@') + .ok_or_else(|| eyre!("Invalid node dial info in bootstrap entry"))?; + let node_id = + NodeId::from_str(id_str).wrap_err("Invalid node id in bootstrap entry")?; + let dial_info = + DialInfo::from_str(di_str).wrap_err("Invalid dial info in bootstrap entry")?; + bootstrap_node_dial_infos.push((node_id, dial_info)); } - for ndi in bootstrap_node_dial_infos { - let node_id = ndi.node_id.key; + for (node_id, dial_info) in bootstrap_node_dial_infos { bsmap - .entry(node_id) + .entry(node_id.key) .or_insert_with(|| BootstrapRecord { min_version: MIN_CRYPTO_VERSION, max_version: MAX_CRYPTO_VERSION, @@ -271,7 +275,7 @@ impl NetworkManager { }) .dial_info_details .push(DialInfoDetail { - dial_info: ndi.dial_info, + dial_info, class: DialInfoClass::Direct, // Bootstraps are always directly reachable }); } diff --git a/veilid-core/src/routing_table/routing_domain_editor.rs b/veilid-core/src/routing_table/routing_domain_editor.rs index 60e76509..b9150196 100644 --- a/veilid-core/src/routing_table/routing_domain_editor.rs +++ b/veilid-core/src/routing_table/routing_domain_editor.rs @@ -139,13 +139,10 @@ impl RoutingDomainEditor { .add_dial_info_detail(dial_info_detail.clone()); info!( - "{:?} Dial Info: {}", + "{:?} Dial Info: {}@{}", self.routing_domain, - NodeDialInfo { - node_id: NodeId::new(node_id), - dial_info: dial_info_detail.dial_info - } - .to_string(), + NodeId::new(node_id), + dial_info_detail.dial_info ); changed = true; } diff --git a/veilid-core/src/rpc_processor/coders/mod.rs b/veilid-core/src/rpc_processor/coders/mod.rs index 3211b338..ccc6dc17 100644 --- a/veilid-core/src/rpc_processor/coders/mod.rs +++ b/veilid-core/src/rpc_processor/coders/mod.rs @@ -5,7 +5,6 @@ mod dial_info; mod dial_info_class; mod dial_info_detail; mod network_class; -mod node_dial_info; mod node_info; mod node_status; mod nonce; @@ -32,7 +31,6 @@ pub use dial_info::*; pub use dial_info_class::*; pub use dial_info_detail::*; pub use network_class::*; -pub use node_dial_info::*; pub use node_info::*; pub use node_status::*; pub use nonce::*; diff --git a/veilid-core/src/rpc_processor/coders/node_dial_info.rs b/veilid-core/src/rpc_processor/coders/node_dial_info.rs deleted file mode 100644 index 47ae96ad..00000000 --- a/veilid-core/src/rpc_processor/coders/node_dial_info.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::*; -use rpc_processor::*; - -pub fn encode_node_dial_info( - ndis: &NodeDialInfo, - builder: &mut veilid_capnp::node_dial_info::Builder, -) -> Result<(), RPCError> { - let mut ni_builder = builder.reborrow().init_node_id(); - encode_public_key(&ndis.node_id.key, &mut ni_builder)?; - let mut di_builder = builder.reborrow().init_dial_info(); - encode_dial_info(&ndis.dial_info, &mut di_builder)?; - Ok(()) -} - -pub fn decode_node_dial_info( - reader: &veilid_capnp::node_dial_info::Reader, -) -> Result { - let node_id = decode_public_key(&reader.get_node_id().map_err(RPCError::map_protocol( - "invalid public key in node_dial_info", - ))?); - let dial_info = decode_dial_info(&reader.get_dial_info().map_err(RPCError::map_protocol( - "invalid dial_info in node_dial_info", - ))?)?; - - Ok(NodeDialInfo { - node_id: NodeId::new(node_id), - dial_info, - }) -} diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index bf2ac7b6..69348803 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -2350,55 +2350,6 @@ impl MatchesDialInfoFilter for ConnectionDescriptor { ////////////////////////////////////////////////////////////////////////// -#[derive( - Clone, - Debug, - Default, - Eq, - PartialEq, - PartialOrd, - Ord, - Serialize, - Deserialize, - RkyvArchive, - RkyvSerialize, - RkyvDeserialize, -)] -#[archive_attr(repr(C), derive(CheckBytes))] -pub struct NodeDialInfo { - pub node_id: NodeId, - pub dial_info: DialInfo, -} - -impl fmt::Display for NodeDialInfo { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!(f, "{}@{}", self.node_id, self.dial_info) - } -} - -impl FromStr for NodeDialInfo { - type Err = VeilidAPIError; - fn from_str(s: &str) -> Result { - // split out node id from the dial info - let (node_id_str, rest) = s.split_once('@').ok_or_else(|| { - VeilidAPIError::parse_error("NodeDialInfo::from_str missing @ node id separator", s) - })?; - - // parse out node id - let node_id = NodeId::new(DHTKey::try_decode(node_id_str).map_err(|e| { - VeilidAPIError::parse_error( - format!("NodeDialInfo::from_str couldn't parse node id: {}", e), - s, - ) - })?); - // parse out dial info - let dial_info = DialInfo::from_str(rest)?; - - // return completed NodeDialInfo - Ok(NodeDialInfo { node_id, dial_info }) - } -} - #[derive( Clone, Debug, diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index d02d7d77..4c6b4135 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -307,16 +307,16 @@ impl serde::Serialize for ParsedUrl { #[derive(Debug, Clone, PartialEq)] pub struct ParsedNodeDialInfo { pub node_dial_info_string: String, - pub node_dial_info: veilid_core::NodeDialInfo, + pub node_id: NodeId, + pub dial_info: DialInfo, } // impl ParsedNodeDialInfo { // pub fn offset_port(&mut self, offset: u16) -> Result<(), ()> { // // Bump port on dial_info -// self.node_dial_info -// .dial_info -// .set_port(self.node_dial_info.dial_info.port() + 1); -// self.node_dial_info_string = self.node_dial_info.to_string(); +// self.dial_info +// .set_port(self.dial_info.port() + 1); +// self.node_dial_info_string = format!("{}@{}",self.node_id, self.dial_info); // Ok(()) // } // } @@ -326,10 +326,21 @@ impl FromStr for ParsedNodeDialInfo { fn from_str( node_dial_info_string: &str, ) -> Result { - let node_dial_info = veilid_core::NodeDialInfo::from_str(node_dial_info_string)?; + let (id_str, di_str) = node_dial_info_string.split_once('@').ok_or_else(|| { + VeilidAPIError::invalid_argument( + "Invalid node dial info in bootstrap entry", + "node_dial_info_string", + node_dial_info_string, + ) + })?; + let node_id = NodeId::from_str(id_str) + .map_err(|e| VeilidAPIError::invalid_argument(e, "node_id", id_str))?; + let dial_info = DialInfo::from_str(di_str) + .map_err(|e| VeilidAPIError::invalid_argument(e, "dial_info", id_str))?; Ok(Self { node_dial_info_string: node_dial_info_string.to_owned(), - node_dial_info, + node_id, + dial_info, }) } } From 662ed03d19e29fbbd0adc05ebfc1c06682188fac Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 12 Nov 2022 22:20:31 -0500 Subject: [PATCH 53/67] capnp ids --- veilid-core/proto/veilid.capnp | 167 +++++++++--------- .../src/rpc_processor/coders/block_id.rs | 38 ---- .../coders/{public_key.rs => dht_key.rs} | 6 +- .../coders/{signature.rs => dht_signature.rs} | 7 +- veilid-core/src/rpc_processor/coders/mod.rs | 10 +- veilid-core/src/rpc_processor/coders/nonce.rs | 7 +- .../coders/operations/operation_find_block.rs | 4 +- .../coders/operations/operation_find_node.rs | 4 +- .../operations/operation_supply_block.rs | 4 +- .../src/rpc_processor/coders/peer_info.rs | 4 +- .../coders/private_safety_route.rs | 16 +- .../coders/signed_relayed_node_info.rs | 4 +- .../src/rpc_processor/coders/value_key.rs | 4 +- veilid-core/src/veilid_api/mod.rs | 4 +- veilid-server/proto/veilid-client.capnp | 8 +- 15 files changed, 117 insertions(+), 170 deletions(-) delete mode 100644 veilid-core/src/rpc_processor/coders/block_id.rs rename veilid-core/src/rpc_processor/coders/{public_key.rs => dht_key.rs} (87%) rename veilid-core/src/rpc_processor/coders/{signature.rs => dht_signature.rs} (90%) diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 3525da9f..1ce4c11a 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -3,14 +3,14 @@ # IDs And Hashes ############################## -struct Curve25519PublicKey { +struct Key256 @0xdde44e3286f6a90d { u0 @0 :UInt64; u1 @1 :UInt64; u2 @2 :UInt64; u3 @3 :UInt64; } -struct Ed25519Signature { +struct Signature512 @0x806749043a129c12 { u0 @0 :UInt64; u1 @1 :UInt64; u2 @2 :UInt64; @@ -21,79 +21,72 @@ struct Ed25519Signature { u7 @7 :UInt64; } -struct XChaCha20Poly1305Nonce { +struct Nonce24 @0xb6260db25d8d7dfc { u0 @0 :UInt64; u1 @1 :UInt64; u2 @2 :UInt64; } -struct BLAKE3Hash { - u0 @0 :UInt64; - u1 @1 :UInt64; - u2 @2 :UInt64; - u3 @3 :UInt64; -} - -using NodeID = Curve25519PublicKey; -using RoutePublicKey = Curve25519PublicKey; -using ValueID = Curve25519PublicKey; -using Nonce = XChaCha20Poly1305Nonce; -using Signature = Ed25519Signature; -using BlockID = BLAKE3Hash; +using NodeID = Key256; +using RoutePublicKey = Key256; +using ValueID = Key256; +using Nonce = Nonce24; +using Signature = Signature512; +using BlockID = Key256; using TunnelID = UInt64; # Node Dial Info ################################################################ -struct AddressIPV4 { +struct AddressIPV4 @0xdb8769881266a6a0 { addr @0 :UInt32; # Address in big endian format } -struct AddressIPV6 { +struct AddressIPV6 @0xb35d6e6011dc5c20 { addr0 @0 :UInt32; # \ addr1 @1 :UInt32; # \ Address in big addr2 @2 :UInt32; # / endian format addr3 @3 :UInt32; # / } -struct Address { +struct Address @0x812706e9e57d108b { union { ipv4 @0 :AddressIPV4; ipv6 @1 :AddressIPV6; } } -struct SocketAddress { +struct SocketAddress @0x82df4272f4dd3a62 { address @0 :Address; port @1 :UInt16; } -enum ProtocolKind { +enum ProtocolKind @0xde0bf5787c067d5a { udp @0; ws @1; wss @2; tcp @3; } -struct DialInfoUDP { +struct DialInfoUDP @0xbb38a8b8b7024a7c { socketAddress @0 :SocketAddress; } -struct DialInfoTCP { +struct DialInfoTCP @0x9e0a9371b9a9f7fc { socketAddress @0 :SocketAddress; } -struct DialInfoWS { +struct DialInfoWS @0xd7795f7a92ab15b0 { socketAddress @0 :SocketAddress; request @1 :Text; } -struct DialInfoWSS { +struct DialInfoWSS @0xe639faa41b7d7b04 { socketAddress @0 :SocketAddress; request @1 :Text; } -struct DialInfo { +struct DialInfo @0xe1cd1c39fc2defdf { union { udp @0 :DialInfoUDP; tcp @1 :DialInfoTCP; @@ -105,12 +98,12 @@ struct DialInfo { # Signals ############################## -struct SignalInfoHolePunch { +struct SignalInfoHolePunch @0xeeb9ab6861890c9a { receipt @0 :Data; # receipt to return with hole punch peerInfo @1 :PeerInfo; # peer info of the signal sender for hole punch attempt } -struct SignalInfoReverseConnect { +struct SignalInfoReverseConnect @0xd9ebd3bd0d46e013 { receipt @0 :Data; # receipt to return with reverse connect peerInfo @1 :PeerInfo; # peer info of the signal sender for reverse connect attempt } @@ -118,7 +111,7 @@ struct SignalInfoReverseConnect { # Private Routes ############################## -struct RouteHopData { +struct RouteHopData @0x8ce231f9d1b7adf2 { nonce @0 :Nonce; # nonce for encrypted blob blob @1 :Data; # encrypted blob with ENC(nonce,DH(PK,SK)) # if this is a safety route RouteHopData, there is a single byte tag appended to the end of the encrypted blob @@ -128,7 +121,7 @@ struct RouteHopData { # if this is a private route RouteHopData, only can decode to RouteHop, no tag is appended } -struct RouteHop { +struct RouteHop @0xf8f672d75cce0c3b { node :union { nodeId @0 :NodeID; # node id only for established routes peerInfo @1 :PeerInfo; # full peer info for this hop to establish the route @@ -137,13 +130,13 @@ struct RouteHop { # if this is a safety route routehop, this field is not optional and must exist } -struct PrivateRoute { +struct PrivateRoute @0x8a83fccb0851e776 { publicKey @0 :RoutePublicKey; # private route public key (unique per private route) hopCount @1 :UInt8; # Count of hops left in the private route (for timeout calculation purposes only) firstHop @2 :RouteHop; # Optional: first hop in the private route, if empty, this is the last hop and payload should be decrypted and processed. } -struct SafetyRoute { +struct SafetyRoute @0xf554734d07cb5d59 { publicKey @0 :RoutePublicKey; # safety route public key (unique per safety route) hopCount @1 :UInt8; # Count of hops left in the safety route (for timeout calculation purposes only) hops :union { @@ -157,7 +150,7 @@ struct SafetyRoute { using ValueSeqNum = UInt32; # sequence numbers for values -struct ValueKey { +struct ValueKey @0xe64b0992c21a0736 { publicKey @0 :ValueID; # the location of the value subkey @1 :Text; # the name of the subkey (or empty if the whole key) } @@ -167,7 +160,7 @@ struct ValueKey { # seq @1 :ValueSeqNum; # the sequence number of the value subkey # } -struct ValueData { +struct ValueData @0xb4b7416f169f2a3d { data @0 :Data; # value or subvalue contents seq @1 :ValueSeqNum; # sequence number of value } @@ -175,14 +168,14 @@ struct ValueData { # Operations ############################## -enum NetworkClass { +enum NetworkClass @0x8cebfc2a6230717f { invalid @0; # X = Invalid network class, network is not yet set up inboundCapable @1; # I = Inbound capable without relay, may require signal outboundOnly @2; # O = Outbound only, inbound relay required except with reverse connect signal webApp @3; # W = PWA, outbound relay is required in most cases } -enum DialInfoClass { +enum DialInfoClass @0x880005edfdd38b1e { direct @0; # D = Directly reachable with public IP and no firewall, with statically configured port mapped @1; # M = Directly reachable with via portmap behind any NAT or firewalled with dynamically negotiated port fullConeNAT @2; # F = Directly reachable device without portmap behind full-cone NAT @@ -191,12 +184,12 @@ enum DialInfoClass { portRestrictedNAT @5; # P = Device without portmap behind address-and-port restricted NAT } -struct DialInfoDetail { +struct DialInfoDetail @0x96423aa1d67b74d8 { dialInfo @0 :DialInfo; class @1 :DialInfoClass; } -struct PublicInternetNodeStatus { +struct PublicInternetNodeStatus @0x9c9d7f1f12eb088f { willRoute @0 :Bool; willTunnel @1 :Bool; willSignal @2 :Bool; @@ -204,35 +197,35 @@ struct PublicInternetNodeStatus { willValidateDialInfo @4 :Bool; } -struct LocalNetworkNodeStatus { +struct LocalNetworkNodeStatus @0x957f5bfed2d0b5a5 { willRelay @0 :Bool; willValidateDialInfo @1 :Bool; } -struct NodeStatus { +struct NodeStatus @0xd36b9e7a3bf3330d { union { publicInternet @0 :PublicInternetNodeStatus; localNetwork @1 :LocalNetworkNodeStatus; } } -struct ProtocolTypeSet { +struct ProtocolTypeSet @0x82f12f55a1b73326 { udp @0 :Bool; tcp @1 :Bool; ws @2 :Bool; wss @3 :Bool; } -struct AddressTypeSet { +struct AddressTypeSet @0x9f52d5430d349e6b { ipv4 @0 :Bool; ipv6 @1 :Bool; } -struct SenderInfo { +struct SenderInfo @0x8a4464fab4b1d101 { socketAddress @0 :SocketAddress; # socket address that for the sending peer } -struct NodeInfo { +struct NodeInfo @0xe125d847e3f9f419 { networkClass @0 :NetworkClass; # network class of this node outboundProtocols @1 :ProtocolTypeSet; # protocols that can go outbound addressTypes @2 :AddressTypeSet; # address types supported @@ -241,13 +234,13 @@ struct NodeInfo { dialInfoDetailList @5 :List(DialInfoDetail); # inbound dial info details for this node } -struct SignedDirectNodeInfo { +struct SignedDirectNodeInfo @0xe0e7ea3e893a3dd7 { nodeInfo @0 :NodeInfo; # node info timestamp @1 :UInt64; # when signed node info was generated signature @2 :Signature; # signature } -struct SignedRelayedNodeInfo { +struct SignedRelayedNodeInfo @0xb39e8428ccd87cbb { nodeInfo @0 :NodeInfo; # node info relayId @1 :NodeID; # node id for relay relayInfo @2 :SignedDirectNodeInfo; # signed node info for relay @@ -255,202 +248,202 @@ struct SignedRelayedNodeInfo { signature @4 :Signature; # signature } -struct SignedNodeInfo { +struct SignedNodeInfo @0xd2478ce5f593406a { union { direct @0 :SignedDirectNodeInfo; # node info for nodes reachable without a relay relayed @1 :SignedRelayedNodeInfo; # node info for nodes requiring a relay } } -struct PeerInfo { +struct PeerInfo @0xfe2d722d5d3c4bcb { nodeId @0 :NodeID; # node id for 'closer peer' signedNodeInfo @1 :SignedNodeInfo; # signed node info for 'closer peer' } -struct RoutedOperation { +struct RoutedOperation @0xcbcb8535b839e9dd { version @0 :UInt8; # crypto version in use for the data signatures @1 :List(Signature); # signatures from nodes that have handled the private route nonce @2 :Nonce; # nonce Xmsg data @3 :Data; # operation encrypted with ENC(Xmsg,DH(PKapr,SKbsr)) } -struct OperationStatusQ { +struct OperationStatusQ @0x865d80cea70d884a { nodeStatus @0 :NodeStatus; # Optional: node status update about the statusq sender } -struct OperationStatusA { +struct OperationStatusA @0xb306f407fa812a55 { nodeStatus @0 :NodeStatus; # Optional: returned node status senderInfo @1 :SenderInfo; # Optional: info about StatusQ sender from the perspective of the replier } -struct OperationValidateDialInfo { +struct OperationValidateDialInfo @0xbc716ad7d5d060c8 { dialInfo @0 :DialInfo; # dial info to use for the receipt receipt @1 :Data; # receipt to return to dial info to prove it is reachable redirect @2 :Bool; # request a different node do the validate } -struct OperationReturnReceipt { +struct OperationReturnReceipt @0xeb0fb5b5a9160eeb { receipt @0 :Data; # receipt being returned to its origin } -struct OperationFindNodeQ { +struct OperationFindNodeQ @0xfdef788fe9623bcd { nodeId @0 :NodeID; # node id to locate } -struct OperationFindNodeA { +struct OperationFindNodeA @0xa84cf2fb40c77089 { peers @0 :List(PeerInfo); # returned 'closer peer' information } -struct OperationRoute { +struct OperationRoute @0x96741859ce6ac7dd { safetyRoute @0 :SafetyRoute; # Where this should go operation @1 :RoutedOperation; # The operation to be routed } -struct OperationNodeInfoUpdate { +struct OperationNodeInfoUpdate @0xc9647b32a48b66ce { signedNodeInfo @0 :SignedNodeInfo; # Our signed node info } -struct OperationAppCallQ { +struct OperationAppCallQ @0xade67b9f09784507 { message @0 :Data; # Opaque request to application } -struct OperationAppCallA { +struct OperationAppCallA @0xf7c797ac85f214b8 { message @0 :Data; # Opaque response from application } -struct OperationAppMessage { +struct OperationAppMessage @0x9baf542d81b411f5 { message @0 :Data; # Opaque message to application } -struct OperationGetValueQ { +struct OperationGetValueQ @0xf88a5b6da5eda5d0 { key @0 :ValueKey; # key for value to get } -struct OperationGetValueA { +struct OperationGetValueA @0xd896bb46f2e0249f { union { data @0 :ValueData; # the value if successful peers @1 :List(PeerInfo); # returned 'closer peer' information if not successful } } -struct OperationSetValueQ { +struct OperationSetValueQ @0xbac06191ff8bdbc5 { key @0 :ValueKey; # key for value to update value @1 :ValueData; # value or subvalue contents (older or equal seq number gets dropped) } -struct OperationSetValueA { +struct OperationSetValueA @0x9378d0732dc95be2 { union { data @0 :ValueData; # the new value if successful, may be a different value than what was set if the seq number was lower or equal peers @1 :List(PeerInfo); # returned 'closer peer' information if not successful } } -struct OperationWatchValueQ { +struct OperationWatchValueQ @0xf9a5a6c547b9b228 { key @0 :ValueKey; # key for value to watch } -struct OperationWatchValueA { +struct OperationWatchValueA @0xa726cab7064ba893 { expiration @0 :UInt64; # timestamp when this watch will expire in usec since epoch (0 if watch failed) peers @1 :List(PeerInfo); # returned list of other nodes to ask that could propagate watches } -struct OperationValueChanged { +struct OperationValueChanged @0xd1c59ebdd8cc1bf6 { key @0 :ValueKey; # key for value that changed value @1 :ValueData; # value or subvalue contents with sequence number } -struct OperationSupplyBlockQ { +struct OperationSupplyBlockQ @0xadbf4c542d749971 { blockId @0 :BlockID; # hash of the block we can supply } -struct OperationSupplyBlockA { +struct OperationSupplyBlockA @0xf003822e83b5c0d7 { union { expiration @0 :UInt64; # when the block supplier entry will need to be refreshed peers @1 :List(PeerInfo); # returned 'closer peer' information if not successful } } -struct OperationFindBlockQ { +struct OperationFindBlockQ @0xaf4353ff004c7156 { blockId @0 :BlockID; # hash of the block to locate } -struct OperationFindBlockA { +struct OperationFindBlockA @0xc51455bc4915465d { data @0 :Data; # Optional: the actual block data if we have that block ourselves # null if we don't have a block to return suppliers @1 :List(PeerInfo); # returned list of suppliers if we have them peers @2 :List(PeerInfo); # returned 'closer peer' information } -struct OperationSignal { +struct OperationSignal @0xd4f94f2a5d207e49 { union { holePunch @0 :SignalInfoHolePunch; reverseConnect @1 :SignalInfoReverseConnect; } } -enum TunnelEndpointMode { +enum TunnelEndpointMode @0xef06f4c29beb7458 { raw @0; # raw tunnel turn @1; # turn tunnel } -enum TunnelError { +enum TunnelError @0xb82c6bfb1ec38c7c { badId @0; # Tunnel ID was rejected noEndpoint @1; # Endpoint was unreachable rejectedMode @2; # Endpoint couldn't provide mode noCapacity @3; # Endpoint is full } -struct TunnelEndpoint { +struct TunnelEndpoint @0xc2602aa983cc337d { mode @0 :TunnelEndpointMode; # what kind of endpoint this is description @1 :Text; # endpoint description (TODO) } -struct FullTunnel { +struct FullTunnel @0x9821c3dc75373f63 { id @0 :TunnelID; # tunnel id to use everywhere timeout @1 :UInt64; # duration from last data when this expires if no data is sent or received local @2 :TunnelEndpoint; # local endpoint remote @3 :TunnelEndpoint; # remote endpoint } -struct PartialTunnel { +struct PartialTunnel @0x827a7ebc02be2fc8 { id @0 :TunnelID; # tunnel id to use everywhere timeout @1 :UInt64; # timestamp when this expires if not completed local @2 :TunnelEndpoint; # local endpoint } -struct OperationStartTunnelQ { +struct OperationStartTunnelQ @0xa9c49afce44187af { id @0 :TunnelID; # tunnel id to use everywhere localMode @1 :TunnelEndpointMode; # what kind of local endpoint mode is being requested depth @2 :UInt8; # the number of nodes in the tunnel } -struct OperationStartTunnelA { +struct OperationStartTunnelA @0x818162e4cc61bf1e { union { partial @0 :PartialTunnel; # the first half of the tunnel error @1 :TunnelError; # if we didn't start the tunnel, why not } } -struct OperationCompleteTunnelQ { +struct OperationCompleteTunnelQ @0xe978594588eb950b { id @0 :TunnelID; # tunnel id to use everywhere localMode @1 :TunnelEndpointMode; # what kind of local endpoint mode is being requested depth @2 :UInt8; # the number of nodes in the tunnel endpoint @3 :TunnelEndpoint; # the remote endpoint to complete } -struct OperationCompleteTunnelA { +struct OperationCompleteTunnelA @0x84090791bb765f2a { union { tunnel @0 :FullTunnel; # the tunnel description error @1 :TunnelError; # if we didn't complete the tunnel, why not } } -struct OperationCancelTunnelQ { +struct OperationCancelTunnelQ @0xae2811ae0a003738 { id @0 :TunnelID; # the tunnel id to cancel } -struct OperationCancelTunnelA { +struct OperationCancelTunnelA @0xbba23c992eff97bc { union { tunnel @0 :TunnelID; # the tunnel id that was cancelled error @1 :TunnelError; # if we couldn't cancel, why not @@ -458,7 +451,7 @@ struct OperationCancelTunnelA { } # Things that want an answer -struct Question { +struct Question @0xd8510bc33492ef70 { respondTo :union { sender @0 :Void; # sender privateRoute @1 :PrivateRoute; # embedded private route to be used for reply @@ -484,7 +477,7 @@ struct Question { } # Things that don't want an answer -struct Statement { +struct Statement @0x990e20828f404ae1 { detail :union { # Direct operations validateDialInfo @0 :OperationValidateDialInfo; @@ -500,7 +493,7 @@ struct Statement { } # Things that are answers -struct Answer { +struct Answer @0xacacb8b6988c1058 { detail :union { # Direct operations statusA @0 :OperationStatusA; @@ -521,7 +514,7 @@ struct Answer { } } -struct Operation { +struct Operation @0xbf2811c435403c3b { opId @0 :UInt64; # Random RPC ID. Must be random to foil reply forgery attacks. senderNodeInfo @1 :SignedNodeInfo; # (optional) SignedNodeInfo for the sender to be cached by the receiver. kind :union { diff --git a/veilid-core/src/rpc_processor/coders/block_id.rs b/veilid-core/src/rpc_processor/coders/block_id.rs deleted file mode 100644 index c42481b6..00000000 --- a/veilid-core/src/rpc_processor/coders/block_id.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::crypto::*; -use crate::*; -use core::convert::TryInto; -use rpc_processor::*; - -pub fn decode_block_id(public_key: &veilid_capnp::b_l_a_k_e3_hash::Reader) -> DHTKey { - let u0 = public_key.get_u0().to_be_bytes(); - let u1 = public_key.get_u1().to_be_bytes(); - let u2 = public_key.get_u2().to_be_bytes(); - let u3 = public_key.get_u3().to_be_bytes(); - - let mut x: [u8; 32] = Default::default(); - x[0..8].copy_from_slice(&u0); - x[8..16].copy_from_slice(&u1); - x[16..24].copy_from_slice(&u2); - x[24..32].copy_from_slice(&u3); - - DHTKey::new(x) -} - -pub fn encode_block_id( - key: &DHTKey, - builder: &mut veilid_capnp::b_l_a_k_e3_hash::Builder, -) -> Result<(), RPCError> { - builder.set_u0(u64::from_be_bytes( - key.bytes[0..8].try_into().map_err(RPCError::internal)?, - )); - builder.set_u1(u64::from_be_bytes( - key.bytes[8..16].try_into().map_err(RPCError::internal)?, - )); - builder.set_u2(u64::from_be_bytes( - key.bytes[16..24].try_into().map_err(RPCError::internal)?, - )); - builder.set_u3(u64::from_be_bytes( - key.bytes[24..32].try_into().map_err(RPCError::internal)?, - )); - Ok(()) -} diff --git a/veilid-core/src/rpc_processor/coders/public_key.rs b/veilid-core/src/rpc_processor/coders/dht_key.rs similarity index 87% rename from veilid-core/src/rpc_processor/coders/public_key.rs rename to veilid-core/src/rpc_processor/coders/dht_key.rs index 1bec140b..dd7a8909 100644 --- a/veilid-core/src/rpc_processor/coders/public_key.rs +++ b/veilid-core/src/rpc_processor/coders/dht_key.rs @@ -3,7 +3,7 @@ use crate::*; use core::convert::TryInto; use rpc_processor::*; -pub fn decode_public_key(public_key: &veilid_capnp::curve25519_public_key::Reader) -> DHTKey { +pub fn decode_dht_key(public_key: &veilid_capnp::key256::Reader) -> DHTKey { let u0 = public_key.get_u0().to_be_bytes(); let u1 = public_key.get_u1().to_be_bytes(); let u2 = public_key.get_u2().to_be_bytes(); @@ -18,9 +18,9 @@ pub fn decode_public_key(public_key: &veilid_capnp::curve25519_public_key::Reade DHTKey::new(x) } -pub fn encode_public_key( +pub fn encode_dht_key( key: &DHTKey, - builder: &mut veilid_capnp::curve25519_public_key::Builder, + builder: &mut veilid_capnp::key256::Builder, ) -> Result<(), RPCError> { builder.set_u0(u64::from_be_bytes( key.bytes[0..8] diff --git a/veilid-core/src/rpc_processor/coders/signature.rs b/veilid-core/src/rpc_processor/coders/dht_signature.rs similarity index 90% rename from veilid-core/src/rpc_processor/coders/signature.rs rename to veilid-core/src/rpc_processor/coders/dht_signature.rs index 956fb74a..9e008faa 100644 --- a/veilid-core/src/rpc_processor/coders/signature.rs +++ b/veilid-core/src/rpc_processor/coders/dht_signature.rs @@ -1,10 +1,7 @@ use crate::*; use rpc_processor::*; -pub fn encode_signature( - sig: &DHTSignature, - builder: &mut veilid_capnp::ed25519_signature::Builder, -) { +pub fn encode_signature(sig: &DHTSignature, builder: &mut veilid_capnp::signature512::Builder) { let sig = &sig.bytes; builder.set_u0(u64::from_be_bytes( @@ -33,7 +30,7 @@ pub fn encode_signature( )); } -pub fn decode_signature(reader: &veilid_capnp::ed25519_signature::Reader) -> DHTSignature { +pub fn decode_signature(reader: &veilid_capnp::signature512::Reader) -> DHTSignature { let u0 = reader.get_u0().to_be_bytes(); let u1 = reader.get_u1().to_be_bytes(); let u2 = reader.get_u2().to_be_bytes(); diff --git a/veilid-core/src/rpc_processor/coders/mod.rs b/veilid-core/src/rpc_processor/coders/mod.rs index ccc6dc17..b5c3c709 100644 --- a/veilid-core/src/rpc_processor/coders/mod.rs +++ b/veilid-core/src/rpc_processor/coders/mod.rs @@ -1,6 +1,7 @@ mod address; mod address_type_set; -mod block_id; +mod dht_key; +mod dht_signature; mod dial_info; mod dial_info_class; mod dial_info_detail; @@ -12,10 +13,8 @@ mod operations; mod peer_info; mod private_safety_route; mod protocol_type_set; -mod public_key; mod sender_info; mod signal_info; -mod signature; mod signed_direct_node_info; mod signed_node_info; mod signed_relayed_node_info; @@ -26,7 +25,8 @@ mod value_key; pub use address::*; pub use address_type_set::*; -pub use block_id::*; +pub use dht_key::*; +pub use dht_signature::*; pub use dial_info::*; pub use dial_info_class::*; pub use dial_info_detail::*; @@ -38,10 +38,8 @@ pub use operations::*; pub use peer_info::*; pub use private_safety_route::*; pub use protocol_type_set::*; -pub use public_key::*; pub use sender_info::*; pub use signal_info::*; -pub use signature::*; pub use signed_direct_node_info::*; pub use signed_node_info::*; pub use signed_relayed_node_info::*; diff --git a/veilid-core/src/rpc_processor/coders/nonce.rs b/veilid-core/src/rpc_processor/coders/nonce.rs index a73bd56a..5eb39dce 100644 --- a/veilid-core/src/rpc_processor/coders/nonce.rs +++ b/veilid-core/src/rpc_processor/coders/nonce.rs @@ -1,10 +1,7 @@ use crate::*; use rpc_processor::*; -pub fn encode_nonce( - nonce: &Nonce, - builder: &mut veilid_capnp::x_cha_cha20_poly1305_nonce::Builder, -) { +pub fn encode_nonce(nonce: &Nonce, builder: &mut veilid_capnp::nonce24::Builder) { builder.set_u0(u64::from_be_bytes( nonce[0..8].try_into().expect("slice with incorrect length"), )); @@ -20,7 +17,7 @@ pub fn encode_nonce( )); } -pub fn decode_nonce(reader: &veilid_capnp::x_cha_cha20_poly1305_nonce::Reader) -> Nonce { +pub fn decode_nonce(reader: &veilid_capnp::nonce24::Reader) -> Nonce { let u0 = reader.get_u0().to_be_bytes(); let u1 = reader.get_u1().to_be_bytes(); let u2 = reader.get_u2().to_be_bytes(); diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs b/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs index 71fcba11..5503ad31 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_find_block.rs @@ -11,7 +11,7 @@ impl RPCOperationFindBlockQ { reader: &veilid_capnp::operation_find_block_q::Reader, ) -> Result { let bi_reader = reader.get_block_id().map_err(RPCError::protocol)?; - let block_id = decode_block_id(&bi_reader); + let block_id = decode_dht_key(&bi_reader); Ok(RPCOperationFindBlockQ { block_id }) } @@ -20,7 +20,7 @@ impl RPCOperationFindBlockQ { builder: &mut veilid_capnp::operation_find_block_q::Builder, ) -> Result<(), RPCError> { let mut bi_builder = builder.reborrow().init_block_id(); - encode_block_id(&self.block_id, &mut bi_builder)?; + encode_dht_key(&self.block_id, &mut bi_builder)?; Ok(()) } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs b/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs index bfd0ded9..95ca3ea5 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_find_node.rs @@ -11,7 +11,7 @@ impl RPCOperationFindNodeQ { reader: &veilid_capnp::operation_find_node_q::Reader, ) -> Result { let ni_reader = reader.get_node_id().map_err(RPCError::protocol)?; - let node_id = decode_public_key(&ni_reader); + let node_id = decode_dht_key(&ni_reader); Ok(RPCOperationFindNodeQ { node_id }) } pub fn encode( @@ -19,7 +19,7 @@ impl RPCOperationFindNodeQ { builder: &mut veilid_capnp::operation_find_node_q::Builder, ) -> Result<(), RPCError> { let mut ni_builder = builder.reborrow().init_node_id(); - encode_public_key(&self.node_id, &mut ni_builder)?; + encode_dht_key(&self.node_id, &mut ni_builder)?; Ok(()) } } diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs b/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs index b094eaaf..67b7ab00 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_supply_block.rs @@ -11,7 +11,7 @@ impl RPCOperationSupplyBlockQ { reader: &veilid_capnp::operation_supply_block_q::Reader, ) -> Result { let bi_reader = reader.get_block_id().map_err(RPCError::protocol)?; - let block_id = decode_block_id(&bi_reader); + let block_id = decode_dht_key(&bi_reader); Ok(RPCOperationSupplyBlockQ { block_id }) } @@ -20,7 +20,7 @@ impl RPCOperationSupplyBlockQ { builder: &mut veilid_capnp::operation_supply_block_q::Builder, ) -> Result<(), RPCError> { let mut bi_builder = builder.reborrow().init_block_id(); - encode_block_id(&self.block_id, &mut bi_builder)?; + encode_dht_key(&self.block_id, &mut bi_builder)?; Ok(()) } diff --git a/veilid-core/src/rpc_processor/coders/peer_info.rs b/veilid-core/src/rpc_processor/coders/peer_info.rs index 428980f4..5c7e67ab 100644 --- a/veilid-core/src/rpc_processor/coders/peer_info.rs +++ b/veilid-core/src/rpc_processor/coders/peer_info.rs @@ -7,7 +7,7 @@ pub fn encode_peer_info( ) -> Result<(), RPCError> { // let mut nid_builder = builder.reborrow().init_node_id(); - encode_public_key(&peer_info.node_id.key, &mut nid_builder)?; + encode_dht_key(&peer_info.node_id.key, &mut nid_builder)?; let mut sni_builder = builder.reborrow().init_signed_node_info(); encode_signed_node_info(&peer_info.signed_node_info, &mut sni_builder)?; @@ -23,7 +23,7 @@ pub fn decode_peer_info(reader: &veilid_capnp::peer_info::Reader) -> Result { let mut ni_builder = node_builder.init_node_id(); - encode_public_key(&ni.key, &mut ni_builder)?; + encode_dht_key(&ni.key, &mut ni_builder)?; } RouteNode::PeerInfo(pi) => { let mut pi_builder = node_builder.init_peer_info(); @@ -72,7 +72,7 @@ pub fn decode_route_hop(reader: &veilid_capnp::route_hop::Reader) -> Result { let ni_reader = ni.map_err(RPCError::protocol)?; - RouteNode::NodeId(NodeId::new(decode_public_key(&ni_reader))) + RouteNode::NodeId(NodeId::new(decode_dht_key(&ni_reader))) } veilid_capnp::route_hop::node::Which::PeerInfo(pi) => { let pi_reader = pi.map_err(RPCError::protocol)?; @@ -101,7 +101,7 @@ pub fn encode_private_route( private_route: &PrivateRoute, builder: &mut veilid_capnp::private_route::Builder, ) -> Result<(), RPCError> { - encode_public_key( + encode_dht_key( &private_route.public_key, &mut builder.reborrow().init_public_key(), )?; @@ -117,9 +117,9 @@ pub fn encode_private_route( pub fn decode_private_route( reader: &veilid_capnp::private_route::Reader, ) -> Result { - let public_key = decode_public_key(&reader.get_public_key().map_err( - RPCError::map_protocol("invalid public key in private route"), - )?); + let public_key = decode_dht_key(&reader.get_public_key().map_err(RPCError::map_protocol( + "invalid public key in private route", + ))?); let hop_count = reader.get_hop_count(); let first_hop = if reader.has_first_hop() { let rh_reader = reader @@ -143,7 +143,7 @@ pub fn encode_safety_route( safety_route: &SafetyRoute, builder: &mut veilid_capnp::safety_route::Builder, ) -> Result<(), RPCError> { - encode_public_key( + encode_dht_key( &safety_route.public_key, &mut builder.reborrow().init_public_key(), )?; @@ -166,7 +166,7 @@ pub fn encode_safety_route( pub fn decode_safety_route( reader: &veilid_capnp::safety_route::Reader, ) -> Result { - let public_key = decode_public_key( + let public_key = decode_dht_key( &reader .get_public_key() .map_err(RPCError::map_protocol("invalid public key in safety route"))?, diff --git a/veilid-core/src/rpc_processor/coders/signed_relayed_node_info.rs b/veilid-core/src/rpc_processor/coders/signed_relayed_node_info.rs index 530f8254..646f6597 100644 --- a/veilid-core/src/rpc_processor/coders/signed_relayed_node_info.rs +++ b/veilid-core/src/rpc_processor/coders/signed_relayed_node_info.rs @@ -10,7 +10,7 @@ pub fn encode_signed_relayed_node_info( encode_node_info(&signed_relayed_node_info.node_info, &mut ni_builder)?; let mut rid_builder = builder.reborrow().init_relay_id(); - encode_public_key(&signed_relayed_node_info.relay_id.key, &mut rid_builder)?; + encode_dht_key(&signed_relayed_node_info.relay_id.key, &mut rid_builder)?; let mut ri_builder = builder.reborrow().init_relay_info(); encode_signed_direct_node_info(&signed_relayed_node_info.relay_info, &mut ri_builder)?; @@ -39,7 +39,7 @@ pub fn decode_signed_relayed_node_info( .reborrow() .get_relay_id() .map_err(RPCError::protocol)?; - let relay_id = decode_public_key(&rid_reader); + let relay_id = decode_dht_key(&rid_reader); let ri_reader = reader .reborrow() diff --git a/veilid-core/src/rpc_processor/coders/value_key.rs b/veilid-core/src/rpc_processor/coders/value_key.rs index 035ab998..7561b3c6 100644 --- a/veilid-core/src/rpc_processor/coders/value_key.rs +++ b/veilid-core/src/rpc_processor/coders/value_key.rs @@ -6,7 +6,7 @@ pub fn encode_value_key( builder: &mut veilid_capnp::value_key::Builder, ) -> Result<(), RPCError> { let mut pk_builder = builder.reborrow().init_public_key(); - encode_public_key(&value_key.key, &mut pk_builder)?; + encode_dht_key(&value_key.key, &mut pk_builder)?; if let Some(subkey) = &value_key.subkey { builder.set_subkey(subkey); } @@ -15,7 +15,7 @@ pub fn encode_value_key( pub fn decode_value_key(reader: &veilid_capnp::value_key::Reader) -> Result { let pk_reader = reader.get_public_key().map_err(RPCError::protocol)?; - let key = decode_public_key(&pk_reader); + let key = decode_dht_key(&pk_reader); let subkey = if !reader.has_subkey() { None } else { diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 69348803..9f4994c5 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -2099,8 +2099,8 @@ impl SignedRelayedNodeInfo { // Add relay id to signature let mut rid_msg = ::capnp::message::Builder::new_default(); - let mut rid_builder = rid_msg.init_root::(); - encode_public_key(&relay_id.key, &mut rid_builder).map_err(VeilidAPIError::internal)?; + let mut rid_builder = rid_msg.init_root::(); + encode_dht_key(&relay_id.key, &mut rid_builder).map_err(VeilidAPIError::internal)?; sig_bytes.append(&mut builder_to_vec(rid_msg).map_err(VeilidAPIError::internal)?); // Add relay info to signature diff --git a/veilid-server/proto/veilid-client.capnp b/veilid-server/proto/veilid-client.capnp index 320ffea4..e12b587a 100644 --- a/veilid-server/proto/veilid-client.capnp +++ b/veilid-server/proto/veilid-client.capnp @@ -1,15 +1,15 @@ @0xd29582d26b2fb073; -struct ApiResult { +struct ApiResult @0x8111724bdb812929 { union { ok @0 :Text; err @1 :Text; } } -interface Registration {} +interface Registration @0xdd45f30a7c22e391 {} -interface VeilidServer { +interface VeilidServer @0xcb2c699f14537f94 { register @0 (veilidClient :VeilidClient) -> (registration :Registration, state :Text); debug @1 (command :Text) -> (result :ApiResult); attach @2 () -> (result :ApiResult); @@ -20,6 +20,6 @@ interface VeilidServer { appCallReply @7 (id :UInt64, message :Data) -> (result :ApiResult); } -interface VeilidClient { +interface VeilidClient @0xbfcea60fb2ba4736 { update @0 (veilidUpdate :Text); } \ No newline at end of file From fd26acec16405b5cd36af5bf3544393ab5808779 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 13 Nov 2022 11:56:27 -0500 Subject: [PATCH 54/67] xfer --- veilid-core/src/rpc_processor/rpc_route.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 56848b18..a6031441 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -73,7 +73,7 @@ impl RPCProcessor { async fn process_route_private_route_hop( &self, mut route: RPCOperationRoute, - next_private_route: PrivateRoute, + mut next_private_route: PrivateRoute, ) -> Result<(), RPCError> { // Make sure hop count makes sense if route.safety_route.hop_count != 0 { @@ -116,14 +116,17 @@ impl RPCProcessor { } }?; - // Sign the operation if this is not our last hop - // as the last hop is already signed by the envelope - if next_private_route.hop_count != 0 { + if first_hop.next_hop.is_some() { + // Sign the operation if this is not our last hop + // as the last hop is already signed by the envelope let node_id = self.routing_table.node_id(); let node_id_secret = self.routing_table.node_id_secret(); let sig = sign(&node_id, &node_id_secret, &route.operation.data) .map_err(RPCError::internal)?; route.operation.signatures.push(sig); + } else { + // If this is our last hop, then we drop the 'first_hop' from private route + next_private_route.first_hop = None; } // Pass along the route @@ -398,6 +401,11 @@ impl RPCProcessor { None }; + // Ensure hop count > 0 + if private_route.hop_count == 0 { + return Err(RPCError::protocol("route should not be at the end")); + } + // Make next PrivateRoute and pass it on let next_private_route = PrivateRoute { public_key: private_route.public_key, @@ -407,6 +415,11 @@ impl RPCProcessor { self.process_route_private_route_hop(route, next_private_route) .await?; } else { + // Ensure hop count == 0 + if private_route.hop_count != 0 { + return Err(RPCError::protocol("route should be at the end")); + } + // No hops left, time to process the routed operation self.process_routed_operation( detail, From 4a9d516f32cee31d03316d208acf32704f50b48d Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 13 Nov 2022 12:48:44 -0500 Subject: [PATCH 55/67] xfer --- .../src/routing_table/route_spec_store.rs | 50 ++++++++++++++++--- veilid-core/src/veilid_api/debug.rs | 22 +++++--- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 0d1f271d..bbc4e7b6 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -388,12 +388,21 @@ impl RouteSpecStore { sequencing: Sequencing, hop_count: usize, directions: DirectionSet, + avoid_node_id: Option, ) -> EyreResult> { let inner = &mut *self.inner.lock(); let routing_table = self.unlocked_inner.routing_table.clone(); let rti = &mut *routing_table.inner.write(); - self.allocate_route_inner(inner, rti, stability, sequencing, hop_count, directions) + self.allocate_route_inner( + inner, + rti, + stability, + sequencing, + hop_count, + directions, + avoid_node_id, + ) } fn allocate_route_inner( @@ -404,6 +413,7 @@ impl RouteSpecStore { sequencing: Sequencing, hop_count: usize, directions: DirectionSet, + avoid_node_id: Option, ) -> EyreResult> { use core::cmp::Ordering; @@ -418,7 +428,7 @@ impl RouteSpecStore { // Get list of all nodes, and sort them for selection let cur_ts = intf::get_timestamp(); let filter = Box::new( - move |rti: &RoutingTableInner, _k: DHTKey, v: Option>| -> bool { + move |rti: &RoutingTableInner, k: DHTKey, v: Option>| -> bool { // Exclude our own node from routes if v.is_none() { return false; @@ -433,6 +443,13 @@ impl RouteSpecStore { return false; } + // Exclude node we have specifically chosen to avoid + if let Some(ani) = avoid_node_id { + if k == ani { + return false; + } + } + // Exclude nodes with no publicinternet nodeinfo, or incompatible nodeinfo or node status won't route v.with(rti, move |_rti, e| { let node_info_ok = @@ -748,6 +765,7 @@ impl RouteSpecStore { stability: Stability, sequencing: Sequencing, directions: DirectionSet, + avoid_node_id: Option, ) -> Option { let inner = self.inner.lock(); @@ -759,7 +777,15 @@ impl RouteSpecStore { && detail.1.directions.is_subset(directions) && !detail.1.published { - return Some(*detail.0); + let mut avoid = false; + if let Some(ani) = &avoid_node_id { + if detail.1.hops.contains(ani) { + avoid = true; + } + } + if !avoid { + return Some(*detail.0); + } } } None @@ -800,16 +826,15 @@ impl RouteSpecStore { if pr_hopcount > max_route_hop_count { bail!("private route hop count too long"); } + let Some(pr_first_hop) = &private_route.first_hop else { + bail!("compiled private route should have first_hop"); + }; // See if we are using a safety route, if not, short circuit this operation let safety_spec = match safety_selection { SafetySelection::Unsafe(sequencing) => { // Safety route stub with the node's public key as the safety route key since it's the 0th hop - if private_route.first_hop.is_none() { - bail!("can't compile zero length route"); - } - let first_hop = private_route.first_hop.as_ref().unwrap(); - let opt_first_hop = match &first_hop.node { + let opt_first_hop = match &pr_first_hop.node { RouteNode::NodeId(id) => rti.lookup_node_ref(routing_table.clone(), id.key), RouteNode::PeerInfo(pi) => rti.register_node_with_signed_node_info( routing_table.clone(), @@ -851,6 +876,13 @@ impl RouteSpecStore { // Safety route exists safety_rsd } else { + // Avoid having the first node in the private route in our chosen safety route + // We would avoid all of them, but by design only the first node is knowable + let avoid_node_id = match &pr_first_hop.node { + RouteNode::NodeId(n) => n.key, + RouteNode::PeerInfo(p) => p.node_id.key, + }; + // Select a safety route from the pool or make one if we don't have one that matches if let Some(sr_pubkey) = self.first_unpublished_route( safety_spec.hop_count, @@ -858,6 +890,7 @@ impl RouteSpecStore { safety_spec.stability, safety_spec.sequencing, Direction::Outbound.into(), + Some(avoid_node_id), ) { // Found a route to use (Self::detail_mut(inner, &sr_pubkey).unwrap(), sr_pubkey) @@ -871,6 +904,7 @@ impl RouteSpecStore { safety_spec.sequencing, safety_spec.hop_count, Direction::Outbound.into(), + Some(avoid_node_id), ) .map_err(RPCError::internal)? { diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 9405db24..c2ed9070 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -566,7 +566,7 @@ impl VeilidAPI { } async fn debug_route_allocate(&self, args: Vec) -> Result { - // [ord|*ord] [rel] [] [in|out] + // [ord|*ord] [rel] [] [in|out] [avoid_node_id] let netman = self.network_manager()?; let routing_table = netman.routing_table(); @@ -582,6 +582,7 @@ impl VeilidAPI { let mut stability = Stability::LowLatency; let mut hop_count = default_route_hop_count; let mut directions = DirectionSet::all(); + let mut avoid_node_id = None; while ai < args.len() { if let Ok(seq) = @@ -600,6 +601,10 @@ impl VeilidAPI { get_debug_argument_at(&args, ai, "debug_route", "direction_set", get_direction_set) { directions = ds; + } else if let Ok(ani) = + get_debug_argument_at(&args, ai, "debug_route", "avoid_node_id", get_dht_key) + { + avoid_node_id = Some(ani); } else { return Ok(format!("Invalid argument specified: {}", args[ai])); } @@ -607,13 +612,14 @@ impl VeilidAPI { } // Allocate route - let out = match rss.allocate_route(stability, sequencing, hop_count, directions) { - Ok(Some(v)) => format!("{}", v.encode()), - Ok(None) => format!(""), - Err(e) => { - format!("Route allocation failed: {}", e) - } - }; + let out = + match rss.allocate_route(stability, sequencing, hop_count, directions, avoid_node_id) { + Ok(Some(v)) => format!("{}", v.encode()), + Ok(None) => format!(""), + Err(e) => { + format!("Route allocation failed: {}", e) + } + }; Ok(out) } From 5935ca9e41fd4e9f19c3fc47f5a4ee9527395ef2 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 13 Nov 2022 19:46:10 -0500 Subject: [PATCH 56/67] route spec store work --- .../src/routing_table/route_spec_store.rs | 315 +++++++++++++----- veilid-core/src/rpc_processor/destination.rs | 181 ++++++++++ veilid-core/src/rpc_processor/mod.rs | 67 ---- veilid-core/src/rpc_processor/origin.rs | 53 --- veilid-core/src/rpc_processor/rpc_app_call.rs | 5 +- .../src/rpc_processor/rpc_find_node.rs | 5 +- veilid-core/src/rpc_processor/rpc_status.rs | 5 +- veilid-core/src/veilid_api/debug.rs | 20 +- veilid-core/src/veilid_api/mod.rs | 2 +- 9 files changed, 441 insertions(+), 212 deletions(-) delete mode 100644 veilid-core/src/rpc_processor/origin.rs diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index bbc4e7b6..17fe558b 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -4,6 +4,11 @@ use rkyv::{ with::Skip, Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, }; +/// The size of the remote private route cache +const REMOTE_PRIVATE_ROUTE_CACHE_SIZE: usize = 1024; +/// Remote private route cache entries expire in 5 minutes if they haven't been used +const REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY: u64 = 300_000_000u64; + /// Compiled route (safety route + private route) #[derive(Clone, Debug)] pub struct CompiledRoute { @@ -73,8 +78,21 @@ pub struct RouteSpecStoreContent { details: HashMap, } +/// What remote private routes have seen +#[derive(Debug, Clone, Default)] +struct RemotePrivateRouteInfo { + // When this remote private route was last modified + modified_ts: u64, + /// Did this remote private route see our node info due to no safety route in use + seen_our_node_info: bool, + /// The time this remote private route last responded + last_replied_ts: Option, + /// Timestamp of when the route was last used for anything + last_used_ts: Option, +} + /// Ephemeral data used to help the RouteSpecStore operate efficiently -#[derive(Debug, Default)] +#[derive(Debug)] pub struct RouteSpecStoreCache { /// How many times nodes have been used used_nodes: HashMap, @@ -82,6 +100,19 @@ pub struct RouteSpecStoreCache { used_end_nodes: HashMap, /// Route spec hop cache, used to quickly disqualify routes hop_cache: HashSet>, + /// Has a remote private route responded to a question and when + remote_private_route_cache: LruCache, +} + +impl Default for RouteSpecStoreCache { + fn default() -> Self { + Self { + used_nodes: Default::default(), + used_end_nodes: Default::default(), + hop_cache: Default::default(), + remote_private_route_cache: LruCache::new(REMOTE_PRIVATE_ROUTE_CACHE_SIZE), + } + } } #[derive(Debug)] @@ -388,7 +419,7 @@ impl RouteSpecStore { sequencing: Sequencing, hop_count: usize, directions: DirectionSet, - avoid_node_id: Option, + avoid_node_ids: &[DHTKey], ) -> EyreResult> { let inner = &mut *self.inner.lock(); let routing_table = self.unlocked_inner.routing_table.clone(); @@ -401,7 +432,7 @@ impl RouteSpecStore { sequencing, hop_count, directions, - avoid_node_id, + avoid_node_ids, ) } @@ -413,7 +444,7 @@ impl RouteSpecStore { sequencing: Sequencing, hop_count: usize, directions: DirectionSet, - avoid_node_id: Option, + avoid_node_ids: &[DHTKey], ) -> EyreResult> { use core::cmp::Ordering; @@ -443,11 +474,9 @@ impl RouteSpecStore { return false; } - // Exclude node we have specifically chosen to avoid - if let Some(ani) = avoid_node_id { - if k == ani { - return false; - } + // Exclude nodes we have specifically chosen to avoid + if avoid_node_ids.contains(&k) { + return false; } // Exclude nodes with no publicinternet nodeinfo, or incompatible nodeinfo or node status won't route @@ -758,17 +787,15 @@ impl RouteSpecStore { } /// Find first matching unpublished route that fits into the selection criteria - pub fn first_unpublished_route( - &self, + fn first_unpublished_route_inner<'a>( + inner: &'a RouteSpecStoreInner, min_hop_count: usize, max_hop_count: usize, stability: Stability, sequencing: Sequencing, directions: DirectionSet, - avoid_node_id: Option, + avoid_node_ids: &[DHTKey], ) -> Option { - let inner = self.inner.lock(); - for detail in &inner.content.details { if detail.1.stability >= stability && detail.1.sequencing >= sequencing @@ -778,9 +805,10 @@ impl RouteSpecStore { && !detail.1.published { let mut avoid = false; - if let Some(ani) = &avoid_node_id { - if detail.1.hops.contains(ani) { + for h in &detail.1.hops { + if avoid_node_ids.contains(h) { avoid = true; + break; } } if !avoid { @@ -864,65 +892,16 @@ impl RouteSpecStore { SafetySelection::Safe(safety_spec) => safety_spec, }; - // See if the preferred route is here - let opt_safety_rsd: Option<(&mut RouteSpecDetail, DHTKey)> = - if let Some(preferred_route) = safety_spec.preferred_route { - Self::detail_mut(inner, &preferred_route).map(|rsd| (rsd, preferred_route)) - } else { - // Preferred safety route was not requested - None - }; - let (safety_rsd, sr_pubkey) = if let Some(safety_rsd) = opt_safety_rsd { - // Safety route exists - safety_rsd - } else { - // Avoid having the first node in the private route in our chosen safety route - // We would avoid all of them, but by design only the first node is knowable - let avoid_node_id = match &pr_first_hop.node { - RouteNode::NodeId(n) => n.key, - RouteNode::PeerInfo(p) => p.node_id.key, - }; - - // Select a safety route from the pool or make one if we don't have one that matches - if let Some(sr_pubkey) = self.first_unpublished_route( - safety_spec.hop_count, - safety_spec.hop_count, - safety_spec.stability, - safety_spec.sequencing, - Direction::Outbound.into(), - Some(avoid_node_id), - ) { - // Found a route to use - (Self::detail_mut(inner, &sr_pubkey).unwrap(), sr_pubkey) - } else { - // No route found, gotta allocate one - let sr_pubkey = match self - .allocate_route_inner( - inner, - rti, - safety_spec.stability, - safety_spec.sequencing, - safety_spec.hop_count, - Direction::Outbound.into(), - Some(avoid_node_id), - ) - .map_err(RPCError::internal)? - { - Some(pk) => pk, - None => return Ok(None), - }; - (Self::detail_mut(inner, &sr_pubkey).unwrap(), sr_pubkey) - } + // Get the safety route to use from the spec + let avoid_node_id = match &pr_first_hop.node { + RouteNode::NodeId(n) => n.key, + RouteNode::PeerInfo(p) => p.node_id.key, }; - - // Ensure the total hop count isn't too long for our config - let sr_hopcount = safety_spec.hop_count; - if sr_hopcount == 0 { - bail!("safety route hop count is zero"); - } - if sr_hopcount > max_route_hop_count { - bail!("safety route hop count too long"); - } + let Some(sr_pubkey) = self.get_route_for_safety_spec_inner(inner, rti, &safety_spec, Direction::Outbound.into(), &[avoid_node_id])? else { + // No safety route could be found for this spec + return Ok(None); + }; + let safety_rsd = Self::detail_mut(inner, &sr_pubkey).unwrap(); // See if we can optimize this compilation yet // We don't want to include full nodeinfo if we don't have to @@ -951,7 +930,7 @@ impl RouteSpecStore { // Each loop mutates 'nonce', and 'blob_data' let mut nonce = Crypto::get_random_nonce(); let crypto = routing_table.network_manager().crypto(); - for h in (1..sr_hopcount).rev() { + for h in (1..safety_rsd.hops.len()).rev() { // Get blob to encrypt for next hop blob_data = { // Encrypt the previous blob ENC(nonce, DH(PKhop,SKsr)) @@ -1046,6 +1025,84 @@ impl RouteSpecStore { Ok(Some(compiled_route)) } + /// Get a route that matches a particular safety spec + fn get_route_for_safety_spec_inner( + &self, + inner: &mut RouteSpecStoreInner, + rti: &RoutingTableInner, + safety_spec: &SafetySpec, + direction: DirectionSet, + avoid_node_ids: &[DHTKey], + ) -> EyreResult> { + // Ensure the total hop count isn't too long for our config + let max_route_hop_count = self.unlocked_inner.max_route_hop_count; + if safety_spec.hop_count == 0 { + bail!("safety route hop count is zero"); + } + if safety_spec.hop_count > max_route_hop_count { + bail!("safety route hop count too long"); + } + + // See if the preferred route is here + if let Some(preferred_route) = safety_spec.preferred_route { + if inner.content.details.contains_key(&preferred_route) { + return Ok(Some(preferred_route)); + } + } + + // Select a safety route from the pool or make one if we don't have one that matches + let sr_pubkey = if let Some(sr_pubkey) = Self::first_unpublished_route_inner( + inner, + safety_spec.hop_count, + safety_spec.hop_count, + safety_spec.stability, + safety_spec.sequencing, + direction, + avoid_node_ids, + ) { + // Found a route to use + sr_pubkey + } else { + // No route found, gotta allocate one + let sr_pubkey = match self + .allocate_route_inner( + inner, + rti, + safety_spec.stability, + safety_spec.sequencing, + safety_spec.hop_count, + direction, + avoid_node_ids, + ) + .map_err(RPCError::internal)? + { + Some(pk) => pk, + None => return Ok(None), + }; + sr_pubkey + }; + Ok(Some(sr_pubkey)) + } + + /// Get a private sroute to use for the answer to question + pub fn get_private_route_for_safety_spec( + &self, + safety_spec: &SafetySpec, + avoid_node_ids: &[DHTKey], + ) -> EyreResult> { + let inner = &mut *self.inner.lock(); + let routing_table = self.unlocked_inner.routing_table.clone(); + let rti = &*routing_table.inner.read(); + + Ok(self.get_route_for_safety_spec_inner( + inner, + rti, + safety_spec, + Direction::Inbound.into(), + avoid_node_ids, + )?) + } + /// Assemble private route for publication pub fn assemble_private_route( &self, @@ -1127,6 +1184,114 @@ impl RouteSpecStore { Ok(private_route) } + // get or create a remote private route cache entry + fn with_create_remote_private_route( + inner: &mut RouteSpecStoreInner, + cur_ts: u64, + key: &DHTKey, + f: F, + ) -> R + where + F: FnOnce(&mut RemotePrivateRouteInfo) -> R, + { + let rpr = inner + .cache + .remote_private_route_cache + .entry(*key) + .and_modify(|rpr| { + if cur_ts - rpr.modified_ts >= REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY { + *rpr = RemotePrivateRouteInfo { + modified_ts: cur_ts, + seen_our_node_info: false, + last_replied_ts: None, + last_used_ts: None, + }; + } else { + rpr.modified_ts = cur_ts; + } + }) + .or_insert_with(|| RemotePrivateRouteInfo { + modified_ts: cur_ts, + seen_our_node_info: false, + last_replied_ts: None, + last_used_ts: None, + }); + f(rpr) + } + + // get a remote private route cache entry + fn with_get_remote_private_route( + inner: &mut RouteSpecStoreInner, + cur_ts: u64, + key: &DHTKey, + f: F, + ) -> Option + where + F: FnOnce(&RemotePrivateRouteInfo) -> R, + { + let rpr = inner.cache.remote_private_route_cache.get(key)?; + if cur_ts - rpr.modified_ts < REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY { + return Some(f(rpr)); + } + inner.cache.remote_private_route_cache.remove(key); + None + } + + /// Check to see if this remote (not ours) private route has seen our node info yet + /// This returns true if we have sent non-safety-route node info to the + /// private route and gotten a response before + pub fn has_remote_private_route_seen_our_node_info(&self, key: &DHTKey) -> bool { + let inner = &mut *self.inner.lock(); + let cur_ts = intf::get_timestamp(); + Self::with_get_remote_private_route(inner, cur_ts, key, |rpr| rpr.seen_our_node_info) + .unwrap_or_default() + } + + /// Mark a remote private route as having seen our node info { + pub fn mark_remote_private_route_seen_our_node_info(&self, key: &DHTKey) { + let inner = &mut *self.inner.lock(); + let cur_ts = intf::get_timestamp(); + Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { + rpr.seen_our_node_info = true; + }) + } + + /// Mark a remote private route as having replied to a question { + pub fn mark_remote_private_route_replied(&self, key: &DHTKey) { + let inner = &mut *self.inner.lock(); + let cur_ts = intf::get_timestamp(); + Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { + rpr.last_replied_ts = Some(cur_ts); + }) + } + + /// Mark a remote private route as having beed used { + pub fn mark_remote_private_route_used(&self, key: &DHTKey) { + let inner = &mut *self.inner.lock(); + let cur_ts = intf::get_timestamp(); + Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { + rpr.last_used_ts = Some(cur_ts); + }) + } + + /// Clear caches when local our local node info changes + pub fn local_node_info_changed(&self) { + let inner = &mut *self.inner.lock(); + + // Clean up local allocated routes + for (_k, v) in &mut inner.content.details { + // Must republish route now + v.published = false; + // Route is not known reachable now + v.reachable = false; + // We have yet to check it since local node info changed + v.last_checked_ts = None; + } + + // Clean up remote private routes + inner.cache.remote_private_route_cache.clear(); + } + /// Mark route as published /// When first deserialized, routes must be re-published in order to ensure they remain /// in the RouteSpecStore. diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index 033b08f7..33647085 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -141,3 +141,184 @@ impl fmt::Display for Destination { } } } + +impl RPCProcessor { + /// Convert the 'Destination' into a 'RespondTo' for a response + pub(super) fn get_destination_respond_to( + &self, + dest: &Destination, + ) -> Result, RPCError> { + let routing_table = self.routing_table(); + let rss = routing_table.route_spec_store(); + + match dest { + Destination::Direct { + target, + safety_selection, + } => match safety_selection { + SafetySelection::Unsafe(_) => { + // Sent directly with no safety route, can respond directly + Ok(NetworkResult::value(RespondTo::Sender)) + } + SafetySelection::Safe(safety_spec) => { + // Sent directly but with a safety route, respond to private route + let Some(pr_key) = rss + .get_private_route_for_safety_spec(safety_spec, &[target.node_id()]) + .map_err(RPCError::internal)? else { + return Ok(NetworkResult::no_connection_other("no private route for response at this time")); + }; + + // Get the assembled route for response + let private_route = rss + .assemble_private_route(&pr_key, None) + .map_err(RPCError::internal)?; + + Ok(NetworkResult::Value(RespondTo::PrivateRoute(private_route))) + } + }, + Destination::Relay { + relay, + target, + safety_selection, + } => match safety_selection { + SafetySelection::Unsafe(_) => { + // Sent via a relay with no safety route, can respond directly + Ok(NetworkResult::value(RespondTo::Sender)) + } + SafetySelection::Safe(safety_spec) => { + // Sent via a relay but with a safety route, respond to private route + let Some(pr_key) = rss + .get_private_route_for_safety_spec(safety_spec, &[relay.node_id(), *target]) + .map_err(RPCError::internal)? else { + return Ok(NetworkResult::no_connection_other("no private route for response at this time")); + }; + + // Get the assembled route for response + let private_route = rss + .assemble_private_route(&pr_key, None) + .map_err(RPCError::internal)?; + + Ok(NetworkResult::Value(RespondTo::PrivateRoute(private_route))) + } + }, + Destination::PrivateRoute { + private_route, + safety_selection, + } => { + let Some(pr_first_hop) = &private_route.first_hop else { + return Err(RPCError::internal("destination private route must have first_hop")); + }; + + match safety_selection { + SafetySelection::Unsafe(_) => { + // Sent to a private route with no safety route, use a stub safety route for the response + + // Determine if we can use optimized nodeinfo + let route_node = match rss + .has_remote_private_route_seen_our_node_info(&private_route.public_key) + { + true => RouteNode::NodeId(NodeId::new(routing_table.node_id())), + false => RouteNode::PeerInfo( + routing_table.get_own_peer_info(RoutingDomain::PublicInternet), + ), + }; + + Ok(NetworkResult::value(RespondTo::PrivateRoute( + PrivateRoute::new_stub(routing_table.node_id(), route_node), + ))) + } + SafetySelection::Safe(safety_spec) => { + // Sent directly but with a safety route, respond to private route + let avoid_node_id = match &pr_first_hop.node { + RouteNode::NodeId(n) => n.key, + RouteNode::PeerInfo(p) => p.node_id.key, + }; + + let Some(pr_key) = rss + .get_private_route_for_safety_spec(safety_spec, &[avoid_node_id]) + .map_err(RPCError::internal)? else { + return Ok(NetworkResult::no_connection_other("no private route for response at this time")); + }; + + // Get the assembled route for response + let private_route = rss + .assemble_private_route(&pr_key, None) + .map_err(RPCError::internal)?; + + Ok(NetworkResult::Value(RespondTo::PrivateRoute(private_route))) + } + } + } + } + } + + /// Convert the 'RespondTo' into a 'Destination' for a response + pub(super) fn get_respond_to_destination( + &self, + request: &RPCMessage, + ) -> NetworkResult { + // Get the question 'respond to' + let respond_to = match request.operation.kind() { + RPCOperationKind::Question(q) => q.respond_to(), + _ => { + panic!("not a question"); + } + }; + + // To where should we respond? + match respond_to { + RespondTo::Sender => { + // Parse out the header detail from the question + let detail = match &request.header.detail { + RPCMessageHeaderDetail::Direct(detail) => detail, + RPCMessageHeaderDetail::SafetyRouted(_) + | RPCMessageHeaderDetail::PrivateRouted(_) => { + // If this was sent via a private route, we don't know what the sender was, so drop this + return NetworkResult::invalid_message( + "can't respond directly to non-direct question", + ); + } + }; + + // Reply directly to the request's source + let sender_id = detail.envelope.get_sender_id(); + + // This may be a different node's reference than the 'sender' in the case of a relay + let peer_noderef = detail.peer_noderef.clone(); + + // If the sender_id is that of the peer, then this is a direct reply + // else it is a relayed reply through the peer + if peer_noderef.node_id() == sender_id { + NetworkResult::value(Destination::direct(peer_noderef)) + } else { + NetworkResult::value(Destination::relay(peer_noderef, sender_id)) + } + } + RespondTo::PrivateRoute(pr) => { + match &request.header.detail { + RPCMessageHeaderDetail::Direct(_) => { + // If this was sent directly, we should only ever respond directly + return NetworkResult::invalid_message( + "not responding to private route from direct question", + ); + } + RPCMessageHeaderDetail::SafetyRouted(detail) => { + // If this was sent via a safety route, but not received over our private route, don't respond with a safety route, + // it would give away which safety routes belong to this node + NetworkResult::value(Destination::private_route( + pr.clone(), + SafetySelection::Unsafe(detail.sequencing), + )) + } + RPCMessageHeaderDetail::PrivateRouted(detail) => { + // If this was received over our private route, it's okay to respond to a private route via our safety route + NetworkResult::value(Destination::private_route( + pr.clone(), + SafetySelection::Safe(detail.safety_spec.clone()), + )) + } + } + } + } + } +} diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index d74527a4..460323ad 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -792,73 +792,6 @@ impl RPCProcessor { Ok(NetworkResult::value(())) } - // Convert the 'RespondTo' into a 'Destination' for a response - fn get_respond_to_destination(&self, request: &RPCMessage) -> NetworkResult { - // Get the question 'respond to' - let respond_to = match request.operation.kind() { - RPCOperationKind::Question(q) => q.respond_to(), - _ => { - panic!("not a question"); - } - }; - - // To where should we respond? - match respond_to { - RespondTo::Sender => { - // Parse out the header detail from the question - let detail = match &request.header.detail { - RPCMessageHeaderDetail::Direct(detail) => detail, - RPCMessageHeaderDetail::SafetyRouted(_) - | RPCMessageHeaderDetail::PrivateRouted(_) => { - // If this was sent via a private route, we don't know what the sender was, so drop this - return NetworkResult::invalid_message( - "can't respond directly to non-direct question", - ); - } - }; - - // Reply directly to the request's source - let sender_id = detail.envelope.get_sender_id(); - - // This may be a different node's reference than the 'sender' in the case of a relay - let peer_noderef = detail.peer_noderef.clone(); - - // If the sender_id is that of the peer, then this is a direct reply - // else it is a relayed reply through the peer - if peer_noderef.node_id() == sender_id { - NetworkResult::value(Destination::direct(peer_noderef)) - } else { - NetworkResult::value(Destination::relay(peer_noderef, sender_id)) - } - } - RespondTo::PrivateRoute(pr) => { - match &request.header.detail { - RPCMessageHeaderDetail::Direct(_) => { - // If this was sent directly, we should only ever respond directly - return NetworkResult::invalid_message( - "not responding to private route from direct question", - ); - } - RPCMessageHeaderDetail::SafetyRouted(detail) => { - // If this was sent via a safety route, but not received over our private route, don't respond with a safety route, - // it would give away which safety routes belong to this node - NetworkResult::value(Destination::private_route( - pr.clone(), - SafetySelection::Unsafe(detail.sequencing), - )) - } - RPCMessageHeaderDetail::PrivateRouted(detail) => { - // If this was received over our private route, it's okay to respond to a private route via our safety route - NetworkResult::value(Destination::private_route( - pr.clone(), - SafetySelection::Safe(detail.safety_spec.clone()), - )) - } - } - } - } - } - // Issue a reply over the network, possibly using an anonymized route // The request must want a response, or this routine fails #[instrument(level = "debug", skip(self, request, answer), err)] diff --git a/veilid-core/src/rpc_processor/origin.rs b/veilid-core/src/rpc_processor/origin.rs deleted file mode 100644 index e1126cab..00000000 --- a/veilid-core/src/rpc_processor/origin.rs +++ /dev/null @@ -1,53 +0,0 @@ -use super::*; - -#[derive(Debug, Clone)] -pub enum Origin { - Sender, - PrivateRoute(PrivateRoute), -} - -impl Origin { - pub fn sender() -> Self { - Self::Sender - } - - pub fn private_route(private_route: PrivateRoute) -> Self { - Self::PrivateRoute(private_route) - } - - pub fn into_respond_to(self, destination: &Destination) -> Result { - match self { - Self::Sender => { - let peer = match destination { - Destination::Direct { - target, - safety_route_spec, - } => todo!(), - Destination::Relay { - relay, - target, - safety_route_spec, - } => todo!(), - Destination::PrivateRoute { - private_route, - safety_route_spec, - } => todo!(), - }; - let routing_table = peer.routing_table(); - let routing_domain = peer.best_routing_domain(); - // Send some signed node info along with the question if this node needs to be replied to - if routing_table.has_valid_own_node_info() - && !peer.has_seen_our_node_info(routing_domain) - { - let our_sni = self - .routing_table() - .get_own_signed_node_info(routing_domain); - RespondTo::Sender(Some(our_sni)) - } else { - RespondTo::Sender(None) - } - } - Self::PrivateRoute(pr) => RespondTo::PrivateRoute(pr), - } - } -} diff --git a/veilid-core/src/rpc_processor/rpc_app_call.rs b/veilid-core/src/rpc_processor/rpc_app_call.rs index 13466f4a..8f57c4af 100644 --- a/veilid-core/src/rpc_processor/rpc_app_call.rs +++ b/veilid-core/src/rpc_processor/rpc_app_call.rs @@ -10,7 +10,10 @@ impl RPCProcessor { message: Vec, ) -> Result>>, RPCError> { let app_call_q = RPCOperationAppCallQ { message }; - let question = RPCQuestion::new(RespondTo::Sender, RPCQuestionDetail::AppCallQ(app_call_q)); + let question = RPCQuestion::new( + network_result_try!(self.get_destination_respond_to(&dest)?), + RPCQuestionDetail::AppCallQ(app_call_q), + ); // Send the app call question let waitable_reply = network_result_try!(self.question(dest, question).await?); diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index 219d7f9b..c75d820b 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -28,7 +28,10 @@ impl RPCProcessor { let find_node_q_detail = RPCQuestionDetail::FindNodeQ(RPCOperationFindNodeQ { node_id: key }); - let find_node_q = RPCQuestion::new(RespondTo::Sender, find_node_q_detail); + let find_node_q = RPCQuestion::new( + network_result_try!(self.get_destination_respond_to(&dest)?), + find_node_q_detail, + ); // Send the find_node request let waitable_reply = network_result_try!(self.question(dest, find_node_q).await?); diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 6725ddfa..32e13a43 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -70,7 +70,10 @@ impl RPCProcessor { }; let status_q = RPCOperationStatusQ { node_status }; - let question = RPCQuestion::new(RespondTo::Sender, RPCQuestionDetail::StatusQ(status_q)); + let question = RPCQuestion::new( + network_result_try!(self.get_destination_respond_to(&dest)?), + RPCQuestionDetail::StatusQ(status_q), + ); // Send the info request let waitable_reply = network_result_try!(self.question(dest.clone(), question).await?); diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index c2ed9070..2b747695 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -582,7 +582,6 @@ impl VeilidAPI { let mut stability = Stability::LowLatency; let mut hop_count = default_route_hop_count; let mut directions = DirectionSet::all(); - let mut avoid_node_id = None; while ai < args.len() { if let Ok(seq) = @@ -601,10 +600,6 @@ impl VeilidAPI { get_debug_argument_at(&args, ai, "debug_route", "direction_set", get_direction_set) { directions = ds; - } else if let Ok(ani) = - get_debug_argument_at(&args, ai, "debug_route", "avoid_node_id", get_dht_key) - { - avoid_node_id = Some(ani); } else { return Ok(format!("Invalid argument specified: {}", args[ai])); } @@ -612,14 +607,13 @@ impl VeilidAPI { } // Allocate route - let out = - match rss.allocate_route(stability, sequencing, hop_count, directions, avoid_node_id) { - Ok(Some(v)) => format!("{}", v.encode()), - Ok(None) => format!(""), - Err(e) => { - format!("Route allocation failed: {}", e) - } - }; + let out = match rss.allocate_route(stability, sequencing, hop_count, directions, &[]) { + Ok(Some(v)) => format!("{}", v.encode()), + Ok(None) => format!(""), + Err(e) => { + format!("Route allocation failed: {}", e) + } + }; Ok(out) } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 9f4994c5..73b0e817 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -608,7 +608,7 @@ pub enum SafetySelection { pub struct SafetySpec { /// preferred safety route if it still exists pub preferred_route: Option, - /// 0 = no safety route, just use node's node id, more hops is safer but slower + /// must be greater than 0 pub hop_count: usize, /// prefer reliability over speed pub stability: Stability, From da9276a77ffcaa05d2c53c38451108de90f4bbb6 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 13 Nov 2022 22:45:03 -0500 Subject: [PATCH 57/67] meh --- veilid-core/src/rpc_processor/rpc_route.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index a6031441..9f771708 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -126,7 +126,7 @@ impl RPCProcessor { route.operation.signatures.push(sig); } else { // If this is our last hop, then we drop the 'first_hop' from private route - next_private_route.first_hop = None; + // XXX ? next_private_route.first_hop = None; } // Pass along the route @@ -372,8 +372,9 @@ impl RPCProcessor { } // No safety route left, now doing private route SafetyRouteHops::Private(ref private_route) => { + // See if we have a hop, if not, we are at the end of the private route if let Some(first_hop) = &private_route.first_hop { - // See if we have a next hop to send to + // See if we have next hop data let opt_next_first_hop = if let Some(next_hop) = &first_hop.next_hop { // Decrypt the blob with DEC(nonce, DH(the PR's public key, this hop's secret) let node_id_secret = self.routing_table.node_id_secret(); From 28c31fe4249c191f681af3b4e7dfa0c80d47cae3 Mon Sep 17 00:00:00 2001 From: John Smith Date: Mon, 14 Nov 2022 13:09:33 -0500 Subject: [PATCH 58/67] fix private routing 1.0 --- veilid-core/proto/veilid.capnp | 8 +- .../src/routing_table/route_spec_store.rs | 6 +- .../coders/private_safety_route.rs | 37 +++-- veilid-core/src/rpc_processor/destination.rs | 4 +- veilid-core/src/rpc_processor/rpc_route.rs | 139 +++++++++--------- veilid-core/src/veilid_api/privacy.rs | 59 +++++--- 6 files changed, 143 insertions(+), 110 deletions(-) diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 1ce4c11a..05bd4de7 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -126,14 +126,18 @@ struct RouteHop @0xf8f672d75cce0c3b { nodeId @0 :NodeID; # node id only for established routes peerInfo @1 :PeerInfo; # full peer info for this hop to establish the route } - nextHop @2 :RouteHopData; # Optional: if the private route is a stub, it contains no route hop data, just the target node for the routed operation. + nextHop @2 :RouteHopData; # optional: If this the end of a private route, this field will not exist # if this is a safety route routehop, this field is not optional and must exist } struct PrivateRoute @0x8a83fccb0851e776 { publicKey @0 :RoutePublicKey; # private route public key (unique per private route) hopCount @1 :UInt8; # Count of hops left in the private route (for timeout calculation purposes only) - firstHop @2 :RouteHop; # Optional: first hop in the private route, if empty, this is the last hop and payload should be decrypted and processed. + hops :union { + firstHop @2 :RouteHop; # first hop of a private route is unencrypted (hopcount > 0) + data @3 :RouteHopData; # private route has more hops (hopcount > 0 && hopcount < total_hopcount) + empty @4 :Void; # private route has ended (hopcount = 0) + } } struct SafetyRoute @0xf554734d07cb5d59 { diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 17fe558b..8d1c0cf3 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -854,8 +854,8 @@ impl RouteSpecStore { if pr_hopcount > max_route_hop_count { bail!("private route hop count too long"); } - let Some(pr_first_hop) = &private_route.first_hop else { - bail!("compiled private route should have first_hop"); + let PrivateRouteHops::FirstHop(pr_first_hop) = &private_route.hops else { + bail!("compiled private route should have first hop"); }; // See if we are using a safety route, if not, short circuit this operation @@ -1179,7 +1179,7 @@ impl RouteSpecStore { let private_route = PrivateRoute { public_key: key.clone(), hop_count: hop_count.try_into().unwrap(), - first_hop: Some(route_hop), + hops: PrivateRouteHops::FirstHop(route_hop), }; Ok(private_route) } diff --git a/veilid-core/src/rpc_processor/coders/private_safety_route.rs b/veilid-core/src/rpc_processor/coders/private_safety_route.rs index 76a280f5..1edbb9fd 100644 --- a/veilid-core/src/rpc_processor/coders/private_safety_route.rs +++ b/veilid-core/src/rpc_processor/coders/private_safety_route.rs @@ -106,11 +106,20 @@ pub fn encode_private_route( &mut builder.reborrow().init_public_key(), )?; builder.set_hop_count(private_route.hop_count); - if let Some(rh) = &private_route.first_hop { - let mut rh_builder = builder.reborrow().init_first_hop(); - encode_route_hop(rh, &mut rh_builder)?; + let mut h_builder = builder.reborrow().init_hops(); + match &private_route.hops { + PrivateRouteHops::FirstHop(first_hop) => { + let mut rh_builder = h_builder.init_first_hop(); + encode_route_hop(first_hop, &mut rh_builder)?; + } + PrivateRouteHops::Data(data) => { + let mut rhd_builder = h_builder.init_data(); + encode_route_hop_data(data, &mut rhd_builder)?; + } + PrivateRouteHops::Empty => { + h_builder.set_empty(()); + } }; - Ok(()) } @@ -121,19 +130,23 @@ pub fn decode_private_route( "invalid public key in private route", ))?); let hop_count = reader.get_hop_count(); - let first_hop = if reader.has_first_hop() { - let rh_reader = reader - .get_first_hop() - .map_err(RPCError::map_protocol("invalid first hop in private route"))?; - Some(decode_route_hop(&rh_reader)?) - } else { - None + + let hops = match reader.get_hops().which().map_err(RPCError::protocol)? { + veilid_capnp::private_route::hops::Which::FirstHop(rh_reader) => { + let rh_reader = rh_reader.map_err(RPCError::protocol)?; + PrivateRouteHops::FirstHop(decode_route_hop(&rh_reader)?) + } + veilid_capnp::private_route::hops::Which::Data(rhd_reader) => { + let rhd_reader = rhd_reader.map_err(RPCError::protocol)?; + PrivateRouteHops::Data(decode_route_hop_data(&rhd_reader)?) + } + veilid_capnp::private_route::hops::Which::Empty(_) => PrivateRouteHops::Empty, }; Ok(PrivateRoute { public_key, hop_count, - first_hop, + hops, }) } diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index 33647085..d7a10d0b 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -205,8 +205,8 @@ impl RPCProcessor { private_route, safety_selection, } => { - let Some(pr_first_hop) = &private_route.first_hop else { - return Err(RPCError::internal("destination private route must have first_hop")); + let PrivateRouteHops::FirstHop(pr_first_hop) = &private_route.hops else { + return Err(RPCError::internal("destination private route must have first hop")); }; match safety_selection { diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 9f771708..3cffa743 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -72,26 +72,20 @@ impl RPCProcessor { #[instrument(level = "trace", skip_all, err)] async fn process_route_private_route_hop( &self, - mut route: RPCOperationRoute, - mut next_private_route: PrivateRoute, + mut routed_operation: RoutedOperation, + next_route_node: RouteNode, + safety_route_public_key: DHTKey, + next_private_route: PrivateRoute, ) -> Result<(), RPCError> { // Make sure hop count makes sense - if route.safety_route.hop_count != 0 { - return Err(RPCError::protocol( - "Safety hop count should be zero if switched to private route", - )); - } if next_private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { return Err(RPCError::protocol( "Private route hop count too high to process", )); } - // Get private route first hop (this is validated to not be None before calling this function) - let first_hop = next_private_route.first_hop.as_ref().unwrap(); - // Get next hop node ref - let next_hop_nr = match &first_hop.node { + let next_hop_nr = match &next_route_node { RouteNode::NodeId(id) => { // self.routing_table @@ -116,27 +110,24 @@ impl RPCProcessor { } }?; - if first_hop.next_hop.is_some() { + if !matches!(next_private_route.hops, PrivateRouteHops::Empty) { // Sign the operation if this is not our last hop // as the last hop is already signed by the envelope let node_id = self.routing_table.node_id(); let node_id_secret = self.routing_table.node_id_secret(); - let sig = sign(&node_id, &node_id_secret, &route.operation.data) + let sig = sign(&node_id, &node_id_secret, &routed_operation.data) .map_err(RPCError::internal)?; - route.operation.signatures.push(sig); - } else { - // If this is our last hop, then we drop the 'first_hop' from private route - // XXX ? next_private_route.first_hop = None; + routed_operation.signatures.push(sig); } // Pass along the route let next_hop_route = RPCOperationRoute { safety_route: SafetyRoute { - public_key: route.safety_route.public_key, + public_key: safety_route_public_key, hop_count: 0, hops: SafetyRouteHops::Private(next_private_route), }, - operation: route.operation, + operation: routed_operation, }; let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route)); @@ -342,19 +333,25 @@ impl RPCProcessor { }; // Get the next hop node ref - if private_route.first_hop.is_some() { - // Switching to private route from safety route - self.process_route_private_route_hop(route, private_route) - .await?; - } else { - // Private route is empty, process routed operation - self.process_routed_operation( - detail, - route.operation, - &route.safety_route, - &private_route, - )?; - } + let PrivateRouteHops::FirstHop(pr_first_hop) = private_route.hops else { + return Err(RPCError::protocol("switching from safety route to private route requires first hop")); + }; + + // Switching to private route from safety route + self.process_route_private_route_hop( + route.operation, + pr_first_hop.node, + route.safety_route.public_key, + PrivateRoute { + public_key: private_route.public_key, + hop_count: private_route.hop_count - 1, + hops: pr_first_hop + .next_hop + .map(|rhd| PrivateRouteHops::Data(rhd)) + .unwrap_or(PrivateRouteHops::Empty), + }, + ) + .await?; } else if blob_tag == 0 { // RouteHop let route_hop = { @@ -373,18 +370,24 @@ impl RPCProcessor { // No safety route left, now doing private route SafetyRouteHops::Private(ref private_route) => { // See if we have a hop, if not, we are at the end of the private route - if let Some(first_hop) = &private_route.first_hop { - // See if we have next hop data - let opt_next_first_hop = if let Some(next_hop) = &first_hop.next_hop { + match &private_route.hops { + PrivateRouteHops::FirstHop(_) => { + return Err(RPCError::protocol("should not have first hop here")); + } + PrivateRouteHops::Data(route_hop_data) => { // Decrypt the blob with DEC(nonce, DH(the PR's public key, this hop's secret) let node_id_secret = self.routing_table.node_id_secret(); let dh_secret = self .crypto .cached_dh(&private_route.public_key, &node_id_secret) .map_err(RPCError::protocol)?; - let dec_blob_data = - Crypto::decrypt_aead(&next_hop.blob, &next_hop.nonce, &dh_secret, None) - .map_err(RPCError::protocol)?; + let dec_blob_data = Crypto::decrypt_aead( + &route_hop_data.blob, + &route_hop_data.nonce, + &dh_secret, + None, + ) + .map_err(RPCError::protocol)?; let dec_blob_reader = RPCMessageData::new(dec_blob_data).get_reader()?; // Decode next RouteHop @@ -394,40 +397,42 @@ impl RPCProcessor { .map_err(RPCError::protocol)?; decode_route_hop(&rh_reader)? }; - Some(route_hop) - } else { - // If the first hop has no RouteHopData, then this is a stub private route - // and we should just pass the operation to its final destination with - // an empty safety and private route - None - }; - // Ensure hop count > 0 - if private_route.hop_count == 0 { - return Err(RPCError::protocol("route should not be at the end")); - } + // Ensure hop count > 0 + if private_route.hop_count == 0 { + return Err(RPCError::protocol("route should not be at the end")); + } - // Make next PrivateRoute and pass it on - let next_private_route = PrivateRoute { - public_key: private_route.public_key, - hop_count: private_route.hop_count - 1, - first_hop: opt_next_first_hop, - }; - self.process_route_private_route_hop(route, next_private_route) + // Make next PrivateRoute and pass it on + self.process_route_private_route_hop( + route.operation, + route_hop.node, + route.safety_route.public_key, + PrivateRoute { + public_key: private_route.public_key, + hop_count: private_route.hop_count - 1, + hops: route_hop + .next_hop + .map(|rhd| PrivateRouteHops::Data(rhd)) + .unwrap_or(PrivateRouteHops::Empty), + }, + ) .await?; - } else { - // Ensure hop count == 0 - if private_route.hop_count != 0 { - return Err(RPCError::protocol("route should be at the end")); } + PrivateRouteHops::Empty => { + // Ensure hop count == 0 + if private_route.hop_count != 0 { + return Err(RPCError::protocol("route should be at the end")); + } - // No hops left, time to process the routed operation - self.process_routed_operation( - detail, - route.operation, - &route.safety_route, - private_route, - )?; + // No hops left, time to process the routed operation + self.process_routed_operation( + detail, + route.operation, + &route.safety_route, + private_route, + )?; + } } } } diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index a3c04603..f3cac31c 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -3,15 +3,21 @@ use super::*; //////////////////////////////////////////////////////////////////////////////////////////////////// // Compiled Privacy Objects +/// An encrypted private/safety route hop #[derive(Clone, Debug)] pub struct RouteHopData { + /// The nonce used in the encryption ENC(Xn,DH(PKn,SKapr)) pub nonce: Nonce, + /// The encrypted blob pub blob: Vec, } +/// How to find a route node #[derive(Clone, Debug)] pub enum RouteNode { + /// Route node is optimized, no contact method information as this node id has been seen before NodeId(NodeId), + /// Route node with full contact method information to ensure the peer is reachable PeerInfo(PeerInfo), } impl fmt::Display for RouteNode { @@ -27,17 +33,33 @@ impl fmt::Display for RouteNode { } } +/// An unencrypted private/safety route hop #[derive(Clone, Debug)] pub struct RouteHop { + /// The location of the hop pub node: RouteNode, + /// The encrypted blob to pass to the next hop as its data (None for stubs) pub next_hop: Option, } +/// The kind of hops a private route can have +#[derive(Clone, Debug)] +pub enum PrivateRouteHops { + /// The first hop of a private route, unencrypted, route_hops == total hop count + FirstHop(RouteHop), + /// Private route internal node. Has > 0 private route hops left but < total hop count + Data(RouteHopData), + /// Private route has ended (hop count = 0) + Empty, +} + +/// A private route for receiver privacy #[derive(Clone, Debug)] pub struct PrivateRoute { + /// The public key used for the entire route pub public_key: DHTKey, pub hop_count: u8, - pub first_hop: Option, + pub hops: PrivateRouteHops, } impl PrivateRoute { @@ -46,7 +68,7 @@ impl PrivateRoute { Self { public_key, hop_count: 0, - first_hop: None, + hops: PrivateRouteHops::Empty, } } /// Stub route is the form used when no privacy is required, but you need to specify the destination for a safety route @@ -54,29 +76,12 @@ impl PrivateRoute { Self { public_key, hop_count: 1, - first_hop: Some(RouteHop { + hops: PrivateRouteHops::FirstHop(RouteHop { node, next_hop: None, }), } } - /// Switch from full node info to simple node id - /// Only simplified single hop, primarily useful for stubs - /// Published routes with >= 1 hops should be in NodeId form already, as they have - /// already been connectivity-verified by the time the route is published - pub fn simplify(self) -> Self { - Self { - public_key: self.public_key, - hop_count: self.hop_count, - first_hop: self.first_hop.map(|h| RouteHop { - node: match h.node { - RouteNode::NodeId(ni) => RouteNode::NodeId(ni), - RouteNode::PeerInfo(pi) => RouteNode::NodeId(pi.node_id), - }, - next_hop: h.next_hop, - }), - } - } } impl fmt::Display for PrivateRoute { @@ -86,10 +91,16 @@ impl fmt::Display for PrivateRoute { "PR({:?}+{}{})", self.public_key, self.hop_count, - if let Some(first_hop) = &self.first_hop { - format!("->{}", first_hop.node) - } else { - "".to_owned() + match &self.hops { + PrivateRouteHops::FirstHop(fh) => { + format!("->{}", fh.node) + } + PrivateRouteHops::Data(_) => { + "->?".to_owned() + } + PrivateRouteHops::Empty => { + "".to_owned() + } } ) } From 688995ed0dfffe87c2c87572148daafd50e05cb3 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 16 Nov 2022 12:49:53 -0500 Subject: [PATCH 59/67] pr work --- Cargo.lock | 2 + veilid-cli/Cargo.toml | 3 +- veilid-cli/src/client_api_connection.rs | 26 +- veilid-cli/src/command_processor.rs | 4 + veilid-cli/src/ui.rs | 33 +- veilid-core/src/core_context.rs | 4 +- veilid-core/src/rpc_processor/rpc_route.rs | 56 ++- .../src/tests/common/test_veilid_config.rs | 15 +- veilid-core/src/veilid_api/mod.rs | 13 + veilid-core/src/veilid_config.rs | 390 ++++++++++++++---- veilid-flutter/lib/veilid.dart | 45 +- veilid-flutter/lib/veilid_ffi.dart | 2 +- veilid-server/Cargo.toml | 1 + veilid-server/proto/veilid-client.capnp | 2 +- veilid-server/src/client_api.rs | 27 +- veilid-server/src/cmdline.rs | 1 + veilid-server/src/server.rs | 32 +- 17 files changed, 534 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61e52043..df0254c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5551,6 +5551,7 @@ dependencies = [ "flexi_logger", "futures", "hex", + "json", "log", "parking_lot 0.12.1", "serde", @@ -5714,6 +5715,7 @@ dependencies = [ "flume", "futures-util", "hostname", + "json", "lazy_static", "nix 0.25.0", "opentelemetry", diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index eb4cc69e..fa613fde 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -43,7 +43,8 @@ flexi_logger = { version = "^0", features = ["use_chrono_for_offset"] } thiserror = "^1" crossbeam-channel = "^0" hex = "^0" -veilid-core = { path = "../veilid-core", default_features = false} +veilid-core = { path = "../veilid-core", default_features = false } +json = "^0" [dev-dependencies] serial_test = "^0" diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index 36276179..03b8343c 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -89,6 +89,9 @@ impl veilid_client::Server for VeilidClientImpl { VeilidUpdate::Network(network) => { self.comproc.update_network_status(network); } + VeilidUpdate::Config(config) => { + self.comproc.update_config(config); + } VeilidUpdate::Shutdown => self.comproc.update_shutdown(), } @@ -101,6 +104,7 @@ struct ClientApiConnectionInner { connect_addr: Option, disconnector: Option>, server: Option>>, + server_settings: Option, disconnect_requested: bool, cancel_eventual: Eventual, } @@ -120,6 +124,7 @@ impl ClientApiConnection { connect_addr: None, disconnector: None, server: None, + server_settings: None, disconnect_requested: false, cancel_eventual: Eventual::new(), })), @@ -141,7 +146,7 @@ impl ClientApiConnection { let mut inner = self.inner.borrow_mut(); inner.comproc.update_attachment(veilid_state.attachment); inner.comproc.update_network_status(veilid_state.network); - + inner.comproc.update_config(veilid_state.config); Ok(()) } @@ -209,6 +214,13 @@ impl ClientApiConnection { .map_err(|e| format!("failed to get deserialize veilid state: {}", e))?; self.process_veilid_state(veilid_state).await?; + // Save server settings + let server_settings = response + .get_settings() + .map_err(|e| format!("failed to get initial veilid server settings: {}", e))? + .to_owned(); + self.inner.borrow_mut().server_settings = Some(server_settings.clone()); + // Don't drop the registration, doing so will remove the client // object mapping from the server which we need for the update backchannel @@ -219,9 +231,10 @@ impl ClientApiConnection { res.map_err(|e| format!("client RPC system error: {}", e)) } - async fn handle_connection(&mut self) -> Result<(), String> { + async fn handle_connection(&mut self, connect_addr: SocketAddr) -> Result<(), String> { trace!("ClientApiConnection::handle_connection"); - let connect_addr = self.inner.borrow().connect_addr.unwrap(); + + self.inner.borrow_mut().connect_addr = Some(connect_addr); // Connect the TCP socket let stream = TcpStream::connect(connect_addr) .await @@ -263,9 +276,11 @@ impl ClientApiConnection { // Drop the server and disconnector too (if we still have it) let mut inner = self.inner.borrow_mut(); let disconnect_requested = inner.disconnect_requested; + inner.server_settings = None; inner.server = None; inner.disconnector = None; inner.disconnect_requested = false; + inner.connect_addr = None; if !disconnect_requested { // Connection lost @@ -456,9 +471,7 @@ impl ClientApiConnection { pub async fn connect(&mut self, connect_addr: SocketAddr) -> Result<(), String> { trace!("ClientApiConnection::connect"); // Save the address to connect to - self.inner.borrow_mut().connect_addr = Some(connect_addr); - - self.handle_connection().await + self.handle_connection(connect_addr).await } // End Client API connection @@ -469,7 +482,6 @@ impl ClientApiConnection { Some(d) => { self.inner.borrow_mut().disconnect_requested = true; d.await.unwrap(); - self.inner.borrow_mut().connect_addr = None; } None => { debug!("disconnector doesn't exist"); diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index ed5f3dd8..e2457d77 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -388,6 +388,7 @@ reply - reply to an AppCall not handled directly by the server // called by client_api_connection // calls into ui //////////////////////////////////////////// + pub fn update_attachment(&mut self, attachment: veilid_core::VeilidStateAttachment) { self.inner_mut().ui.set_attachment_state(attachment.state); } @@ -400,6 +401,9 @@ reply - reply to an AppCall not handled directly by the server network.peers, ); } + pub fn update_config(&mut self, config: veilid_core::VeilidStateConfig) { + self.inner_mut().ui.set_config(config.config) + } pub fn update_log(&mut self, log: veilid_core::VeilidLog) { self.inner().ui.add_node_event(format!( diff --git a/veilid-cli/src/ui.rs b/veilid-cli/src/ui.rs index b1c93065..017152bf 100644 --- a/veilid-cli/src/ui.rs +++ b/veilid-cli/src/ui.rs @@ -55,6 +55,7 @@ struct UIState { network_down_up: Dirty<(f32, f32)>, connection_state: Dirty, peers_state: Dirty>, + node_id: Dirty, } impl UIState { @@ -65,6 +66,7 @@ impl UIState { network_down_up: Dirty::new((0.0, 0.0)), connection_state: Dirty::new(ConnectionState::Disconnected), peers_state: Dirty::new(Vec::new()), + node_id: Dirty::new("".to_owned()), } } } @@ -214,6 +216,11 @@ impl UI { }); } + fn node_events_panel( + s: &mut Cursive, + ) -> ViewRef>>>> { + s.find_name("node-events-panel").unwrap() + } fn command_line(s: &mut Cursive) -> ViewRef { s.find_name("command-line").unwrap() } @@ -572,6 +579,12 @@ impl UI { } } + fn refresh_main_titlebar(s: &mut Cursive) { + let mut main_window = UI::node_events_panel(s); + let inner = Self::inner_mut(s); + main_window.set_title(format!("Node: {}", inner.ui_state.node_id.get())); + } + fn refresh_statusbar(s: &mut Cursive) { let mut statusbar = UI::status_bar(s); @@ -634,6 +647,7 @@ impl UI { let mut refresh_button_attach = false; let mut refresh_connection_dialog = false; let mut refresh_peers = false; + let mut refresh_main_titlebar = false; if inner.ui_state.attachment_state.take_dirty() { refresh_statusbar = true; refresh_button_attach = true; @@ -654,6 +668,9 @@ impl UI { if inner.ui_state.peers_state.take_dirty() { refresh_peers = true; } + if inner.ui_state.node_id.take_dirty() { + refresh_main_titlebar = true; + } drop(inner); @@ -669,6 +686,9 @@ impl UI { if refresh_peers { Self::refresh_peers(s); } + if refresh_main_titlebar { + Self::refresh_main_titlebar(s); + } } //////////////////////////////////////////////////////////////////////////// @@ -722,7 +742,8 @@ impl UI { .full_screen(), ) .title_position(HAlign::Left) - .title("Node Events"); + .title("Node Events") + .with_name("node-events-panel"); let peers_table_view = PeersTableView::new() .column(PeerTableColumn::NodeId, "Node Id", |c| c.width(43)) @@ -839,6 +860,16 @@ impl UI { inner.ui_state.peers_state.set(peers); let _ = inner.cb_sink.send(Box::new(UI::update_cb)); } + pub fn set_config(&mut self, config: VeilidConfigInner) { + let mut inner = self.inner.borrow_mut(); + inner.ui_state.node_id.set( + config + .network + .node_id + .map(|x| x.encode()) + .unwrap_or("".to_owned()), + ); + } pub fn set_connection_state(&mut self, state: ConnectionState) { let mut inner = self.inner.borrow_mut(); inner.ui_state.connection_state.set(state); diff --git a/veilid-core/src/core_context.rs b/veilid-core/src/core_context.rs index 698a6a84..6f39129a 100644 --- a/veilid-core/src/core_context.rs +++ b/veilid-core/src/core_context.rs @@ -177,7 +177,7 @@ impl VeilidCoreContext { // Set up config from callback trace!("setup config with callback"); let mut config = VeilidConfig::new(); - config.setup(config_callback)?; + config.setup(config_callback, update_callback.clone())?; Self::new_common(update_callback, config).await } @@ -190,7 +190,7 @@ impl VeilidCoreContext { // Set up config from callback trace!("setup config with json"); let mut config = VeilidConfig::new(); - config.setup_from_json(config_json)?; + config.setup_from_json(config_json, update_callback.clone())?; Self::new_common(update_callback, config).await } diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 3cffa743..4267d43d 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -271,6 +271,34 @@ impl RPCProcessor { ) } } + #[instrument(level = "trace", skip_all, err)] + pub(crate) async fn process_private_route_first_hop( + &self, + operation: RoutedOperation, + sr_pubkey: DHTKey, + private_route: &PrivateRoute, + ) -> Result<(), RPCError> { + let PrivateRouteHops::FirstHop(pr_first_hop) = &private_route.hops else { + return Err(RPCError::protocol("switching from safety route to private route requires first hop")); + }; + + // Switching to private route from safety route + self.process_route_private_route_hop( + operation, + pr_first_hop.node.clone(), + sr_pubkey, + PrivateRoute { + public_key: private_route.public_key, + hop_count: private_route.hop_count - 1, + hops: pr_first_hop + .next_hop + .clone() + .map(|rhd| PrivateRouteHops::Data(rhd)) + .unwrap_or(PrivateRouteHops::Empty), + }, + ) + .await + } #[instrument(level = "trace", skip(self, msg), err)] pub(crate) async fn process_route(&self, msg: RPCMessage) -> Result<(), RPCError> { @@ -332,24 +360,11 @@ impl RPCProcessor { decode_private_route(&pr_reader)? }; - // Get the next hop node ref - let PrivateRouteHops::FirstHop(pr_first_hop) = private_route.hops else { - return Err(RPCError::protocol("switching from safety route to private route requires first hop")); - }; - - // Switching to private route from safety route - self.process_route_private_route_hop( + // Switching from full safety route to private route first hop + self.process_private_route_first_hop( route.operation, - pr_first_hop.node, route.safety_route.public_key, - PrivateRoute { - public_key: private_route.public_key, - hop_count: private_route.hop_count - 1, - hops: pr_first_hop - .next_hop - .map(|rhd| PrivateRouteHops::Data(rhd)) - .unwrap_or(PrivateRouteHops::Empty), - }, + &private_route, ) .await?; } else if blob_tag == 0 { @@ -361,6 +376,7 @@ impl RPCProcessor { decode_route_hop(&rh_reader)? }; + // Continue the full safety route with another hop self.process_route_safety_route_hop(route, route_hop) .await?; } else { @@ -372,7 +388,13 @@ impl RPCProcessor { // See if we have a hop, if not, we are at the end of the private route match &private_route.hops { PrivateRouteHops::FirstHop(_) => { - return Err(RPCError::protocol("should not have first hop here")); + // Safety route was a stub, start with the beginning of the private route + self.process_private_route_first_hop( + route.operation, + route.safety_route.public_key, + private_route, + ) + .await?; } PrivateRouteHops::Data(route_hop_data) => { // Decrypt the blob with DEC(nonce, DH(the PR's public key, this hop's secret) diff --git a/veilid-core/src/tests/common/test_veilid_config.rs b/veilid-core/src/tests/common/test_veilid_config.rs index 7304c45d..4e56d4e7 100644 --- a/veilid-core/src/tests/common/test_veilid_config.rs +++ b/veilid-core/src/tests/common/test_veilid_config.rs @@ -156,13 +156,12 @@ cfg_if! { } } +fn update_callback(update: VeilidUpdate) { + println!("update_callback: {:?}", update); +} + pub fn setup_veilid_core() -> (UpdateCallback, ConfigCallback) { - ( - Arc::new(move |veilid_update: VeilidUpdate| { - println!("update_callback: {:?}", veilid_update); - }), - Arc::new(config_callback), - ) + (Arc::new(update_callback), Arc::new(config_callback)) } fn config_callback(key: String) -> ConfigCallbackReturn { @@ -268,7 +267,7 @@ fn config_callback(key: String) -> ConfigCallbackReturn { pub fn get_config() -> VeilidConfig { let mut vc = VeilidConfig::new(); - match vc.setup(Arc::new(config_callback)) { + match vc.setup(Arc::new(config_callback), Arc::new(update_callback)) { Ok(()) => (), Err(e) => { error!("Error: {}", e); @@ -280,7 +279,7 @@ pub fn get_config() -> VeilidConfig { pub async fn test_config() { let mut vc = VeilidConfig::new(); - match vc.setup(Arc::new(config_callback)) { + match vc.setup(Arc::new(config_callback), Arc::new(update_callback)) { Ok(()) => (), Err(e) => { error!("Error: {}", e); diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 73b0e817..0512a391 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -319,6 +319,14 @@ pub struct VeilidStateNetwork { pub peers: Vec, } +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] +#[archive_attr(repr(C), derive(CheckBytes))] +pub struct VeilidStateConfig { + pub config: VeilidConfigInner, +} + #[derive(Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] #[archive_attr(repr(u8), derive(CheckBytes))] #[serde(tag = "kind")] @@ -328,6 +336,7 @@ pub enum VeilidUpdate { AppCall(VeilidAppCall), Attachment(VeilidStateAttachment), Network(VeilidStateNetwork), + Config(VeilidStateConfig), Shutdown, } @@ -336,6 +345,7 @@ pub enum VeilidUpdate { pub struct VeilidState { pub attachment: VeilidStateAttachment, pub network: VeilidStateNetwork, + pub config: VeilidStateConfig, } ///////////////////////////////////////////////////////////////////////////////////////////////////// @@ -2679,13 +2689,16 @@ impl VeilidAPI { pub async fn get_state(&self) -> Result { let attachment_manager = self.attachment_manager()?; let network_manager = attachment_manager.network_manager(); + let config = self.config()?; let attachment = attachment_manager.get_veilid_state(); let network = network_manager.get_veilid_state(); + let config = config.get_veilid_state(); Ok(VeilidState { attachment, network, + config, }) } diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index 87ea38fb..79c37d22 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -1,5 +1,6 @@ use crate::xx::*; use crate::*; +use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; use serde::*; //////////////////////////////////////////////////////////////////////////////////////////////// @@ -16,7 +17,18 @@ pub type ConfigCallback = Arc ConfigCallbackReturn + Send + Sy /// url: 'https://localhost:5150' /// ``` /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigHTTPS { pub enabled: bool, pub listen_address: String, @@ -34,7 +46,18 @@ pub struct VeilidConfigHTTPS { /// url: 'https://localhost:5150' /// ``` /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigHTTP { pub enabled: bool, pub listen_address: String, @@ -48,7 +71,18 @@ pub struct VeilidConfigHTTP { /// /// To be implemented... /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigApplication { pub https: VeilidConfigHTTPS, pub http: VeilidConfigHTTP, @@ -64,7 +98,18 @@ pub struct VeilidConfigApplication { /// public_address: '' /// ``` /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigUDP { pub enabled: bool, pub socket_pool_size: u32, @@ -82,7 +127,18 @@ pub struct VeilidConfigUDP { /// listen_address: ':5150' /// public_address: '' /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigTCP { pub connect: bool, pub listen: bool, @@ -102,7 +158,18 @@ pub struct VeilidConfigTCP { /// path: 'ws' /// url: 'ws://localhost:5150/ws' /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigWS { pub connect: bool, pub listen: bool, @@ -123,7 +190,18 @@ pub struct VeilidConfigWS { /// path: 'ws' /// url: '' /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigWSS { pub connect: bool, pub listen: bool, @@ -140,7 +218,18 @@ pub struct VeilidConfigWSS { /// All protocols are available by default, and the Veilid node will /// sort out which protocol is used for each peer connection. /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigProtocol { pub udp: VeilidConfigUDP, pub tcp: VeilidConfigTCP, @@ -156,7 +245,18 @@ pub struct VeilidConfigProtocol { /// private_key_path: /path/to/private/key /// connection_initial_timeout_ms: 2000 /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigTLS { pub certificate_path: String, pub private_key_path: String, @@ -165,7 +265,18 @@ pub struct VeilidConfigTLS { /// Configure the Distributed Hash Table (DHT) /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigDHT { pub resolve_node_timeout_ms: Option, pub resolve_node_count: u32, @@ -184,7 +295,18 @@ pub struct VeilidConfigDHT { /// Configure RPC /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigRPC { pub concurrency: u32, pub queue_size: u32, @@ -197,7 +319,18 @@ pub struct VeilidConfigRPC { /// Configure the network routing table /// -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigRoutingTable { pub limit_over_attached: u32, pub limit_fully_attached: u32, @@ -206,7 +339,18 @@ pub struct VeilidConfigRoutingTable { pub limit_attached_weak: u32, } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigNetwork { pub connection_initial_timeout_ms: u32, pub connection_inactivity_timeout_ms: u32, @@ -233,19 +377,52 @@ pub struct VeilidConfigNetwork { pub protocol: VeilidConfigProtocol, } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigTableStore { pub directory: String, pub delete: bool, } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigBlockStore { pub directory: String, pub delete: bool, } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigProtectedStore { pub allow_insecure_fallback: bool, pub always_use_insecure_storage: bool, @@ -253,7 +430,18 @@ pub struct VeilidConfigProtectedStore { pub delete: bool, } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigCapabilities { pub protocol_udp: bool, pub protocol_connect_tcp: bool, @@ -264,7 +452,18 @@ pub struct VeilidConfigCapabilities { pub protocol_accept_wss: bool, } -#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive( + Clone, + Copy, + PartialEq, + Eq, + Debug, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub enum VeilidConfigLogLevel { Off, Error, @@ -322,7 +521,18 @@ impl Default for VeilidConfigLogLevel { } } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive( + Default, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, +)] pub struct VeilidConfigInner { pub program_name: String, pub namespace: String, @@ -338,6 +548,7 @@ pub struct VeilidConfigInner { /// Veilid is configured #[derive(Clone)] pub struct VeilidConfig { + update_cb: Option, inner: Arc>, } @@ -362,23 +573,29 @@ impl VeilidConfig { pub fn new() -> Self { Self { + update_cb: None, inner: Arc::new(RwLock::new(Self::new_inner())), } } - pub fn setup_from_json(&mut self, config: String) -> Result<(), VeilidAPIError> { - { - let mut inner = self.inner.write(); + pub fn setup_from_json( + &mut self, + config: String, + update_cb: UpdateCallback, + ) -> Result<(), VeilidAPIError> { + self.update_cb = Some(update_cb); + + self.with_mut(|inner| { *inner = serde_json::from_str(&config).map_err(VeilidAPIError::generic)?; - } - - // Validate settings - self.validate()?; - - Ok(()) + Ok(()) + }) } - pub fn setup(&mut self, cb: ConfigCallback) -> Result<(), VeilidAPIError> { + pub fn setup( + &mut self, + cb: ConfigCallback, + update_cb: UpdateCallback, + ) -> Result<(), VeilidAPIError> { macro_rules! get_config { ($key:expr) => { let keyname = &stringify!($key)[6..]; @@ -389,8 +606,9 @@ impl VeilidConfig { })?; }; } - { - let mut inner = self.inner.write(); + + self.update_cb = Some(update_cb); + self.with_mut(|inner| { get_config!(inner.program_name); get_config!(inner.namespace); get_config!(inner.capabilities.protocol_udp); @@ -482,19 +700,44 @@ impl VeilidConfig { get_config!(inner.network.protocol.wss.listen_address); get_config!(inner.network.protocol.wss.path); get_config!(inner.network.protocol.wss.url); - } - // Validate settings - self.validate()?; + Ok(()) + }) + } - Ok(()) + pub fn get_veilid_state(&self) -> VeilidStateConfig { + let inner = self.inner.read(); + VeilidStateConfig { + config: inner.clone(), + } } pub fn get(&self) -> RwLockReadGuard { self.inner.read() } - pub fn get_mut(&self) -> RwLockWriteGuard { - self.inner.write() + pub fn with_mut(&self, f: F) -> Result + where + F: FnOnce(&mut VeilidConfigInner) -> Result, + { + let (out, config) = { + let inner = &mut *self.inner.write(); + // Edit a copy + let mut editedinner = inner.clone(); + // Make changes + let out = f(&mut editedinner)?; + // Validate + Self::validate(&mut editedinner)?; + // Commit changes + *inner = editedinner.clone(); + (out, editedinner) + }; + + // Send configuration update to clients + if let Some(update_cb) = &self.update_cb { + update_cb(VeilidUpdate::Config(VeilidStateConfig { config })); + } + + Ok(out) } pub fn get_key_json(&self, key: &str) -> Result { @@ -521,47 +764,43 @@ impl VeilidConfig { } } pub fn set_key_json(&self, key: &str, value: &str) -> Result<(), VeilidAPIError> { - let mut c = self.get_mut(); + self.with_mut(|c| { + // Split key into path parts + let keypath: Vec<&str> = key.split('.').collect(); - // Split key into path parts - let keypath: Vec<&str> = key.split('.').collect(); + // Convert value into jsonvalue + let newval = json::parse(value).map_err(VeilidAPIError::generic)?; - // Convert value into jsonvalue - let newval = json::parse(value).map_err(VeilidAPIError::generic)?; + // Generate json from whole config + let jc = serde_json::to_string(&*c).map_err(VeilidAPIError::generic)?; + let mut jvc = json::parse(&jc).map_err(VeilidAPIError::generic)?; - // Generate json from whole config - let jc = serde_json::to_string(&*c).map_err(VeilidAPIError::generic)?; - let mut jvc = json::parse(&jc).map_err(VeilidAPIError::generic)?; - - // Find requested subkey - let newconfigstring = if let Some((objkeyname, objkeypath)) = keypath.split_last() { - // Replace subkey - let mut out = &mut jvc; - for k in objkeypath { - if !out.has_key(*k) { - apibail_parse!(format!("invalid subkey in key '{}'", key), k); + // Find requested subkey + let newconfigstring = if let Some((objkeyname, objkeypath)) = keypath.split_last() { + // Replace subkey + let mut out = &mut jvc; + for k in objkeypath { + if !out.has_key(*k) { + apibail_parse!(format!("invalid subkey in key '{}'", key), k); + } + out = &mut out[*k]; } - out = &mut out[*k]; - } - if !out.has_key(objkeyname) { - apibail_parse!(format!("invalid subkey in key '{}'", key), objkeyname); - } - out[*objkeyname] = newval; - jvc.to_string() - } else { - newval.to_string() - }; - // Generate and validate new config - let mut newconfig = VeilidConfig::new(); - newconfig.setup_from_json(newconfigstring)?; - // Replace whole config - *c = newconfig.get().clone(); - Ok(()) + if !out.has_key(objkeyname) { + apibail_parse!(format!("invalid subkey in key '{}'", key), objkeyname); + } + out[*objkeyname] = newval; + jvc.to_string() + } else { + newval.to_string() + }; + + // Generate new config + *c = serde_json::from_str(&newconfigstring).map_err(VeilidAPIError::generic)?; + Ok(()) + }) } - fn validate(&self) -> Result<(), VeilidAPIError> { - let inner = self.inner.read(); - + fn validate(inner: &VeilidConfigInner) -> Result<(), VeilidAPIError> { if inner.program_name.is_empty() { apibail_generic!("Program name must not be empty in 'program_name'"); } @@ -731,8 +970,11 @@ impl VeilidConfig { .await .map_err(VeilidAPIError::internal)?; - self.inner.write().network.node_id = Some(node_id); - self.inner.write().network.node_id_secret = Some(node_id_secret); + self.with_mut(|c| { + c.network.node_id = Some(node_id); + c.network.node_id_secret = Some(node_id_secret); + Ok(()) + })?; trace!("init_node_id complete"); diff --git a/veilid-flutter/lib/veilid.dart b/veilid-flutter/lib/veilid.dart index 52ff4a78..4c539782 100644 --- a/veilid-flutter/lib/veilid.dart +++ b/veilid-flutter/lib/veilid.dart @@ -1262,6 +1262,10 @@ abstract class VeilidUpdate { { return VeilidUpdateNetwork(state: VeilidStateNetwork.fromJson(json)); } + case "Config": + { + return VeilidUpdateConfig(state: VeilidStateConfig.fromJson(json)); + } default: { throw VeilidAPIExceptionInternal( @@ -1363,6 +1367,19 @@ class VeilidUpdateNetwork implements VeilidUpdate { } } +class VeilidUpdateConfig implements VeilidUpdate { + final VeilidStateConfig state; + // + VeilidUpdateConfig({required this.state}); + + @override + Map get json { + var jsonRep = state.json; + jsonRep['kind'] = "Config"; + return jsonRep; + } +} + ////////////////////////////////////// /// VeilidStateAttachment @@ -1413,19 +1430,43 @@ class VeilidStateNetwork { } } +////////////////////////////////////// +/// VeilidStateConfig + +class VeilidStateConfig { + final Map config; + + VeilidStateConfig({ + required this.config, + }); + + VeilidStateConfig.fromJson(Map json) + : config = jsonDecode(json['config']); + + Map get json { + return {'config': jsonEncode(config)}; + } +} + ////////////////////////////////////// /// VeilidState class VeilidState { final VeilidStateAttachment attachment; final VeilidStateNetwork network; + final VeilidStateConfig config; VeilidState.fromJson(Map json) : attachment = VeilidStateAttachment.fromJson(json['attachment']), - network = VeilidStateNetwork.fromJson(json['network']); + network = VeilidStateNetwork.fromJson(json['network']), + config = VeilidStateConfig.fromJson(json['config']); Map get json { - return {'attachment': attachment.json, 'network': network.json}; + return { + 'attachment': attachment.json, + 'network': network.json, + 'config': config.json + }; } } diff --git a/veilid-flutter/lib/veilid_ffi.dart b/veilid-flutter/lib/veilid_ffi.dart index 23ce9ef6..f4b87f41 100644 --- a/veilid-flutter/lib/veilid_ffi.dart +++ b/veilid-flutter/lib/veilid_ffi.dart @@ -18,7 +18,7 @@ final _path = Platform.isWindows : Platform.isMacOS ? 'lib$_base.dylib' : 'lib$_base.so'; -late final _dylib = +final _dylib = Platform.isIOS ? DynamicLibrary.process() : DynamicLibrary.open(_path); // Linkage for initialization diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index a78a2026..9a31f676 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -44,6 +44,7 @@ cfg-if = "^1" serde = "^1" serde_derive = "^1" serde_yaml = "^0" +json = "^0" futures-util = { version = "^0", default_features = false, features = ["alloc"] } url = "^2" ctrlc = "^3" diff --git a/veilid-server/proto/veilid-client.capnp b/veilid-server/proto/veilid-client.capnp index e12b587a..469a3b09 100644 --- a/veilid-server/proto/veilid-client.capnp +++ b/veilid-server/proto/veilid-client.capnp @@ -10,7 +10,7 @@ struct ApiResult @0x8111724bdb812929 { interface Registration @0xdd45f30a7c22e391 {} interface VeilidServer @0xcb2c699f14537f94 { - register @0 (veilidClient :VeilidClient) -> (registration :Registration, state :Text); + register @0 (veilidClient :VeilidClient) -> (registration :Registration, state :Text, settings :Text); debug @1 (command :Text) -> (result :ApiResult); attach @2 () -> (result :ApiResult); detach @3 () -> (result :ApiResult); diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index 700ea79e..86d3a645 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -1,3 +1,4 @@ +use crate::settings::*; use crate::tools::*; use crate::veilid_client_capnp::*; use crate::veilid_logs::VeilidLogs; @@ -81,18 +82,24 @@ impl registration::Server for RegistrationImpl {} struct VeilidServerImpl { veilid_api: veilid_core::VeilidAPI, veilid_logs: VeilidLogs, + settings: Settings, next_id: u64, pub registration_map: Rc>, } impl VeilidServerImpl { #[instrument(level = "trace", skip_all)] - pub fn new(veilid_api: veilid_core::VeilidAPI, veilid_logs: VeilidLogs) -> Self { + pub fn new( + veilid_api: veilid_core::VeilidAPI, + veilid_logs: VeilidLogs, + settings: Settings, + ) -> Self { Self { next_id: 0, registration_map: Rc::new(RefCell::new(RegistrationMap::new())), veilid_api, veilid_logs, + settings, } } } @@ -115,6 +122,7 @@ impl veilid_server::Server for VeilidServerImpl { ); let veilid_api = self.veilid_api.clone(); + let settings = self.settings.clone(); let registration = capnp_rpc::new_client(RegistrationImpl::new( self.next_id, self.registration_map.clone(), @@ -132,6 +140,14 @@ impl veilid_server::Server for VeilidServerImpl { res.set_registration(registration); res.set_state(&state); + let settings = &*settings.read(); + let settings_json_string = serialize_json(settings); + let mut settings_json = json::parse(&settings_json_string) + .map_err(|e| ::capnp::Error::failed(format!("{:?}", e)))?; + settings_json["core"]["network"].remove("node_id_secret"); + let safe_settings_json = settings_json.to_string(); + res.set_settings(&safe_settings_json); + Ok(()) }) } @@ -265,6 +281,7 @@ type ClientApiAllFuturesJoinHandle = struct ClientApiInner { veilid_api: veilid_core::VeilidAPI, veilid_logs: VeilidLogs, + settings: Settings, registration_map: Rc>, stop: Option, join_handle: Option, @@ -276,11 +293,16 @@ pub struct ClientApi { impl ClientApi { #[instrument(level = "trace", skip_all)] - pub fn new(veilid_api: veilid_core::VeilidAPI, veilid_logs: VeilidLogs) -> Rc { + pub fn new( + veilid_api: veilid_core::VeilidAPI, + veilid_logs: VeilidLogs, + settings: Settings, + ) -> Rc { Rc::new(Self { inner: RefCell::new(ClientApiInner { veilid_api, veilid_logs, + settings, registration_map: Rc::new(RefCell::new(RegistrationMap::new())), stop: Some(StopSource::new()), join_handle: None, @@ -427,6 +449,7 @@ impl ClientApi { let veilid_server_impl = VeilidServerImpl::new( self.inner.borrow().veilid_api.clone(), self.inner.borrow().veilid_logs.clone(), + self.inner.borrow().settings.clone(), ); self.inner.borrow_mut().registration_map = veilid_server_impl.registration_map.clone(); diff --git a/veilid-server/src/cmdline.rs b/veilid-server/src/cmdline.rs index ee40f09a..1ef31d99 100644 --- a/veilid-server/src/cmdline.rs +++ b/veilid-server/src/cmdline.rs @@ -301,6 +301,7 @@ pub fn process_command_line() -> EyreResult<(Settings, ArgMatches)> { settingsrw.core.network.bootstrap_nodes = bootstrap_list; } + #[cfg(feature = "rt-tokio")] if matches.occurrences_of("console") != 0 { settingsrw.logging.console.enabled = true; } diff --git a/veilid-server/src/server.rs b/veilid-server/src/server.rs index f5f2af2f..ca936a30 100644 --- a/veilid-server/src/server.rs +++ b/veilid-server/src/server.rs @@ -4,6 +4,8 @@ use crate::tools::*; use crate::veilid_logs::*; use crate::*; use flume::{unbounded, Receiver, Sender}; +use futures_util::select; +use futures_util::FutureExt; use lazy_static::*; use parking_lot::Mutex; use std::sync::Arc; @@ -70,7 +72,8 @@ pub async fn run_veilid_server_internal( // Start client api if one is requested let mut capi = if settingsr.client_api.enabled && matches!(server_mode, ServerMode::Normal) { - let some_capi = client_api::ClientApi::new(veilid_api.clone(), veilid_logs.clone()); + let some_capi = + client_api::ClientApi::new(veilid_api.clone(), veilid_logs.clone(), settings.clone()); some_capi .clone() .run(settingsr.client_api.listen_address.addrs.clone()); @@ -85,12 +88,29 @@ pub async fn run_veilid_server_internal( // Process all updates let capi2 = capi.clone(); + let mut shutdown_switch = { + let shutdown_switch_locked = SHUTDOWN_SWITCH.lock(); + (*shutdown_switch_locked).as_ref().map(|ss| ss.instance()) + } + .unwrap() + .fuse(); let update_receiver_jh = spawn_local(async move { - while let Ok(change) = receiver.recv_async().await { - if let Some(capi) = &capi2 { - // Handle state changes on main thread for capnproto rpc - capi.clone().handle_update(change); - } + loop { + select! { + res = receiver.recv_async() => { + if let Ok(change) = res { + if let Some(capi) = &capi2 { + // Handle state changes on main thread for capnproto rpc + capi.clone().handle_update(change); + } + } else { + break; + } + } + _ = shutdown_switch => { + break; + } + }; } }); From c9595d85498d2c2711d5aa309d575a78ff10444e Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 19 Nov 2022 19:05:43 -0500 Subject: [PATCH 60/67] pr fixes --- veilid-core/src/crypto/key.rs | 39 +++++++++-------- veilid-core/src/crypto/tests/test_dht_key.rs | 4 +- .../src/routing_table/route_spec_store.rs | 43 ++++++++++++++----- veilid-core/src/rpc_processor/rpc_route.rs | 41 +++++++++--------- veilid-core/src/veilid_api/privacy.rs | 27 ++++++++++++ 5 files changed, 102 insertions(+), 52 deletions(-) diff --git a/veilid-core/src/crypto/key.rs b/veilid-core/src/crypto/key.rs index 90f57297..c5468084 100644 --- a/veilid-core/src/crypto/key.rs +++ b/veilid-core/src/crypto/key.rs @@ -175,7 +175,8 @@ macro_rules! byte_array_type { impl fmt::Display for $name { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", String::from(self)) + //write!(f, "{}", String::from(self)) + write!(f, "{}", self.encode()) } } @@ -189,12 +190,13 @@ macro_rules! byte_array_type { impl From<&$name> for String { fn from(value: &$name) -> Self { - let mut s = String::new(); - for n in 0..($size / 8) { - let b: [u8; 8] = value.bytes[n * 8..(n + 1) * 8].try_into().unwrap(); - s.push_str(hex::encode(b).as_str()); - } - s + // let mut s = String::new(); + // for n in 0..($size / 8) { + // let b: [u8; 8] = value.bytes[n * 8..(n + 1) * 8].try_into().unwrap(); + // s.push_str(hex::encode(b).as_str()); + // } + // s + value.encode() } } @@ -208,17 +210,18 @@ macro_rules! byte_array_type { impl TryFrom<&str> for $name { type Error = VeilidAPIError; fn try_from(value: &str) -> Result { - let mut out = $name::default(); - if value == "" { - return Ok(out); - } - if value.len() != ($size * 2) { - apibail_generic!(concat!(stringify!($name), " is incorrect length")); - } - match hex::decode_to_slice(value, &mut out.bytes) { - Ok(_) => Ok(out), - Err(err) => Err(VeilidAPIError::generic(err)), - } + // let mut out = $name::default(); + // if value == "" { + // return Ok(out); + // } + // if value.len() != ($size * 2) { + // apibail_generic!(concat!(stringify!($name), " is incorrect length")); + // } + // match hex::decode_to_slice(value, &mut out.bytes) { + // Ok(_) => Ok(out), + // Err(err) => Err(VeilidAPIError::generic(err)), + // } + Self::try_decode(value) } } }; diff --git a/veilid-core/src/crypto/tests/test_dht_key.rs b/veilid-core/src/crypto/tests/test_dht_key.rs index 3b6ded61..d6876ac3 100644 --- a/veilid-core/src/crypto/tests/test_dht_key.rs +++ b/veilid-core/src/crypto/tests/test_dht_key.rs @@ -138,11 +138,11 @@ pub async fn test_key_conversions() { // Assert string roundtrip assert_eq!(String::from(&dht_key2_back), dht_key2_string); - assert!(key::DHTKey::try_from("") == Ok(key::DHTKey::default())); - assert!(key::DHTKeySecret::try_from("") == Ok(key::DHTKeySecret::default())); // These conversions should fail assert!(key::DHTKey::try_from("whatever").is_err()); assert!(key::DHTKeySecret::try_from("whatever").is_err()); + assert!(key::DHTKey::try_from("").is_err()); + assert!(key::DHTKeySecret::try_from("").is_err()); assert!(key::DHTKey::try_from(" ").is_err()); assert!(key::DHTKeySecret::try_from(" ").is_err()); assert!(key::DHTKey::try_from( diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 8d1c0cf3..bf0fc2db 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -456,6 +456,11 @@ impl RouteSpecStore { bail!("Not allocating route longer than max route hop count"); } + // Get relay node id if we have one + let opt_relay_id = rti + .relay_node(RoutingDomain::PublicInternet) + .map(|nr| nr.node_id()); + // Get list of all nodes, and sort them for selection let cur_ts = intf::get_timestamp(); let filter = Box::new( @@ -466,6 +471,13 @@ impl RouteSpecStore { } let v = v.unwrap(); + // Exclude our relay if we have one + if let Some(relay_id) = opt_relay_id { + if k == relay_id { + return false; + } + } + // Exclude nodes on our local network let on_local_network = v.with(rti, |_rti, e| { e.node_info(RoutingDomain::LocalNetwork).is_some() @@ -714,7 +726,8 @@ impl RouteSpecStore { return Ok(None); } // Validate signatures to ensure the route was handled by the nodes and not messed with - for (hop_n, hop_public_key) in rsd.hops.iter().enumerate() { + // This is in private route (reverse) order as we are receiving over the route + for (hop_n, hop_public_key) in rsd.hops.iter().rev().enumerate() { // The last hop is not signed, as the whole packet is signed if hop_n == signatures.len() { // Verify the node we received the routed operation from is the last hop in our route @@ -843,7 +856,7 @@ impl RouteSpecStore { pub fn compile_safety_route( &self, safety_selection: SafetySelection, - private_route: PrivateRoute, + mut private_route: PrivateRoute, ) -> EyreResult> { let inner = &mut *self.inner.lock(); let routing_table = self.unlocked_inner.routing_table.clone(); @@ -854,15 +867,17 @@ impl RouteSpecStore { if pr_hopcount > max_route_hop_count { bail!("private route hop count too long"); } - let PrivateRouteHops::FirstHop(pr_first_hop) = &private_route.hops else { - bail!("compiled private route should have first hop"); - }; - // See if we are using a safety route, if not, short circuit this operation let safety_spec = match safety_selection { + // Safety route spec to use + SafetySelection::Safe(safety_spec) => safety_spec, + // Safety route stub with the node's public key as the safety route key since it's the 0th hop SafetySelection::Unsafe(sequencing) => { - // Safety route stub with the node's public key as the safety route key since it's the 0th hop - let opt_first_hop = match &pr_first_hop.node { + let Some(pr_first_hop_node) = private_route.pop_first_hop() else { + bail!("compiled private route should have first hop"); + }; + + let opt_first_hop = match pr_first_hop_node { RouteNode::NodeId(id) => rti.lookup_node_ref(routing_table.clone(), id.key), RouteNode::PeerInfo(pi) => rti.register_node_with_signed_node_info( routing_table.clone(), @@ -889,7 +904,10 @@ impl RouteSpecStore { first_hop, })); } - SafetySelection::Safe(safety_spec) => safety_spec, + }; + + let PrivateRouteHops::FirstHop(pr_first_hop) = &private_route.hops else { + bail!("compiled private route should have first hop"); }; // Get the safety route to use from the spec @@ -930,6 +948,7 @@ impl RouteSpecStore { // Each loop mutates 'nonce', and 'blob_data' let mut nonce = Crypto::get_random_nonce(); let crypto = routing_table.network_manager().crypto(); + // Forward order (safety route), but inside-out for h in (1..safety_rsd.hops.len()).rev() { // Get blob to encrypt for next hop blob_data = { @@ -1132,7 +1151,8 @@ impl RouteSpecStore { let crypto = routing_table.network_manager().crypto(); // Loop for each hop let hop_count = rsd.hops.len(); - for h in (0..hop_count).rev() { + // iterate hops in private route order (reverse, but inside out) + for h in 0..hop_count { let nonce = Crypto::get_random_nonce(); let blob_data = { @@ -1178,7 +1198,8 @@ impl RouteSpecStore { let private_route = PrivateRoute { public_key: key.clone(), - hop_count: hop_count.try_into().unwrap(), + // add hop for 'FirstHop' + hop_count: (hop_count + 1).try_into().unwrap(), hops: PrivateRouteHops::FirstHop(route_hop), }; Ok(private_route) diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 4267d43d..ac2513db 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -110,16 +110,6 @@ impl RPCProcessor { } }?; - if !matches!(next_private_route.hops, PrivateRouteHops::Empty) { - // Sign the operation if this is not our last hop - // as the last hop is already signed by the envelope - let node_id = self.routing_table.node_id(); - let node_id_secret = self.routing_table.node_id_secret(); - let sig = sign(&node_id, &node_id_secret, &routed_operation.data) - .map_err(RPCError::internal)?; - routed_operation.signatures.push(sig); - } - // Pass along the route let next_hop_route = RPCOperationRoute { safety_route: SafetyRoute { @@ -313,7 +303,7 @@ impl RPCProcessor { }; // Get the statement - let route = match msg.operation.into_kind() { + let mut route = match msg.operation.into_kind() { RPCOperationKind::Statement(s) => match s.into_detail() { RPCStatementDetail::Route(s) => s, _ => panic!("not a route statement"), @@ -333,25 +323,24 @@ impl RPCProcessor { match route.safety_route.hops { // There is a safety route hop SafetyRouteHops::Data(ref d) => { - // See if this is last hop in safety route, if so, we're decoding a PrivateRoute not a RouteHop - let (blob_tag, blob_data) = if let Some(b) = d.blob.last() { - (*b, &d.blob[0..d.blob.len() - 1]) - } else { - return Err(RPCError::protocol("no bytes in blob")); - }; - // Decrypt the blob with DEC(nonce, DH(the SR's public key, this hop's secret) let node_id_secret = self.routing_table.node_id_secret(); let dh_secret = self .crypto .cached_dh(&route.safety_route.public_key, &node_id_secret) .map_err(RPCError::protocol)?; - let dec_blob_data = Crypto::decrypt_aead(blob_data, &d.nonce, &dh_secret, None) + let mut dec_blob_data = Crypto::decrypt_aead(&d.blob, &d.nonce, &dh_secret, None) .map_err(RPCError::protocol)?; + + // See if this is last hop in safety route, if so, we're decoding a PrivateRoute not a RouteHop + let Some(dec_blob_tag) = dec_blob_data.pop() else { + return Err(RPCError::protocol("no bytes in blob")); + }; + let dec_blob_reader = RPCMessageData::new(dec_blob_data).get_reader()?; // Decode the blob appropriately - if blob_tag == 1 { + if dec_blob_tag == 1 { // PrivateRoute let private_route = { let pr_reader = dec_blob_reader @@ -367,7 +356,7 @@ impl RPCProcessor { &private_route, ) .await?; - } else if blob_tag == 0 { + } else if dec_blob_tag == 0 { // RouteHop let route_hop = { let rh_reader = dec_blob_reader @@ -425,6 +414,16 @@ impl RPCProcessor { return Err(RPCError::protocol("route should not be at the end")); } + // Sign the operation if this is not our last hop + // as the last hop is already signed by the envelope + if route_hop.next_hop.is_some() { + let node_id = self.routing_table.node_id(); + let node_id_secret = self.routing_table.node_id_secret(); + let sig = sign(&node_id, &node_id_secret, &route.operation.data) + .map_err(RPCError::internal)?; + route.operation.signatures.push(sig); + } + // Make next PrivateRoute and pass it on self.process_route_private_route_hop( route.operation, diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/veilid_api/privacy.rs index f3cac31c..352b716b 100644 --- a/veilid-core/src/veilid_api/privacy.rs +++ b/veilid-core/src/veilid_api/privacy.rs @@ -82,6 +82,32 @@ impl PrivateRoute { }), } } + + /// Remove the first unencrypted hop if possible + pub fn pop_first_hop(&mut self) -> Option { + match &mut self.hops { + PrivateRouteHops::FirstHop(first_hop) => { + let first_hop_node = first_hop.node.clone(); + + // Reduce hop count + if self.hop_count > 0 { + self.hop_count -= 1; + } else { + error!("hop count should not be 0 for first hop"); + } + + // Go to next hop + self.hops = match first_hop.next_hop.take() { + Some(rhd) => PrivateRouteHops::Data(rhd), + None => PrivateRouteHops::Empty, + }; + + return Some(first_hop_node); + } + PrivateRouteHops::Data(_) => return None, + PrivateRouteHops::Empty => return None, + } + } } impl fmt::Display for PrivateRoute { @@ -123,6 +149,7 @@ pub struct SafetyRoute { impl SafetyRoute { pub fn new_stub(public_key: DHTKey, private_route: PrivateRoute) -> Self { + assert!(matches!(private_route.hops, PrivateRouteHops::Data(_))); Self { public_key, hop_count: 0, From 5453eb2e92535aacd3aa82384a819e61941d8c5a Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 20 Nov 2022 21:03:34 -0500 Subject: [PATCH 61/67] upnp improvements --- .../native/network_class_discovery.rs | 128 ++++++++++++------ 1 file changed, 83 insertions(+), 45 deletions(-) diff --git a/veilid-core/src/network_manager/native/network_class_discovery.rs b/veilid-core/src/network_manager/native/network_class_discovery.rs index 7a2daa08..8c24687d 100644 --- a/veilid-core/src/network_manager/native/network_class_discovery.rs +++ b/veilid-core/src/network_manager/native/network_class_discovery.rs @@ -3,6 +3,10 @@ use futures_util::stream::FuturesUnordered; use futures_util::FutureExt; use stop_token::future::FutureExt as StopTokenFutureExt; +const PORT_MAP_VALIDATE_TRY_COUNT: usize = 3; +const PORT_MAP_VALIDATE_DELAY_MS: u32 = 500; +const PORT_MAP_TRY_COUNT: usize = 3; + struct DetectedPublicDialInfo { dial_info: DialInfo, class: DialInfoClass, @@ -220,6 +224,84 @@ impl DiscoveryContext { out } + #[instrument(level = "trace", skip(self), ret)] + async fn try_upnp_port_mapping(&self) -> Option { + let (pt, llpt, at, external_address_1, node_1, local_port) = { + let inner = self.inner.lock(); + let pt = inner.protocol_type.unwrap(); + let llpt = pt.low_level_protocol_type(); + let at = inner.address_type.unwrap(); + let external_address_1 = inner.external_1_address.unwrap(); + let node_1 = inner.node_1.as_ref().unwrap().clone(); + let local_port = self.net.get_local_port(pt); + (pt, llpt, at, external_address_1, node_1, local_port) + }; + + let mut tries = 0; + loop { + tries += 1; + + // Attempt a port mapping. If this doesn't succeed, it's not going to + let Some(mapped_external_address) = self + .net + .unlocked_inner + .igd_manager + .map_any_port(llpt, at, local_port, Some(external_address_1.to_ip_addr())) + .await else + { + return None; + }; + + // Make dial info from the port mapping + let external_mapped_dial_info = + self.make_dial_info(SocketAddress::from_socket_addr(mapped_external_address), pt); + + // Attempt to validate the port mapping + let mut validate_tries = 0; + loop { + validate_tries += 1; + + // Ensure people can reach us. If we're firewalled off, this is useless + if self + .validate_dial_info(node_1.clone(), external_mapped_dial_info.clone(), false) + .await + { + return Some(external_mapped_dial_info); + } + + if validate_tries == PORT_MAP_VALIDATE_TRY_COUNT { + log_net!(debug "UPNP port mapping succeeded but port {}/{} is still unreachable.\nretrying\n", + local_port, match llpt { + LowLevelProtocolType::UDP => "udp", + LowLevelProtocolType::TCP => "tcp", + }); + intf::sleep(PORT_MAP_VALIDATE_DELAY_MS).await + } else { + break; + } + } + + // Release the mapping if we're still unreachable + let _ = self + .net + .unlocked_inner + .igd_manager + .unmap_port(llpt, at, external_address_1.port()) + .await; + + if tries == PORT_MAP_TRY_COUNT { + warn!("UPNP port mapping succeeded but port {}/{} is still unreachable.\nYou may need to add a local firewall allowed port on this machine.\n", + local_port, match llpt { + LowLevelProtocolType::UDP => "udp", + LowLevelProtocolType::TCP => "tcp", + } + ); + break; + } + } + None + } + #[instrument(level = "trace", skip(self), ret)] async fn try_port_mapping(&self) -> Option { let (enable_upnp, _enable_natpmp) = { @@ -228,51 +310,7 @@ impl DiscoveryContext { }; if enable_upnp { - let (pt, llpt, at, external_address_1, node_1, local_port) = { - let inner = self.inner.lock(); - let pt = inner.protocol_type.unwrap(); - let llpt = pt.low_level_protocol_type(); - let at = inner.address_type.unwrap(); - let external_address_1 = inner.external_1_address.unwrap(); - let node_1 = inner.node_1.as_ref().unwrap().clone(); - let local_port = self.net.get_local_port(pt); - (pt, llpt, at, external_address_1, node_1, local_port) - }; - - if let Some(mapped_external_address) = self - .net - .unlocked_inner - .igd_manager - .map_any_port(llpt, at, local_port, Some(external_address_1.to_ip_addr())) - .await - { - // make dial info from the port mapping - let external_mapped_dial_info = self - .make_dial_info(SocketAddress::from_socket_addr(mapped_external_address), pt); - - // ensure people can reach us. if we're firewalled off, this is useless - if self - .validate_dial_info(node_1.clone(), external_mapped_dial_info.clone(), false) - .await - { - return Some(external_mapped_dial_info); - } else { - warn!("UPNP port mapping succeeded but port {}/{} is still unreachable.\nYou may need to add a local firewall allowed port on this machine.\n", - local_port, match llpt { - LowLevelProtocolType::UDP => "udp", - LowLevelProtocolType::TCP => "tcp", - } - ); - - // release the mapping if we're still unreachable - let _ = self - .net - .unlocked_inner - .igd_manager - .unmap_port(llpt, at, external_address_1.port()) - .await; - } - } + return self.try_upnp_port_mapping().await; } None From 749ba91b8bf411bf70f611a405a69b6b4a210440 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 20 Nov 2022 22:30:45 -0500 Subject: [PATCH 62/67] checkpoint --- veilid-core/src/rpc_processor/mod.rs | 17 ++++++-- veilid-core/src/rpc_processor/rpc_app_call.rs | 11 +++-- .../src/rpc_processor/rpc_app_message.rs | 9 ++-- .../src/rpc_processor/rpc_cancel_tunnel.rs | 4 +- .../src/rpc_processor/rpc_complete_tunnel.rs | 7 +++- .../src/rpc_processor/rpc_find_block.rs | 7 +++- .../src/rpc_processor/rpc_find_node.rs | 11 +++-- .../src/rpc_processor/rpc_get_value.rs | 7 +++- .../src/rpc_processor/rpc_node_info_update.rs | 19 ++++++--- .../src/rpc_processor/rpc_return_receipt.rs | 35 +++++++--------- veilid-core/src/rpc_processor/rpc_route.rs | 41 +++++++++++-------- .../src/rpc_processor/rpc_set_value.rs | 5 ++- veilid-core/src/rpc_processor/rpc_signal.rs | 18 ++++---- .../src/rpc_processor/rpc_start_tunnel.rs | 7 +++- veilid-core/src/rpc_processor/rpc_status.rs | 19 +++++---- .../src/rpc_processor/rpc_supply_block.rs | 7 +++- .../rpc_processor/rpc_validate_dial_info.rs | 25 +++++++---- .../src/rpc_processor/rpc_value_changed.rs | 5 ++- .../src/rpc_processor/rpc_watch_value.rs | 7 +++- 19 files changed, 165 insertions(+), 96 deletions(-) diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 460323ad..ec18eb91 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -846,7 +846,10 @@ impl RPCProcessor { ////////////////////////////////////////////////////////////////////// #[instrument(level = "trace", skip(self, encoded_msg), err)] - async fn process_rpc_message(&self, encoded_msg: RPCMessageEncoded) -> Result<(), RPCError> { + async fn process_rpc_message( + &self, + encoded_msg: RPCMessageEncoded, + ) -> Result, RPCError> { // Decode operation appropriately based on header detail let msg = match &encoded_msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => { @@ -990,11 +993,19 @@ impl RPCProcessor { let rpc_worker_span = span!(parent: None, Level::TRACE, "rpc_worker recv"); // xxx: causes crash (Missing otel data span extensions) // rpc_worker_span.follows_from(span_id); - let _ = self + let res = match self .process_rpc_message(msg) .instrument(rpc_worker_span) .await - .map_err(logthru_rpc!("couldn't process rpc message")); + { + Err(e) => { + log_rpc!(error "couldn't process rpc message: {}", e); + continue; + } + + Ok(v) => v, + }; + network_result_value_or_log!(debug res => {}); } } diff --git a/veilid-core/src/rpc_processor/rpc_app_call.rs b/veilid-core/src/rpc_processor/rpc_app_call.rs index 8f57c4af..3fc60207 100644 --- a/veilid-core/src/rpc_processor/rpc_app_call.rs +++ b/veilid-core/src/rpc_processor/rpc_app_call.rs @@ -39,8 +39,11 @@ impl RPCProcessor { ))) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_app_call_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_app_call_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // Get the question let app_call_q = match msg.operation.kind() { RPCOperationKind::Question(q) => match q.detail() { @@ -76,7 +79,7 @@ impl RPCProcessor { TimeoutOr::Timeout => { // No message sent on timeout, but this isn't an error log_rpc!(debug "App call timed out for id {}", id); - return Ok(()); + return Ok(NetworkResult::timeout()); } TimeoutOr::Value(v) => v, }; @@ -89,7 +92,7 @@ impl RPCProcessor { .answer(msg, RPCAnswer::new(RPCAnswerDetail::AppCallA(app_call_a))) .await?; tracing::Span::current().record("res", &tracing::field::display(res)); - Ok(()) + Ok(res) } /// Exposed to API for apps to return app call answers diff --git a/veilid-core/src/rpc_processor/rpc_app_message.rs b/veilid-core/src/rpc_processor/rpc_app_message.rs index 0b069280..29f152b4 100644 --- a/veilid-core/src/rpc_processor/rpc_app_message.rs +++ b/veilid-core/src/rpc_processor/rpc_app_message.rs @@ -18,8 +18,11 @@ impl RPCProcessor { Ok(NetworkResult::value(())) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_app_message(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_app_message( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // Get the statement let app_message = match msg.operation.into_kind() { RPCOperationKind::Statement(s) => match s.into_detail() { @@ -37,6 +40,6 @@ impl RPCProcessor { message, })); - Ok(()) + Ok(NetworkResult::value(())) } } diff --git a/veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs b/veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs index 4d7d046b..04a7a332 100644 --- a/veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs +++ b/veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs @@ -1,8 +1,8 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] - pub(crate) async fn process_cancel_tunnel_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + pub(crate) async fn process_cancel_tunnel_q(&self, msg: RPCMessage) -> Result, RPCError> { // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_cancel_tunnel_q")) diff --git a/veilid-core/src/rpc_processor/rpc_complete_tunnel.rs b/veilid-core/src/rpc_processor/rpc_complete_tunnel.rs index 66975971..9270921b 100644 --- a/veilid-core/src/rpc_processor/rpc_complete_tunnel.rs +++ b/veilid-core/src/rpc_processor/rpc_complete_tunnel.rs @@ -1,8 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] - pub(crate) async fn process_complete_tunnel_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + pub(crate) async fn process_complete_tunnel_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_complete_tunnel_q")) } diff --git a/veilid-core/src/rpc_processor/rpc_find_block.rs b/veilid-core/src/rpc_processor/rpc_find_block.rs index 8c0dcf94..4d7ca42b 100644 --- a/veilid-core/src/rpc_processor/rpc_find_block.rs +++ b/veilid-core/src/rpc_processor/rpc_find_block.rs @@ -1,8 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] - pub(crate) async fn process_find_block_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + pub(crate) async fn process_find_block_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_find_block_q")) } diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index c75d820b..4f80df2d 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -66,13 +66,16 @@ impl RPCProcessor { ))) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] - pub(crate) async fn process_find_node_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + pub(crate) async fn process_find_node_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // Ensure this never came over a private route, safety route is okay though match &msg.header.detail { RPCMessageHeaderDetail::Direct(_) | RPCMessageHeaderDetail::SafetyRouted(_) => {} RPCMessageHeaderDetail::PrivateRouted(_) => { - return Err(RPCError::protocol( + return Ok(NetworkResult::invalid_message( "not processing find node request over private route", )) } @@ -130,6 +133,6 @@ impl RPCProcessor { .answer(msg, RPCAnswer::new(RPCAnswerDetail::FindNodeA(find_node_a))) .await?; tracing::Span::current().record("res", &tracing::field::display(res)); - Ok(()) + Ok(res) } } diff --git a/veilid-core/src/rpc_processor/rpc_get_value.rs b/veilid-core/src/rpc_processor/rpc_get_value.rs index 23fb5175..65aad72f 100644 --- a/veilid-core/src/rpc_processor/rpc_get_value.rs +++ b/veilid-core/src/rpc_processor/rpc_get_value.rs @@ -1,8 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] - pub(crate) async fn process_get_value_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + pub(crate) async fn process_get_value_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { //tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_get_value_q")) } diff --git a/veilid-core/src/rpc_processor/rpc_node_info_update.rs b/veilid-core/src/rpc_processor/rpc_node_info_update.rs index b264baeb..a676f6b2 100644 --- a/veilid-core/src/rpc_processor/rpc_node_info_update.rs +++ b/veilid-core/src/rpc_processor/rpc_node_info_update.rs @@ -30,12 +30,17 @@ impl RPCProcessor { Ok(NetworkResult::value(())) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_node_info_update(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_node_info_update( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { let detail = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { - return Err(RPCError::protocol("node_info_update must be direct")); + return Ok(NetworkResult::invalid_message( + "node_info_update must be direct", + )); } }; let sender_node_id = detail.envelope.get_sender_id(); @@ -52,8 +57,10 @@ impl RPCProcessor { // Update our routing table with signed node info if !self.filter_node_info(routing_domain, &node_info_update.signed_node_info) { - log_rpc!(debug "node info doesn't belong in {:?} routing domain: {}", routing_domain, sender_node_id); - return Ok(()); + return Ok(NetworkResult::invalid_message(format!( + "node info doesn't belong in {:?} routing domain: {}", + routing_domain, sender_node_id + ))); } self.routing_table().register_node_with_signed_node_info( @@ -63,6 +70,6 @@ impl RPCProcessor { false, ); - Ok(()) + Ok(NetworkResult::value(())) } } diff --git a/veilid-core/src/rpc_processor/rpc_return_receipt.rs b/veilid-core/src/rpc_processor/rpc_return_receipt.rs index c4ae3b4c..a5537369 100644 --- a/veilid-core/src/rpc_processor/rpc_return_receipt.rs +++ b/veilid-core/src/rpc_processor/rpc_return_receipt.rs @@ -20,8 +20,11 @@ impl RPCProcessor { Ok(NetworkResult::value(())) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_return_receipt(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_return_receipt( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // Get the statement let RPCOperationReturnReceipt { receipt } = match msg.operation.into_kind() { RPCOperationKind::Statement(s) => match s.into_detail() { @@ -34,30 +37,22 @@ impl RPCProcessor { // Handle it let network_manager = self.network_manager(); - match msg.header.detail { + let res = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => { - network_result_value_or_log!(debug - network_manager - .handle_in_band_receipt(receipt, detail.peer_noderef) - .await => {} - ); + network_manager + .handle_in_band_receipt(receipt, detail.peer_noderef) + .await } RPCMessageHeaderDetail::SafetyRouted(_) => { - network_result_value_or_log!(debug - network_manager - .handle_safety_receipt(receipt) - .await => {} - ); + network_manager.handle_safety_receipt(receipt).await } RPCMessageHeaderDetail::PrivateRouted(detail) => { - network_result_value_or_log!(debug - network_manager - .handle_private_receipt(receipt, detail.private_route) - .await => {} - ); + network_manager + .handle_private_receipt(receipt, detail.private_route) + .await } - } + }; - Ok(()) + Ok(res) } } diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index ac2513db..1e360ab7 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -1,12 +1,14 @@ use super::*; +xxx go through and convert errs to networkresult + impl RPCProcessor { #[instrument(level = "trace", skip_all, err)] async fn process_route_safety_route_hop( &self, route: RPCOperationRoute, route_hop: RouteHop, - ) -> Result<(), RPCError> { + ) -> Result, RPCError> { // Make sure hop count makes sense if route.safety_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { return Err(RPCError::protocol( @@ -72,11 +74,11 @@ impl RPCProcessor { #[instrument(level = "trace", skip_all, err)] async fn process_route_private_route_hop( &self, - mut routed_operation: RoutedOperation, + routed_operation: RoutedOperation, next_route_node: RouteNode, safety_route_public_key: DHTKey, next_private_route: PrivateRoute, - ) -> Result<(), RPCError> { + ) -> Result, RPCError> { // Make sure hop count makes sense if next_private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { return Err(RPCError::protocol( @@ -143,7 +145,7 @@ impl RPCProcessor { detail: RPCMessageHeaderDetailDirect, routed_operation: RoutedOperation, safety_route: &SafetyRoute, - ) -> Result<(), RPCError> { + ) -> Result, RPCError> { // Get sequencing preference let sequencing = if detail .connection_descriptor @@ -187,7 +189,7 @@ impl RPCProcessor { routed_operation: RoutedOperation, safety_route: &SafetyRoute, private_route: &PrivateRoute, - ) -> Result<(), RPCError> { + ) -> Result, RPCError> { // Get sender id let sender_id = detail.envelope.get_sender_id(); @@ -233,7 +235,7 @@ impl RPCProcessor { routed_operation: RoutedOperation, safety_route: &SafetyRoute, private_route: &PrivateRoute, - ) -> Result<(), RPCError> { + ) -> Result, RPCError> { // Make sure hop count makes sense if safety_route.hop_count != 0 { return Err(RPCError::protocol( @@ -267,7 +269,7 @@ impl RPCProcessor { operation: RoutedOperation, sr_pubkey: DHTKey, private_route: &PrivateRoute, - ) -> Result<(), RPCError> { + ) -> Result, RPCError> { let PrivateRouteHops::FirstHop(pr_first_hop) = &private_route.hops else { return Err(RPCError::protocol("switching from safety route to private route requires first hop")); }; @@ -290,13 +292,16 @@ impl RPCProcessor { .await } - #[instrument(level = "trace", skip(self, msg), err)] - pub(crate) async fn process_route(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), ret, err)] + pub(crate) async fn process_route( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // Get header detail, must be direct and not inside a route itself let detail = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { - return Err(RPCError::protocol( + return Ok(NetworkResult::invalid_message( "route operation can not be inside route", )) } @@ -314,7 +319,7 @@ impl RPCProcessor { // Process routed operation version // xxx switch this to a Crypto trait factory method per issue#140 if route.operation.version != MAX_CRYPTO_VERSION { - return Err(RPCError::protocol( + return Ok(NetworkResult::invalid_message( "routes operation crypto is not valid version", )); } @@ -334,7 +339,7 @@ impl RPCProcessor { // See if this is last hop in safety route, if so, we're decoding a PrivateRoute not a RouteHop let Some(dec_blob_tag) = dec_blob_data.pop() else { - return Err(RPCError::protocol("no bytes in blob")); + return Ok(NetworkResult::invalid_message("no bytes in blob")); }; let dec_blob_reader = RPCMessageData::new(dec_blob_data).get_reader()?; @@ -369,7 +374,7 @@ impl RPCProcessor { self.process_route_safety_route_hop(route, route_hop) .await?; } else { - return Err(RPCError::protocol("invalid blob tag")); + return Ok(NetworkResult::invalid_message("invalid blob tag")); } } // No safety route left, now doing private route @@ -411,7 +416,9 @@ impl RPCProcessor { // Ensure hop count > 0 if private_route.hop_count == 0 { - return Err(RPCError::protocol("route should not be at the end")); + return Ok(NetworkResult::invalid_message( + "route should not be at the end", + )); } // Sign the operation if this is not our last hop @@ -443,7 +450,9 @@ impl RPCProcessor { PrivateRouteHops::Empty => { // Ensure hop count == 0 if private_route.hop_count != 0 { - return Err(RPCError::protocol("route should be at the end")); + return Ok(NetworkResult::invalid_message( + "route should be at the end", + )); } // No hops left, time to process the routed operation @@ -458,6 +467,6 @@ impl RPCProcessor { } } - Ok(()) + Ok(NetworkResult::value(())) } } diff --git a/veilid-core/src/rpc_processor/rpc_set_value.rs b/veilid-core/src/rpc_processor/rpc_set_value.rs index 8ff11f49..9f89e13c 100644 --- a/veilid-core/src/rpc_processor/rpc_set_value.rs +++ b/veilid-core/src/rpc_processor/rpc_set_value.rs @@ -2,7 +2,10 @@ use super::*; impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_set_value_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + pub(crate) async fn process_set_value_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_set_value_q")) } diff --git a/veilid-core/src/rpc_processor/rpc_signal.rs b/veilid-core/src/rpc_processor/rpc_signal.rs index 0180fb6c..592c2ed8 100644 --- a/veilid-core/src/rpc_processor/rpc_signal.rs +++ b/veilid-core/src/rpc_processor/rpc_signal.rs @@ -31,14 +31,17 @@ impl RPCProcessor { Ok(NetworkResult::value(())) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_signal(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_signal( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // Can't allow anything other than direct packets here, as handling reverse connections // or anything like via signals over private routes would deanonymize the route match &msg.header.detail { RPCMessageHeaderDetail::Direct(_) => {} RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { - return Err(RPCError::protocol("signal must be direct")); + return Ok(NetworkResult::invalid_message("signal must be direct")); } }; @@ -53,14 +56,11 @@ impl RPCProcessor { // Handle it let network_manager = self.network_manager(); - network_result_value_or_log!(debug network_manager + let res = network_manager .handle_signal(signal.signal_info) .await - .map_err(RPCError::network)? => { - return Ok(()); - } - ); + .map_err(RPCError::network)?; - Ok(()) + Ok(res) } } diff --git a/veilid-core/src/rpc_processor/rpc_start_tunnel.rs b/veilid-core/src/rpc_processor/rpc_start_tunnel.rs index 616f7fb0..95218748 100644 --- a/veilid-core/src/rpc_processor/rpc_start_tunnel.rs +++ b/veilid-core/src/rpc_processor/rpc_start_tunnel.rs @@ -1,8 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_start_tunnel_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_start_tunnel_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_start_tunnel_q")) } diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 32e13a43..4bbce6e1 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -179,8 +179,11 @@ impl RPCProcessor { Ok(NetworkResult::value(Answer::new(latency, opt_sender_info))) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), err)] - pub(crate) async fn process_status_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + pub(crate) async fn process_status_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // Get the question let status_q = match msg.operation.kind() { RPCOperationKind::Question(q) => match q.detail() { @@ -200,14 +203,16 @@ impl RPCProcessor { match routing_domain { RoutingDomain::PublicInternet => { if !matches!(node_status, NodeStatus::PublicInternet(_)) { - log_rpc!(debug "node status doesn't match PublicInternet routing domain"); - return Ok(()); + return Ok(NetworkResult::invalid_message( + "node status doesn't match PublicInternet routing domain", + )); } } RoutingDomain::LocalNetwork => { if !matches!(node_status, NodeStatus::LocalNetwork(_)) { - log_rpc!(debug "node status doesn't match LocalNetwork routing domain"); - return Ok(()); + return Ok(NetworkResult::invalid_message( + "node status doesn't match LocalNetwork routing domain", + )); } } } @@ -249,6 +254,6 @@ impl RPCProcessor { .answer(msg, RPCAnswer::new(RPCAnswerDetail::StatusA(status_a))) .await?; tracing::Span::current().record("res", &tracing::field::display(res)); - Ok(()) + Ok(res) } } diff --git a/veilid-core/src/rpc_processor/rpc_supply_block.rs b/veilid-core/src/rpc_processor/rpc_supply_block.rs index 2de9a72e..9a98f3ff 100644 --- a/veilid-core/src/rpc_processor/rpc_supply_block.rs +++ b/veilid-core/src/rpc_processor/rpc_supply_block.rs @@ -1,8 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_supply_block_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_supply_block_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_supply_block_q")) diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 14e3c905..954b8d44 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -54,12 +54,17 @@ impl RPCProcessor { } } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_validate_dial_info(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_validate_dial_info( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { let detail = match msg.header.detail { RPCMessageHeaderDetail::Direct(detail) => detail, RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => { - return Err(RPCError::protocol("validate_dial_info must be direct")); + return Ok(NetworkResult::invalid_message( + "validate_dial_info must be direct", + )); } }; @@ -117,7 +122,7 @@ impl RPCProcessor { // Find nodes matching filter to redirect this to let peers = routing_table.find_fast_public_nodes_filtered(node_count, filters); if peers.is_empty() { - return Err(RPCError::internal(format!( + return Ok(NetworkResult::no_connection_other(format!( "no peers able to reach dialinfo '{:?}'", dial_info ))); @@ -139,13 +144,17 @@ impl RPCProcessor { // Send the validate_dial_info request // This can only be sent directly, as relays can not validate dial info - network_result_value_or_log!(debug self.statement(Destination::direct(peer), statement) + let res = network_result_value_or_log!(debug self.statement(Destination::direct(peer), statement) .await? => { - return Ok(()); + continue; } ); + return Ok(NetworkResult::value(())); } - return Ok(()); + + return Ok(NetworkResult::no_connection_other( + "could not redirect, no peers were reachable", + )); }; // Otherwise send a return receipt directly @@ -156,6 +165,6 @@ impl RPCProcessor { .await .map_err(RPCError::network)?; - Ok(()) + Ok(NetworkResult::value(())) } } diff --git a/veilid-core/src/rpc_processor/rpc_value_changed.rs b/veilid-core/src/rpc_processor/rpc_value_changed.rs index dfa631ea..6c46d4d7 100644 --- a/veilid-core/src/rpc_processor/rpc_value_changed.rs +++ b/veilid-core/src/rpc_processor/rpc_value_changed.rs @@ -2,7 +2,10 @@ use super::*; impl RPCProcessor { #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_value_changed(&self, msg: RPCMessage) -> Result<(), RPCError> { + pub(crate) async fn process_value_changed( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_value_changed")) diff --git a/veilid-core/src/rpc_processor/rpc_watch_value.rs b/veilid-core/src/rpc_processor/rpc_watch_value.rs index b55f2a8d..8526e037 100644 --- a/veilid-core/src/rpc_processor/rpc_watch_value.rs +++ b/veilid-core/src/rpc_processor/rpc_watch_value.rs @@ -1,8 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] - pub(crate) async fn process_watch_value_q(&self, msg: RPCMessage) -> Result<(), RPCError> { + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_watch_value_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_watch_value_q")) } From e98fae711ec15202257f8e208377df74624d088c Mon Sep 17 00:00:00 2001 From: John Smith Date: Mon, 21 Nov 2022 20:21:46 -0500 Subject: [PATCH 63/67] cli --- veilid-cli/src/ui.rs | 16 ++- veilid-core/src/rpc_processor/mod.rs | 11 +- veilid-core/src/rpc_processor/rpc_app_call.rs | 7 +- .../src/rpc_processor/rpc_app_message.rs | 4 +- .../src/rpc_processor/rpc_cancel_tunnel.rs | 9 +- .../src/rpc_processor/rpc_complete_tunnel.rs | 3 +- .../src/rpc_processor/rpc_find_block.rs | 3 +- .../src/rpc_processor/rpc_find_node.rs | 9 +- .../src/rpc_processor/rpc_get_value.rs | 3 +- .../src/rpc_processor/rpc_node_info_update.rs | 21 +++- veilid-core/src/rpc_processor/rpc_route.rs | 113 +++++++++--------- .../src/rpc_processor/rpc_set_value.rs | 2 +- veilid-core/src/rpc_processor/rpc_signal.rs | 10 +- .../src/rpc_processor/rpc_start_tunnel.rs | 1 - veilid-core/src/rpc_processor/rpc_status.rs | 9 +- .../src/rpc_processor/rpc_supply_block.rs | 2 - .../rpc_processor/rpc_validate_dial_info.rs | 2 +- .../src/rpc_processor/rpc_value_changed.rs | 2 - .../src/rpc_processor/rpc_watch_value.rs | 1 - veilid-core/src/xx/network_result.rs | 57 ++++++++- 20 files changed, 169 insertions(+), 116 deletions(-) diff --git a/veilid-cli/src/ui.rs b/veilid-cli/src/ui.rs index 017152bf..2d07a345 100644 --- a/veilid-cli/src/ui.rs +++ b/veilid-cli/src/ui.rs @@ -7,6 +7,7 @@ use cursive::event::*; use cursive::theme::*; use cursive::traits::*; use cursive::utils::markup::StyledString; +use cursive::view::ScrollStrategy; use cursive::views::*; use cursive::Cursive; use cursive::CursiveRunnable; @@ -215,7 +216,13 @@ impl UI { UI::setup_quit_handler(s); }); } - + fn clear_handler(siv: &mut Cursive) { + cursive_flexi_logger_view::clear_log(); + UI::update_cb(siv); + } + fn node_events(s: &mut Cursive) -> ViewRef { + s.find_name("node-events").unwrap() + } fn node_events_panel( s: &mut Cursive, ) -> ViewRef>>>> { @@ -737,8 +744,12 @@ impl UI { // Create layouts let node_events_view = Panel::new( - FlexiLoggerView::new_scrollable() + FlexiLoggerView::new() .with_name("node-events") + .scrollable() + .scroll_x(true) + .scroll_y(true) + .scroll_strategy(ScrollStrategy::StickToBottom) .full_screen(), ) .title_position(HAlign::Left) @@ -822,6 +833,7 @@ impl UI { UI::setup_colors(&mut siv, &mut inner, settings); UI::setup_quit_handler(&mut siv); + siv.set_global_callback(cursive::event::Event::Ctrl(Key::K), UI::clear_handler); drop(inner); drop(siv); diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index ec18eb91..9119719e 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -977,7 +977,8 @@ impl RPCProcessor { self.unlocked_inner .waiting_rpc_table .complete_op_waiter(msg.operation.op_id(), msg) - .await + .await?; + Ok(NetworkResult::value(())) } } } @@ -1005,7 +1006,13 @@ impl RPCProcessor { Ok(v) => v, }; - network_result_value_or_log!(debug res => {}); + cfg_if::cfg_if! { + if #[cfg(debug_assertions)] { + network_result_value_or_log!(warn res => {}); + } else { + network_result_value_or_log!(debug res => {}); + } + } } } diff --git a/veilid-core/src/rpc_processor/rpc_app_call.rs b/veilid-core/src/rpc_processor/rpc_app_call.rs index 3fc60207..e068e133 100644 --- a/veilid-core/src/rpc_processor/rpc_app_call.rs +++ b/veilid-core/src/rpc_processor/rpc_app_call.rs @@ -88,11 +88,8 @@ impl RPCProcessor { let app_call_a = RPCOperationAppCallA { message }; // Send status answer - let res = self - .answer(msg, RPCAnswer::new(RPCAnswerDetail::AppCallA(app_call_a))) - .await?; - tracing::Span::current().record("res", &tracing::field::display(res)); - Ok(res) + self.answer(msg, RPCAnswer::new(RPCAnswerDetail::AppCallA(app_call_a))) + .await } /// Exposed to API for apps to return app call answers diff --git a/veilid-core/src/rpc_processor/rpc_app_message.rs b/veilid-core/src/rpc_processor/rpc_app_message.rs index 29f152b4..6793803f 100644 --- a/veilid-core/src/rpc_processor/rpc_app_message.rs +++ b/veilid-core/src/rpc_processor/rpc_app_message.rs @@ -13,9 +13,7 @@ impl RPCProcessor { let statement = RPCStatement::new(RPCStatementDetail::AppMessage(app_message)); // Send the app message request - network_result_try!(self.statement(dest, statement).await?); - - Ok(NetworkResult::value(())) + self.statement(dest, statement).await } #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] diff --git a/veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs b/veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs index 04a7a332..36cd9edf 100644 --- a/veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs +++ b/veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs @@ -1,10 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] - pub(crate) async fn process_cancel_tunnel_q(&self, msg: RPCMessage) -> Result, RPCError> { - // tracing::Span::current().record("res", &tracing::field::display(res)); - + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] + pub(crate) async fn process_cancel_tunnel_q( + &self, + msg: RPCMessage, + ) -> Result, RPCError> { Err(RPCError::unimplemented("process_cancel_tunnel_q")) } } diff --git a/veilid-core/src/rpc_processor/rpc_complete_tunnel.rs b/veilid-core/src/rpc_processor/rpc_complete_tunnel.rs index 9270921b..5143f143 100644 --- a/veilid-core/src/rpc_processor/rpc_complete_tunnel.rs +++ b/veilid-core/src/rpc_processor/rpc_complete_tunnel.rs @@ -1,12 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_complete_tunnel_q( &self, msg: RPCMessage, ) -> Result, RPCError> { - // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_complete_tunnel_q")) } } diff --git a/veilid-core/src/rpc_processor/rpc_find_block.rs b/veilid-core/src/rpc_processor/rpc_find_block.rs index 4d7ca42b..f4fe9057 100644 --- a/veilid-core/src/rpc_processor/rpc_find_block.rs +++ b/veilid-core/src/rpc_processor/rpc_find_block.rs @@ -1,12 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_find_block_q( &self, msg: RPCMessage, ) -> Result, RPCError> { - // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_find_block_q")) } } diff --git a/veilid-core/src/rpc_processor/rpc_find_node.rs b/veilid-core/src/rpc_processor/rpc_find_node.rs index 4f80df2d..94549971 100644 --- a/veilid-core/src/rpc_processor/rpc_find_node.rs +++ b/veilid-core/src/rpc_processor/rpc_find_node.rs @@ -66,7 +66,7 @@ impl RPCProcessor { ))) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_find_node_q( &self, msg: RPCMessage, @@ -129,10 +129,7 @@ impl RPCProcessor { }; // Send status answer - let res = self - .answer(msg, RPCAnswer::new(RPCAnswerDetail::FindNodeA(find_node_a))) - .await?; - tracing::Span::current().record("res", &tracing::field::display(res)); - Ok(res) + self.answer(msg, RPCAnswer::new(RPCAnswerDetail::FindNodeA(find_node_a))) + .await } } diff --git a/veilid-core/src/rpc_processor/rpc_get_value.rs b/veilid-core/src/rpc_processor/rpc_get_value.rs index 65aad72f..3a697bb0 100644 --- a/veilid-core/src/rpc_processor/rpc_get_value.rs +++ b/veilid-core/src/rpc_processor/rpc_get_value.rs @@ -1,12 +1,11 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_get_value_q( &self, msg: RPCMessage, ) -> Result, RPCError> { - //tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_get_value_q")) } } diff --git a/veilid-core/src/rpc_processor/rpc_node_info_update.rs b/veilid-core/src/rpc_processor/rpc_node_info_update.rs index a676f6b2..427ec30f 100644 --- a/veilid-core/src/rpc_processor/rpc_node_info_update.rs +++ b/veilid-core/src/rpc_processor/rpc_node_info_update.rs @@ -63,12 +63,21 @@ impl RPCProcessor { ))); } - self.routing_table().register_node_with_signed_node_info( - routing_domain, - sender_node_id, - node_info_update.signed_node_info, - false, - ); + if self + .routing_table() + .register_node_with_signed_node_info( + routing_domain, + sender_node_id, + node_info_update.signed_node_info, + false, + ) + .is_none() + { + return Ok(NetworkResult::invalid_message(format!( + "could not register node info update {}", + sender_node_id + ))); + } Ok(NetworkResult::value(())) } diff --git a/veilid-core/src/rpc_processor/rpc_route.rs b/veilid-core/src/rpc_processor/rpc_route.rs index 1e360ab7..4abdd681 100644 --- a/veilid-core/src/rpc_processor/rpc_route.rs +++ b/veilid-core/src/rpc_processor/rpc_route.rs @@ -1,7 +1,5 @@ use super::*; -xxx go through and convert errs to networkresult - impl RPCProcessor { #[instrument(level = "trace", skip_all, err)] async fn process_route_safety_route_hop( @@ -11,44 +9,48 @@ impl RPCProcessor { ) -> Result, RPCError> { // Make sure hop count makes sense if route.safety_route.hop_count as usize > self.unlocked_inner.max_route_hop_count { - return Err(RPCError::protocol( + return Ok(NetworkResult::invalid_message( "Safety route hop count too high to process", )); } if route.safety_route.hop_count == 0 { - return Err(RPCError::protocol( + return Ok(NetworkResult::invalid_message( "Safety route hop count should not be zero if there are more hops", )); } if route_hop.next_hop.is_none() { - return Err(RPCError::protocol("Safety route hop must have next hop")); + return Ok(NetworkResult::invalid_message( + "Safety route hop must have next hop", + )); } // Get next hop node ref let next_hop_nr = match route_hop.node { RouteNode::NodeId(id) => { // - self.routing_table - .lookup_node_ref(id.key) - .ok_or_else(|| RPCError::network(format!("node hop {} not found", id.key))) + let Some(nr) = self.routing_table.lookup_node_ref(id.key) else { + return Ok(NetworkResult::invalid_message(format!("node hop {} not found", id.key))); + }; + nr } RouteNode::PeerInfo(pi) => { // - self.routing_table + let Some(nr) = self.routing_table .register_node_with_signed_node_info( RoutingDomain::PublicInternet, pi.node_id.key, pi.signed_node_info, false, - ) - .ok_or_else(|| { - RPCError::network(format!( + ) else + { + return Ok(NetworkResult::invalid_message(format!( "node hop {} could not be registered", pi.node_id.key - )) - }) + ))); + }; + nr } - }?; + }; // Pass along the route let next_hop_route = RPCOperationRoute { @@ -62,13 +64,8 @@ impl RPCProcessor { let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route)); // Send the next route statement - network_result_value_or_log!(debug - self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt) - .await? => { - return Err(RPCError::network("unable to send route statement for next safety route hop")); - } - ); - Ok(()) + self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt) + .await } #[instrument(level = "trace", skip_all, err)] @@ -124,14 +121,8 @@ impl RPCProcessor { let next_hop_route_stmt = RPCStatement::new(RPCStatementDetail::Route(next_hop_route)); // Send the next route statement - network_result_value_or_log!(debug - self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt) - .await? => { - return Err(RPCError::network("unable to send route statement for private route hop")); - } - ); - - Ok(()) + self.statement(Destination::direct(next_hop_nr), next_hop_route_stmt) + .await } /// Process a routed operation that came in over a safety route but no private route @@ -164,21 +155,23 @@ impl RPCProcessor { .crypto .cached_dh(&safety_route.public_key, &node_id_secret) .map_err(RPCError::protocol)?; - let body = Crypto::decrypt_aead( + let body = match Crypto::decrypt_aead( &routed_operation.data, &routed_operation.nonce, &dh_secret, None, - ) - .map_err(RPCError::map_internal( - "decryption of routed operation failed", - ))?; - + ) { + Ok(v) => v, + Err(e) => { + return Ok(NetworkResult::invalid_message(format!("decryption of routed operation failed: {}", e))); + } + }; + // Pass message to RPC system self.enqueue_safety_routed_message(sequencing, body) .map_err(RPCError::internal)?; - Ok(()) + Ok(NetworkResult::value(())) } /// Process a routed operation that came in over both a safety route and a private route @@ -195,15 +188,17 @@ impl RPCProcessor { // Look up the private route and ensure it's one in our spec store let rss = self.routing_table.route_spec_store(); - let (secret_key, safety_spec) = rss + let Some((secret_key, safety_spec)) = rss .validate_signatures( &private_route.public_key, &routed_operation.signatures, &routed_operation.data, sender_id, ) - .map_err(RPCError::protocol)? - .ok_or_else(|| RPCError::protocol("signatures did not validate for private route"))?; + .map_err(RPCError::protocol)? + else { + return Ok(NetworkResult::invalid_message("signatures did not validate for private route")); + }; // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret) // xxx: punish nodes that send messages that fail to decrypt eventually. How to do this for private routes? @@ -225,7 +220,7 @@ impl RPCProcessor { self.enqueue_private_routed_message(private_route.public_key, safety_spec, body) .map_err(RPCError::internal)?; - Ok(()) + Ok(NetworkResult::value(())) } #[instrument(level = "trace", skip_all, err)] @@ -238,12 +233,12 @@ impl RPCProcessor { ) -> Result, RPCError> { // Make sure hop count makes sense if safety_route.hop_count != 0 { - return Err(RPCError::protocol( + return Ok(NetworkResult::invalid_message( "Safety hop count should be zero if switched to private route", )); } if private_route.hop_count != 0 { - return Err(RPCError::protocol( + return Ok(NetworkResult::invalid_message( "Private route hop count should be zero if we are at the end", )); } @@ -271,7 +266,7 @@ impl RPCProcessor { private_route: &PrivateRoute, ) -> Result, RPCError> { let PrivateRouteHops::FirstHop(pr_first_hop) = &private_route.hops else { - return Err(RPCError::protocol("switching from safety route to private route requires first hop")); + return Ok(NetworkResult::invalid_message("switching from safety route to private route requires first hop")); }; // Switching to private route from safety route @@ -355,12 +350,12 @@ impl RPCProcessor { }; // Switching from full safety route to private route first hop - self.process_private_route_first_hop( + network_result_try!(self.process_private_route_first_hop( route.operation, route.safety_route.public_key, &private_route, ) - .await?; + .await?); } else if dec_blob_tag == 0 { // RouteHop let route_hop = { @@ -371,8 +366,8 @@ impl RPCProcessor { }; // Continue the full safety route with another hop - self.process_route_safety_route_hop(route, route_hop) - .await?; + network_result_try!(self.process_route_safety_route_hop(route, route_hop) + .await?); } else { return Ok(NetworkResult::invalid_message("invalid blob tag")); } @@ -383,12 +378,12 @@ impl RPCProcessor { match &private_route.hops { PrivateRouteHops::FirstHop(_) => { // Safety route was a stub, start with the beginning of the private route - self.process_private_route_first_hop( + network_result_try!(self.process_private_route_first_hop( route.operation, route.safety_route.public_key, private_route, ) - .await?; + .await?); } PrivateRouteHops::Data(route_hop_data) => { // Decrypt the blob with DEC(nonce, DH(the PR's public key, this hop's secret) @@ -397,13 +392,17 @@ impl RPCProcessor { .crypto .cached_dh(&private_route.public_key, &node_id_secret) .map_err(RPCError::protocol)?; - let dec_blob_data = Crypto::decrypt_aead( + let dec_blob_data = match Crypto::decrypt_aead( &route_hop_data.blob, &route_hop_data.nonce, &dh_secret, None, - ) - .map_err(RPCError::protocol)?; + ) { + Ok(v) => v, + Err(e) => { + return Ok(NetworkResult::invalid_message(format!("unable to decrypt private route hop data: {}", e))); + } + }; let dec_blob_reader = RPCMessageData::new(dec_blob_data).get_reader()?; // Decode next RouteHop @@ -432,7 +431,7 @@ impl RPCProcessor { } // Make next PrivateRoute and pass it on - self.process_route_private_route_hop( + network_result_try!(self.process_route_private_route_hop( route.operation, route_hop.node, route.safety_route.public_key, @@ -445,7 +444,7 @@ impl RPCProcessor { .unwrap_or(PrivateRouteHops::Empty), }, ) - .await?; + .await?); } PrivateRouteHops::Empty => { // Ensure hop count == 0 @@ -456,12 +455,12 @@ impl RPCProcessor { } // No hops left, time to process the routed operation - self.process_routed_operation( + network_result_try!(self.process_routed_operation( detail, route.operation, &route.safety_route, private_route, - )?; + )?); } } } diff --git a/veilid-core/src/rpc_processor/rpc_set_value.rs b/veilid-core/src/rpc_processor/rpc_set_value.rs index 9f89e13c..2f195412 100644 --- a/veilid-core/src/rpc_processor/rpc_set_value.rs +++ b/veilid-core/src/rpc_processor/rpc_set_value.rs @@ -1,7 +1,7 @@ use super::*; impl RPCProcessor { - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)] + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_set_value_q( &self, msg: RPCMessage, diff --git a/veilid-core/src/rpc_processor/rpc_signal.rs b/veilid-core/src/rpc_processor/rpc_signal.rs index 592c2ed8..d717b395 100644 --- a/veilid-core/src/rpc_processor/rpc_signal.rs +++ b/veilid-core/src/rpc_processor/rpc_signal.rs @@ -26,9 +26,7 @@ impl RPCProcessor { let statement = RPCStatement::new(RPCStatementDetail::Signal(signal)); // Send the signal request - network_result_try!(self.statement(dest, statement).await?); - - Ok(NetworkResult::value(())) + self.statement(dest, statement).await } #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] @@ -56,11 +54,9 @@ impl RPCProcessor { // Handle it let network_manager = self.network_manager(); - let res = network_manager + network_manager .handle_signal(signal.signal_info) .await - .map_err(RPCError::network)?; - - Ok(res) + .map_err(RPCError::network) } } diff --git a/veilid-core/src/rpc_processor/rpc_start_tunnel.rs b/veilid-core/src/rpc_processor/rpc_start_tunnel.rs index 95218748..881f9f51 100644 --- a/veilid-core/src/rpc_processor/rpc_start_tunnel.rs +++ b/veilid-core/src/rpc_processor/rpc_start_tunnel.rs @@ -6,7 +6,6 @@ impl RPCProcessor { &self, msg: RPCMessage, ) -> Result, RPCError> { - // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_start_tunnel_q")) } } diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 4bbce6e1..ef03f631 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -179,7 +179,7 @@ impl RPCProcessor { Ok(NetworkResult::value(Answer::new(latency, opt_sender_info))) } - #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id, res), ret, err)] + #[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err)] pub(crate) async fn process_status_q( &self, msg: RPCMessage, @@ -250,10 +250,7 @@ impl RPCProcessor { }; // Send status answer - let res = self - .answer(msg, RPCAnswer::new(RPCAnswerDetail::StatusA(status_a))) - .await?; - tracing::Span::current().record("res", &tracing::field::display(res)); - Ok(res) + self.answer(msg, RPCAnswer::new(RPCAnswerDetail::StatusA(status_a))) + .await } } diff --git a/veilid-core/src/rpc_processor/rpc_supply_block.rs b/veilid-core/src/rpc_processor/rpc_supply_block.rs index 9a98f3ff..234fb71b 100644 --- a/veilid-core/src/rpc_processor/rpc_supply_block.rs +++ b/veilid-core/src/rpc_processor/rpc_supply_block.rs @@ -6,8 +6,6 @@ impl RPCProcessor { &self, msg: RPCMessage, ) -> Result, RPCError> { - // tracing::Span::current().record("res", &tracing::field::display(res)); - Err(RPCError::unimplemented("process_supply_block_q")) } } diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 954b8d44..d1910709 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -144,7 +144,7 @@ impl RPCProcessor { // Send the validate_dial_info request // This can only be sent directly, as relays can not validate dial info - let res = network_result_value_or_log!(debug self.statement(Destination::direct(peer), statement) + network_result_value_or_log!(debug self.statement(Destination::direct(peer), statement) .await? => { continue; } diff --git a/veilid-core/src/rpc_processor/rpc_value_changed.rs b/veilid-core/src/rpc_processor/rpc_value_changed.rs index 6c46d4d7..6df3c332 100644 --- a/veilid-core/src/rpc_processor/rpc_value_changed.rs +++ b/veilid-core/src/rpc_processor/rpc_value_changed.rs @@ -6,8 +6,6 @@ impl RPCProcessor { &self, msg: RPCMessage, ) -> Result, RPCError> { - // tracing::Span::current().record("res", &tracing::field::display(res)); - Err(RPCError::unimplemented("process_value_changed")) } } diff --git a/veilid-core/src/rpc_processor/rpc_watch_value.rs b/veilid-core/src/rpc_processor/rpc_watch_value.rs index 8526e037..08b3ee23 100644 --- a/veilid-core/src/rpc_processor/rpc_watch_value.rs +++ b/veilid-core/src/rpc_processor/rpc_watch_value.rs @@ -6,7 +6,6 @@ impl RPCProcessor { &self, msg: RPCMessage, ) -> Result, RPCError> { - // tracing::Span::current().record("res", &tracing::field::display(res)); Err(RPCError::unimplemented("process_watch_value_q")) } } diff --git a/veilid-core/src/xx/network_result.rs b/veilid-core/src/xx/network_result.rs index e4c72ba1..dd50d33c 100644 --- a/veilid-core/src/xx/network_result.rs +++ b/veilid-core/src/xx/network_result.rs @@ -303,24 +303,73 @@ macro_rules! network_result_try { }; } +#[macro_export] +macro_rules! log_network_result { + ($text:expr) => { + cfg_if::cfg_if! { + if #[cfg(debug_assertions)] { + info!(target: "network_result", "{}", $text) + } else { + debug!(target: "network_result", "{}", $text) + } + } + }; + ($fmt:literal, $($arg:expr),+) => { + cfg_if::cfg_if! { + if #[cfg(debug_assertions)] { + info!(target: "network_result", $fmt, $($arg),+); + } else { + debug!(target: "network_result", $fmt, $($arg),+); + } + } + }; +} + #[macro_export] macro_rules! network_result_value_or_log { ($level: ident $r: expr => $f:tt) => { match $r { NetworkResult::Timeout => { - log_net!($level "{} at {}@{}:{}", "Timeout".green(), file!(), line!(), column!()); + log_network_result!( + "{} at {}@{}:{}", + "Timeout".cyan(), + file!(), + line!(), + column!() + ); $f } NetworkResult::NoConnection(e) => { - log_net!($level "{}({}) at {}@{}:{}", "No connection".green(), e.to_string(), file!(), line!(), column!()); + log_network_result!( + "{}({}) at {}@{}:{}", + "No connection".cyan(), + e.to_string(), + file!(), + line!(), + column!() + ); $f } NetworkResult::AlreadyExists(e) => { - log_net!($level "{}({}) at {}@{}:{}", "Already exists".green(), e.to_string(), file!(), line!(), column!()); + log_network_result!( + "{}({}) at {}@{}:{}", + "Already exists".cyan(), + e.to_string(), + file!(), + line!(), + column!() + ); $f } NetworkResult::InvalidMessage(s) => { - log_net!($level "{}({}) at {}@{}:{}", "Invalid message".green(), s, file!(), line!(), column!()); + log_network_result!( + "{}({}) at {}@{}:{}", + "Invalid message".cyan(), + s, + file!(), + line!(), + column!() + ); $f } NetworkResult::Value(v) => v, From 88132de86e1ce422c739b41b348fb33310b9e6df Mon Sep 17 00:00:00 2001 From: John Smith Date: Mon, 21 Nov 2022 20:55:10 -0500 Subject: [PATCH 64/67] ui stuff --- external/cursive-flexi-logger-view | 2 +- veilid-cli/src/ui.rs | 26 +++++++------------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/external/cursive-flexi-logger-view b/external/cursive-flexi-logger-view index fd560c49..effa60ce 160000 --- a/external/cursive-flexi-logger-view +++ b/external/cursive-flexi-logger-view @@ -1 +1 @@ -Subproject commit fd560c499be0f34305e0d48aca7f1bc3d015a17f +Subproject commit effa60cea24e99f294865ed325ffc57612d72785 diff --git a/veilid-cli/src/ui.rs b/veilid-cli/src/ui.rs index 2d07a345..bacdc3ad 100644 --- a/veilid-cli/src/ui.rs +++ b/veilid-cli/src/ui.rs @@ -220,12 +220,7 @@ impl UI { cursive_flexi_logger_view::clear_log(); UI::update_cb(siv); } - fn node_events(s: &mut Cursive) -> ViewRef { - s.find_name("node-events").unwrap() - } - fn node_events_panel( - s: &mut Cursive, - ) -> ViewRef>>>> { + fn node_events_panel(s: &mut Cursive) -> ViewRef>> { s.find_name("node-events-panel").unwrap() } fn command_line(s: &mut Cursive) -> ViewRef { @@ -743,18 +738,11 @@ impl UI { // Create layouts - let node_events_view = Panel::new( - FlexiLoggerView::new() - .with_name("node-events") - .scrollable() - .scroll_x(true) - .scroll_y(true) - .scroll_strategy(ScrollStrategy::StickToBottom) - .full_screen(), - ) - .title_position(HAlign::Left) - .title("Node Events") - .with_name("node-events-panel"); + let node_events_view = Panel::new(FlexiLoggerView::new_scrollable()) + .title_position(HAlign::Left) + .title("Node Events") + .with_name("node-events-panel") + .full_screen(); let peers_table_view = PeersTableView::new() .column(PeerTableColumn::NodeId, "Node Id", |c| c.width(43)) @@ -833,7 +821,7 @@ impl UI { UI::setup_colors(&mut siv, &mut inner, settings); UI::setup_quit_handler(&mut siv); - siv.set_global_callback(cursive::event::Event::Ctrl(Key::K), UI::clear_handler); + siv.set_global_callback(cursive::event::Event::CtrlChar('k'), UI::clear_handler); drop(inner); drop(siv); From 27f7f49d4f7c166e46eb2f09d109f3437e8cc05a Mon Sep 17 00:00:00 2001 From: John Smith Date: Mon, 21 Nov 2022 22:50:42 -0500 Subject: [PATCH 65/67] checkpoint --- doc/config/sample.config | 2 +- doc/config/veilid-server-config.md | 2 +- veilid-core/src/routing_table/bucket_entry.rs | 5 ++- .../src/routing_table/route_spec_store.rs | 13 +++---- .../routing_table/routing_domain_editor.rs | 11 ++++++ veilid-core/src/rpc_processor/mod.rs | 36 +++++++++++++++++-- .../src/tests/common/test_veilid_config.rs | 4 +-- veilid-core/src/xx/tick_task.rs | 3 +- veilid-flutter/example/lib/config.dart | 2 +- veilid-server/src/settings.rs | 4 +-- veilid-wasm/tests/web.rs | 2 +- 11 files changed, 64 insertions(+), 20 deletions(-) diff --git a/doc/config/sample.config b/doc/config/sample.config index 88665c76..c7e65f0f 100644 --- a/doc/config/sample.config +++ b/doc/config/sample.config @@ -64,7 +64,7 @@ core: max_timestamp_ahead_ms: 10000 timeout_ms: 10000 max_route_hop_count: 4 - default_route_hop_count: 2 + default_route_hop_count: 1 dht: resolve_node_timeout: diff --git a/doc/config/veilid-server-config.md b/doc/config/veilid-server-config.md index b7968693..83203e3b 100644 --- a/doc/config/veilid-server-config.md +++ b/doc/config/veilid-server-config.md @@ -229,7 +229,7 @@ rpc: max_timestamp_ahead_ms: 10000 timeout_ms: 10000 max_route_hop_count: 4 - default_route_hop_count: 2 + default_route_hop_count: 1 ``` #### core:network:dht diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 2ee36d1c..c4cf88c6 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -85,8 +85,11 @@ pub struct BucketEntryInner { /// The minimum and maximum range of cryptography versions supported by the node, /// inclusive of the requirements of any relay the node may be using min_max_version: Option, - /// Whether or not we have updated this peer with our node info since our network + /// If this node has updated it's SignedNodeInfo since our network /// and dial info has last changed, for example when our IP address changes + /// Used to determine if we should make this entry 'live' again when we receive a signednodeinfo update that + /// has the same timestamp, because if we change our own IP address or network class it may be possible for nodes that were + /// unreachable may now be reachable with the same SignedNodeInfo/DialInfo updated_since_last_network_change: bool, /// The last connection descriptors used to contact this node, per protocol type #[with(Skip)] diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index bf0fc2db..9bb8d50d 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -1268,35 +1268,32 @@ impl RouteSpecStore { .unwrap_or_default() } - /// Mark a remote private route as having seen our node info { - pub fn mark_remote_private_route_seen_our_node_info(&self, key: &DHTKey) { + /// Mark a remote private route as having seen our node info + pub fn mark_remote_private_route_seen_our_node_info(&self, key: &DHTKey, cur_ts: u64) { let inner = &mut *self.inner.lock(); - let cur_ts = intf::get_timestamp(); Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { rpr.seen_our_node_info = true; }) } /// Mark a remote private route as having replied to a question { - pub fn mark_remote_private_route_replied(&self, key: &DHTKey) { + pub fn mark_remote_private_route_replied(&self, key: &DHTKey, cur_ts: u64) { let inner = &mut *self.inner.lock(); - let cur_ts = intf::get_timestamp(); Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { rpr.last_replied_ts = Some(cur_ts); }) } /// Mark a remote private route as having beed used { - pub fn mark_remote_private_route_used(&self, key: &DHTKey) { + pub fn mark_remote_private_route_used(&self, key: &DHTKey, cur_ts: u64) { let inner = &mut *self.inner.lock(); - let cur_ts = intf::get_timestamp(); Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { rpr.last_used_ts = Some(cur_ts); }) } /// Clear caches when local our local node info changes - pub fn local_node_info_changed(&self) { + pub fn reset(&self) { let inner = &mut *self.inner.lock(); // Clean up local allocated routes diff --git a/veilid-core/src/routing_table/routing_domain_editor.rs b/veilid-core/src/routing_table/routing_domain_editor.rs index b9150196..08aba320 100644 --- a/veilid-core/src/routing_table/routing_domain_editor.rs +++ b/veilid-core/src/routing_table/routing_domain_editor.rs @@ -194,14 +194,25 @@ impl RoutingDomainEditor { } } if changed { + // Clear our 'peer info' cache, the peerinfo for this routing domain will get regenerated next time it is asked for detail.common_mut().clear_cache() } }); if changed { + // Mark that nothing in the routing table has seen our new node info inner.reset_all_seen_our_node_info(self.routing_domain); + // inner.reset_all_updated_since_last_network_change(); } } + // Clear the routespecstore cache if our PublicInternet dial info has changed + if changed { + if self.routing_domain == RoutingDomain::PublicInternet { + let rss = self.routing_table.route_spec_store(); + rss.reset(); + } + } + // Send our updated node info to all the nodes in the routing table if changed && self.send_node_info_updates { let network_manager = self.routing_table.unlocked_inner.network_manager.clone(); network_manager diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 9119719e..c661dec7 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -162,6 +162,7 @@ where #[derive(Debug)] struct WaitableReply { + dest: Destination, handle: OperationWaitHandle, timeout: u64, node_ref: NodeRef, @@ -441,7 +442,27 @@ impl RPCProcessor { waitable_reply.send_ts, recv_ts, rpcreader.header.body_len, - ) + ); + // Process private route replies + if let Destination::PrivateRoute { + private_route, + safety_selection, + } = &waitable_reply.dest + { + let rss = self.routing_table.route_spec_store(); + + // If we received a reply from a private route, mark it as such + rss.mark_remote_private_route_replied(&private_route.public_key, recv_ts); + + // If we sent to a private route without a safety route + // We need to mark our own node info as having been seen so we can optimize sending it + if let SafetySelection::Unsafe(_) = safety_selection { + rss.mark_remote_private_route_seen_our_node_info( + &private_route.public_key, + recv_ts, + ); + } + } } }; @@ -702,7 +723,7 @@ impl RPCProcessor { node_id, node_ref, hop_count, - } = network_result_try!(self.render_operation(dest, &operation)?); + } = network_result_try!(self.render_operation(dest.clone(), &operation)?); // Calculate answer timeout // Timeout is number of hops times the timeout per hop @@ -733,8 +754,19 @@ impl RPCProcessor { // Successfully sent node_ref.stats_question_sent(send_ts, bytes, true); + // Private route stats + if let Destination::PrivateRoute { + private_route, + safety_selection: _, + } = &dest + { + let rss = self.routing_table.route_spec_store(); + rss.mark_remote_private_route_used(&private_route.public_key, intf::get_timestamp()); + } + // Pass back waitable reply completion Ok(NetworkResult::value(WaitableReply { + dest, handle, timeout, node_ref, diff --git a/veilid-core/src/tests/common/test_veilid_config.rs b/veilid-core/src/tests/common/test_veilid_config.rs index 4e56d4e7..e73e79a1 100644 --- a/veilid-core/src/tests/common/test_veilid_config.rs +++ b/veilid-core/src/tests/common/test_veilid_config.rs @@ -207,7 +207,7 @@ fn config_callback(key: String) -> ConfigCallbackReturn { "network.rpc.max_timestamp_ahead_ms" => Ok(Box::new(Some(10_000u32))), "network.rpc.timeout_ms" => Ok(Box::new(10_000u32)), "network.rpc.max_route_hop_count" => Ok(Box::new(4u8)), - "network.rpc.default_route_hop_count" => Ok(Box::new(2u8)), + "network.rpc.default_route_hop_count" => Ok(Box::new(1u8)), "network.dht.resolve_node_timeout_ms" => Ok(Box::new(Option::::None)), "network.dht.resolve_node_count" => Ok(Box::new(20u32)), "network.dht.resolve_node_fanout" => Ok(Box::new(3u32)), @@ -325,7 +325,7 @@ pub async fn test_config() { assert_eq!(inner.network.rpc.queue_size, 1024u32); assert_eq!(inner.network.rpc.timeout_ms, 10_000u32); assert_eq!(inner.network.rpc.max_route_hop_count, 4u8); - assert_eq!(inner.network.rpc.default_route_hop_count, 2u8); + assert_eq!(inner.network.rpc.default_route_hop_count, 1u8); assert_eq!(inner.network.routing_table.limit_over_attached, 64u32); assert_eq!(inner.network.routing_table.limit_fully_attached, 32u32); assert_eq!(inner.network.routing_table.limit_attached_strong, 16u32); diff --git a/veilid-core/src/xx/tick_task.rs b/veilid-core/src/xx/tick_task.rs index 56ea442e..2d5d1e70 100644 --- a/veilid-core/src/xx/tick_task.rs +++ b/veilid-core/src/xx/tick_task.rs @@ -83,7 +83,8 @@ impl TickTask { let now = intf::get_timestamp(); let last_timestamp_us = self.last_timestamp_us.load(Ordering::Acquire); - if last_timestamp_us != 0u64 && (now - last_timestamp_us) < self.tick_period_us { + if last_timestamp_us != 0u64 && now.saturating_sub(last_timestamp_us) < self.tick_period_us + { // It's not time yet return Ok(()); } diff --git a/veilid-flutter/example/lib/config.dart b/veilid-flutter/example/lib/config.dart index 9eb209c3..aaeea802 100644 --- a/veilid-flutter/example/lib/config.dart +++ b/veilid-flutter/example/lib/config.dart @@ -66,7 +66,7 @@ Future getDefaultVeilidConfig() async { maxTimestampAheadMs: 10000, timeoutMs: 10000, maxRouteHopCount: 4, - defaultRouteHopCount: 2, + defaultRouteHopCount: 1, ), dht: VeilidConfigDHT( resolveNodeTimeoutMs: null, diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index 4c6b4135..203076cd 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -84,7 +84,7 @@ core: max_timestamp_ahead_ms: 10000 timeout_ms: 10000 max_route_hop_count: 4 - default_route_hop_count: 2 + default_route_hop_count: 1 dht: resolve_node_timeout: resolve_node_count: 20 @@ -1510,7 +1510,7 @@ mod tests { assert_eq!(s.core.network.rpc.max_timestamp_ahead_ms, Some(10_000u32)); assert_eq!(s.core.network.rpc.timeout_ms, 10_000u32); assert_eq!(s.core.network.rpc.max_route_hop_count, 4); - assert_eq!(s.core.network.rpc.default_route_hop_count, 2); + assert_eq!(s.core.network.rpc.default_route_hop_count, 1); // assert_eq!(s.core.network.dht.resolve_node_timeout_ms, None); assert_eq!(s.core.network.dht.resolve_node_count, 20u32); diff --git a/veilid-wasm/tests/web.rs b/veilid-wasm/tests/web.rs index 230f0609..8856b033 100644 --- a/veilid-wasm/tests/web.rs +++ b/veilid-wasm/tests/web.rs @@ -46,7 +46,7 @@ fn init_callbacks() { case "network.rpc.max_timestamp_ahead": return 10000000; case "network.rpc.timeout": return 10000000; case "network.rpc.max_route_hop_count": return 4; - case "network.rpc.default_route_hop_count": return 2; + case "network.rpc.default_route_hop_count": return 1; case "network.dht.resolve_node_timeout": return null; case "network.dht.resolve_node_count": return 20; case "network.dht.resolve_node_fanout": return 3; From dec7bd27dad0ba72ba42534728b5f02264eebbf5 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 22 Nov 2022 18:26:39 -0500 Subject: [PATCH 66/67] checkpoint --- veilid-core/src/routing_table/mod.rs | 2 + .../{veilid_api => routing_table}/privacy.rs | 0 .../src/routing_table/route_spec_store.rs | 209 +++++++++++++----- veilid-core/src/rpc_processor/mod.rs | 18 +- veilid-core/src/veilid_api/debug.rs | 36 ++- veilid-core/src/veilid_api/mod.rs | 84 ++++++- veilid-core/src/veilid_api/routing_context.rs | 19 +- 7 files changed, 280 insertions(+), 88 deletions(-) rename veilid-core/src/{veilid_api => routing_table}/privacy.rs (100%) diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 2336396c..89370fc4 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -3,6 +3,7 @@ mod bucket_entry; mod debug; mod node_ref; mod node_ref_filter; +mod privacy; mod route_spec_store; mod routing_domain_editor; mod routing_domains; @@ -21,6 +22,7 @@ pub use debug::*; use hashlink::LruCache; pub use node_ref::*; pub use node_ref_filter::*; +pub use privacy::*; pub use route_spec_store::*; pub use routing_domain_editor::*; pub use routing_domains::*; diff --git a/veilid-core/src/veilid_api/privacy.rs b/veilid-core/src/routing_table/privacy.rs similarity index 100% rename from veilid-core/src/veilid_api/privacy.rs rename to veilid-core/src/routing_table/privacy.rs diff --git a/veilid-core/src/routing_table/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store.rs index 9bb8d50d..66d80cf6 100644 --- a/veilid-core/src/routing_table/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store.rs @@ -81,14 +81,14 @@ pub struct RouteSpecStoreContent { /// What remote private routes have seen #[derive(Debug, Clone, Default)] struct RemotePrivateRouteInfo { - // When this remote private route was last modified - modified_ts: u64, - /// Did this remote private route see our node info due to no safety route in use - seen_our_node_info: bool, + // The private route itself + private_route: Option, + /// Timestamp of when the route was last used for anything + last_used_ts: u64, /// The time this remote private route last responded last_replied_ts: Option, - /// Timestamp of when the route was last used for anything - last_used_ts: Option, + /// Did this remote private route see our node info due to no safety route in use + seen_our_node_info: bool, } /// Ephemeral data used to help the RouteSpecStore operate efficiently @@ -743,7 +743,7 @@ impl RouteSpecStore { } } } - // We got the correct signatures, return a key ans + // We got the correct signatures, return a key and response safety spec Ok(Some(( rsd.secret_key, SafetySpec { @@ -755,34 +755,56 @@ impl RouteSpecStore { ))) } + /// Test an allocated route for continuity + pub async fn test_route(&self, key: &DHTKey) -> EyreResult { + let inner = &mut *self.inner.lock(); + let rsd = Self::detail(inner, &key).ok_or_else(|| eyre!("route does not exist"))?; + let rpc_processor = self.unlocked_inner.routing_table.rpc_processor(); + + // Target is the last hop + let target = rsd.hop_node_refs.last().unwrap().clone(); + let hop_count = rsd.hops.len(); + let stability = rsd.stability; + let sequencing = rsd.sequencing; + + // Test with ping to end + let res = match rpc_processor + .rpc_call_status(Destination::Direct { + target, + safety_selection: SafetySelection::Safe(SafetySpec { + preferred_route: Some(key.clone()), + hop_count, + stability, + sequencing, + }), + }) + .await? + { + NetworkResult::Value(v) => v, + _ => { + // Did not error, but did not come back, just return false + return Ok(false); + } + }; + + Ok(true) + } + + /// Release an allocated route that is no longer in use pub fn release_route(&self, public_key: DHTKey) -> EyreResult<()> { let mut inner = self.inner.lock(); - if let Some(detail) = inner.content.details.remove(&public_key) { - // Remove from hop cache - let cache_key = route_hops_to_hop_cache(&detail.hops); - if !inner.cache.hop_cache.remove(&cache_key) { - panic!("hop cache should have contained cache key"); - } - // Remove from used nodes cache - for h in &detail.hops { - match inner.cache.used_nodes.entry(*h) { - std::collections::hash_map::Entry::Occupied(mut o) => { - *o.get_mut() -= 1; - if *o.get() == 0 { - o.remove(); - } - } - std::collections::hash_map::Entry::Vacant(_) => { - panic!("used_nodes cache should have contained hop"); - } - } - } - // Remove from end nodes cache - match inner - .cache - .used_end_nodes - .entry(*detail.hops.last().unwrap()) - { + let Some(detail) = inner.content.details.remove(&public_key) else { + bail!("can't release route that was never allocated"); + }; + + // Remove from hop cache + let cache_key = route_hops_to_hop_cache(&detail.hops); + if !inner.cache.hop_cache.remove(&cache_key) { + panic!("hop cache should have contained cache key"); + } + // Remove from used nodes cache + for h in &detail.hops { + match inner.cache.used_nodes.entry(*h) { std::collections::hash_map::Entry::Occupied(mut o) => { *o.get_mut() -= 1; if *o.get() == 0 { @@ -790,11 +812,25 @@ impl RouteSpecStore { } } std::collections::hash_map::Entry::Vacant(_) => { - panic!("used_end_nodes cache should have contained hop"); + panic!("used_nodes cache should have contained hop"); } } - } else { - bail!("can't release route that was never allocated"); + } + // Remove from end nodes cache + match inner + .cache + .used_end_nodes + .entry(*detail.hops.last().unwrap()) + { + std::collections::hash_map::Entry::Occupied(mut o) => { + *o.get_mut() -= 1; + if *o.get() == 0 { + o.remove(); + } + } + std::collections::hash_map::Entry::Vacant(_) => { + panic!("used_end_nodes cache should have contained hop"); + } } Ok(()) } @@ -1205,37 +1241,66 @@ impl RouteSpecStore { Ok(private_route) } + /// Import a remote private route for compilation + pub fn import_remote_private_route(&self, blob: Vec) -> EyreResult { + // decode the pr blob + let private_route = RouteSpecStore::blob_to_private_route(blob)?; + + // store the private route in our cache + let inner = &mut *self.inner.lock(); + let cur_ts = intf::get_timestamp(); + + let key = Self::with_create_remote_private_route(inner, cur_ts, private_route, |r| { + r.private_route.as_ref().unwrap().public_key.clone() + }); + Ok(key) + } + + /// Retrieve an imported remote private route by its public key + pub fn get_remote_private_route(&self, key: &DHTKey) -> EyreResult { + let inner = &mut *self.inner.lock(); + let cur_ts = intf::get_timestamp(); + let Some(pr) = Self::with_get_remote_private_route(inner, cur_ts, key, |r| { + r.private_route.as_ref().unwrap().clone() + }) else { + bail!("remote private route not found"); + }; + Ok(pr) + } + // get or create a remote private route cache entry fn with_create_remote_private_route( inner: &mut RouteSpecStoreInner, cur_ts: u64, - key: &DHTKey, + private_route: PrivateRoute, f: F, ) -> R where F: FnOnce(&mut RemotePrivateRouteInfo) -> R, { + let pr_pubkey = private_route.public_key; + let rpr = inner .cache .remote_private_route_cache - .entry(*key) + .entry(pr_pubkey) .and_modify(|rpr| { - if cur_ts - rpr.modified_ts >= REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY { - *rpr = RemotePrivateRouteInfo { - modified_ts: cur_ts, - seen_our_node_info: false, - last_replied_ts: None, - last_used_ts: None, - }; + if cur_ts - rpr.last_used_ts >= REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY { + // Start fresh if this had expired + rpr.last_used_ts = cur_ts; + rpr.last_replied_ts = None; + rpr.seen_our_node_info = false; } else { - rpr.modified_ts = cur_ts; + // If not expired, just mark as being used + rpr.last_used_ts = cur_ts; } }) .or_insert_with(|| RemotePrivateRouteInfo { - modified_ts: cur_ts, - seen_our_node_info: false, + // New remote private route cache entry + private_route: Some(private_route), + last_used_ts: cur_ts, last_replied_ts: None, - last_used_ts: None, + seen_our_node_info: false, }); f(rpr) } @@ -1248,10 +1313,10 @@ impl RouteSpecStore { f: F, ) -> Option where - F: FnOnce(&RemotePrivateRouteInfo) -> R, + F: FnOnce(&mut RemotePrivateRouteInfo) -> R, { - let rpr = inner.cache.remote_private_route_cache.get(key)?; - if cur_ts - rpr.modified_ts < REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY { + let rpr = inner.cache.remote_private_route_cache.get_mut(key)?; + if cur_ts - rpr.last_used_ts < REMOTE_PRIVATE_ROUTE_CACHE_EXPIRY { return Some(f(rpr)); } inner.cache.remote_private_route_cache.remove(key); @@ -1269,27 +1334,46 @@ impl RouteSpecStore { } /// Mark a remote private route as having seen our node info - pub fn mark_remote_private_route_seen_our_node_info(&self, key: &DHTKey, cur_ts: u64) { + pub fn mark_remote_private_route_seen_our_node_info( + &self, + key: &DHTKey, + cur_ts: u64, + ) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { + if Self::with_get_remote_private_route(inner, cur_ts, key, |rpr| { rpr.seen_our_node_info = true; }) + .is_none() + { + bail!("private route is missing from store: {}", key); + } + Ok(()) } /// Mark a remote private route as having replied to a question { - pub fn mark_remote_private_route_replied(&self, key: &DHTKey, cur_ts: u64) { + pub fn mark_remote_private_route_replied(&self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { + if Self::with_get_remote_private_route(inner, cur_ts, key, |rpr| { rpr.last_replied_ts = Some(cur_ts); }) + .is_none() + { + bail!("private route is missing from store: {}", key); + } + Ok(()) } /// Mark a remote private route as having beed used { - pub fn mark_remote_private_route_used(&self, key: &DHTKey, cur_ts: u64) { + pub fn mark_remote_private_route_used(&self, key: &DHTKey, cur_ts: u64) -> EyreResult<()> { let inner = &mut *self.inner.lock(); - Self::with_create_remote_private_route(inner, cur_ts, key, |rpr| { - rpr.last_used_ts = Some(cur_ts); + if Self::with_get_remote_private_route(inner, cur_ts, key, |rpr| { + rpr.last_used_ts = cur_ts; }) + .is_none() + { + bail!("private route is missing from store: {}", key); + } + Ok(()) } /// Clear caches when local our local node info changes @@ -1306,8 +1390,11 @@ impl RouteSpecStore { v.last_checked_ts = None; } - // Clean up remote private routes - inner.cache.remote_private_route_cache.clear(); + // Reset private route cache + for (_k, v) in &mut inner.cache.remote_private_route_cache { + v.last_replied_ts = None; + v.seen_our_node_info = false; + } } /// Mark route as published diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index c661dec7..1a5c1781 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -452,15 +452,21 @@ impl RPCProcessor { let rss = self.routing_table.route_spec_store(); // If we received a reply from a private route, mark it as such - rss.mark_remote_private_route_replied(&private_route.public_key, recv_ts); + if let Err(e) = + rss.mark_remote_private_route_replied(&private_route.public_key, recv_ts) + { + log_rpc!(error "private route missing: {}", e); + } // If we sent to a private route without a safety route // We need to mark our own node info as having been seen so we can optimize sending it if let SafetySelection::Unsafe(_) = safety_selection { - rss.mark_remote_private_route_seen_our_node_info( + if let Err(e) = rss.mark_remote_private_route_seen_our_node_info( &private_route.public_key, recv_ts, - ); + ) { + log_rpc!(error "private route missing: {}", e); + } } } } @@ -761,7 +767,11 @@ impl RPCProcessor { } = &dest { let rss = self.routing_table.route_spec_store(); - rss.mark_remote_private_route_used(&private_route.public_key, intf::get_timestamp()); + if let Err(e) = + rss.mark_remote_private_route_used(&private_route.public_key, intf::get_timestamp()) + { + log_rpc!(error "private route missing: {}", e); + } } // Pass back waitable reply completion diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 2b747695..ef0ecef7 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -7,7 +7,7 @@ use routing_table::*; #[derive(Default, Debug)] struct DebugCache { - imported_routes: Vec, + imported_routes: Vec, } static DEBUG_CACHE: Mutex = Mutex::new(DebugCache { @@ -123,10 +123,20 @@ fn get_destination(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option { + // Remove imported route + dc.imported_routes.remove(n); + info!("removed dead imported route {}", n); + return None; + } + Ok(v) => v, + }; Some(Destination::private_route( - r.clone(), + private_route, ss.unwrap_or(SafetySelection::Unsafe(Sequencing::NoPreference)), )) } else { @@ -734,17 +744,24 @@ impl VeilidAPI { let blob_dec = BASE64URL_NOPAD .decode(blob.as_bytes()) .map_err(VeilidAPIError::generic)?; - let pr = - RouteSpecStore::blob_to_private_route(blob_dec).map_err(VeilidAPIError::generic)?; + let rss = self.routing_table()?.route_spec_store(); + let pr_pubkey = rss + .import_remote_private_route(blob_dec) + .map_err(VeilidAPIError::generic)?; let mut dc = DEBUG_CACHE.lock(); let n = dc.imported_routes.len(); - let out = format!("Private route #{} imported: {}", n, pr.public_key); - dc.imported_routes.push(pr); + let out = format!("Private route #{} imported: {}", n, pr_pubkey); + dc.imported_routes.push(pr_pubkey); return Ok(out); } + async fn debug_route_test(&self, _args: Vec) -> Result { + let out = "xxx".to_string(); + return Ok(out); + } + async fn debug_route(&self, args: String) -> Result { let args: Vec = args.split_whitespace().map(|s| s.to_owned()).collect(); @@ -764,6 +781,8 @@ impl VeilidAPI { self.debug_route_list(args).await } else if command == "import" { self.debug_route_import(args).await + } else if command == "test" { + self.debug_route_test(args).await } else { Ok(">>> Unknown command\n".to_owned()) } @@ -791,6 +810,7 @@ impl VeilidAPI { print list import + test is: * direct: [+][] diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 0512a391..605fcf90 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -1,12 +1,10 @@ #![allow(dead_code)] mod debug; -mod privacy; mod routing_context; mod serialize_helpers; pub use debug::*; -pub use privacy::*; pub use routing_context::*; pub use serialize_helpers::*; @@ -25,12 +23,13 @@ pub use intf::BlockStore; pub use intf::ProtectedStore; pub use intf::TableStore; pub use network_manager::NetworkManager; -pub use routing_table::{NodeRef, NodeRefBase, RoutingTable}; +pub use routing_table::{NodeRef, NodeRefBase}; use core::fmt; use core_context::{api_shutdown, VeilidCoreContext}; use enumset::*; use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; +use routing_table::{RouteSpecStore, RoutingTable}; use rpc_processor::*; use serde::*; use xx::*; @@ -86,8 +85,8 @@ pub enum VeilidAPIError { Timeout, #[error("Shutdown")] Shutdown, - #[error("Node not found: {node_id}")] - NodeNotFound { node_id: NodeId }, + #[error("Key not found: {key}")] + KeyNotFound { key: DHTKey }, #[error("No connection: {message}")] NoConnection { message: String }, #[error("No peer info: {node_id}")] @@ -123,11 +122,13 @@ impl VeilidAPIError { pub fn shutdown() -> Self { Self::Shutdown } - pub fn node_not_found(node_id: NodeId) -> Self { - Self::NodeNotFound { node_id } + pub fn key_not_found(key: DHTKey) -> Self { + Self::KeyNotFound { key } } - pub fn no_connection(message: String) -> Self { - Self::NoConnection { message } + pub fn no_connection(msg: T) -> Self { + Self::NoConnection { + message: msg.to_string(), + } } pub fn no_peer_info(node_id: NodeId) -> Self { Self::NoPeerInfo { node_id } @@ -2681,6 +2682,13 @@ impl VeilidAPI { } Err(VeilidAPIError::NotInitialized) } + pub fn routing_table(&self) -> Result { + let inner = self.inner.lock(); + if let Some(context) = &inner.context { + return Ok(context.attachment_manager.network_manager().routing_table()); + } + Err(VeilidAPIError::NotInitialized) + } //////////////////////////////////////////////////////////////// // Attach/Detach @@ -2732,6 +2740,64 @@ impl VeilidAPI { RoutingContext::new(self.clone()) } + //////////////////////////////////////////////////////////////// + // Private route allocation + + #[instrument(level = "debug", skip(self))] + pub async fn new_default_private_route(&self) -> Result<(DHTKey, Vec), VeilidAPIError> { + let config = self.config()?; + let c = config.get(); + self.new_private_route( + Stability::LowLatency, + Sequencing::NoPreference, + c.network.rpc.default_route_hop_count.into(), + ) + .await + } + + #[instrument(level = "debug", skip(self))] + pub async fn new_private_route( + &self, + stability: Stability, + sequencing: Sequencing, + hop_count: usize, + ) -> Result<(DHTKey, Vec), VeilidAPIError> { + let rss = self.routing_table()?.route_spec_store(); + let r = rss + .allocate_route( + stability, + sequencing, + hop_count, + Direction::Inbound.into(), + &[], + ) + .map_err(VeilidAPIError::internal)?; + let Some(pr_pubkey) = r else { + return Err(VeilidAPIError::generic("unable to allocate route")); + }; + if !rss + .test_route(&pr_pubkey) + .await + .map_err(VeilidAPIError::no_connection)? + { + rss.release_route(pr_pubkey) + .map_err(VeilidAPIError::generic)?; + return Err(VeilidAPIError::generic("allocated route failed to test")); + } + let private_route = rss + .assemble_private_route(&pr_pubkey, Some(true)) + .map_err(VeilidAPIError::generic)?; + let blob = match RouteSpecStore::private_route_to_blob(&private_route) { + Ok(v) => v, + Err(e) => { + rss.release_route(pr_pubkey) + .map_err(VeilidAPIError::generic)?; + return Err(VeilidAPIError::internal(e)); + } + }; + Ok((pr_pubkey, blob)) + } + //////////////////////////////////////////////////////////////// // App Calls diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index 47efb369..8e17b7de 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -4,7 +4,7 @@ use super::*; #[derive(Clone, Debug)] pub enum Target { NodeId(NodeId), - PrivateRoute(PrivateRoute), + PrivateRoute(DHTKey), } pub struct RoutingContextInner {} @@ -115,7 +115,7 @@ impl RoutingContext { // Resolve node let mut nr = match rpc_processor.resolve_node(node_id.key).await { Ok(Some(nr)) => nr, - Ok(None) => return Err(VeilidAPIError::NodeNotFound { node_id }), + Ok(None) => return Err(VeilidAPIError::KeyNotFound { key: node_id.key }), Err(e) => return Err(e.into()), }; // Apply sequencing to match safety selection @@ -126,10 +126,17 @@ impl RoutingContext { safety_selection: self.unlocked_inner.safety_selection, }) } - Target::PrivateRoute(pr) => Ok(rpc_processor::Destination::PrivateRoute { - private_route: pr, - safety_selection: self.unlocked_inner.safety_selection, - }), + Target::PrivateRoute(pr) => { + // Get remote private route + let rss = self.api.routing_table()?.route_spec_store(); + let private_route = rss + .get_remote_private_route(&pr) + .map_err(|_| VeilidAPIError::KeyNotFound { key: pr })?; + Ok(rpc_processor::Destination::PrivateRoute { + private_route, + safety_selection: self.unlocked_inner.safety_selection, + }) + } } } From cc49eaa6c52faf1cadd2ef6562e40b1bb641805a Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 22 Nov 2022 18:39:52 -0500 Subject: [PATCH 67/67] cli fix --- veilid-cli/src/ui.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/veilid-cli/src/ui.rs b/veilid-cli/src/ui.rs index bacdc3ad..9f127056 100644 --- a/veilid-cli/src/ui.rs +++ b/veilid-cli/src/ui.rs @@ -7,7 +7,6 @@ use cursive::event::*; use cursive::theme::*; use cursive::traits::*; use cursive::utils::markup::StyledString; -use cursive::view::ScrollStrategy; use cursive::views::*; use cursive::Cursive; use cursive::CursiveRunnable;