liz/veilid
1
0
Fork 0
Code Pull Requests Packages Releases Activity

app call/message and private routing checkpoint

Browse Source
This commit is contained in:
John Smith
2022-09-25 18:04:53 -04:00
parent 507d02974c
commit baa1714943
29 changed files with 1139 additions and 538 deletions
Download Patch File Download Diff File Expand all files Collapse all files
veilid-core/src/rpc_processor/coders/operations/answer.rs
+8 -3
View File
@@ -11,9 +11,6 @@ impl RPCAnswer {
pub fn new(detail: RPCAnswerDetail) -> Self {
Self { detail }
}
// pub fn detail(&self) -> &RPCAnswerDetail {
// &self.detail
// }
pub fn into_detail(self) -> RPCAnswerDetail {
self.detail
}
@@ -35,6 +32,7 @@ impl RPCAnswer {
pub enum RPCAnswerDetail {
StatusA(RPCOperationStatusA),
FindNodeA(RPCOperationFindNodeA),
AppCallA(RPCOperationAppCallA),
GetValueA(RPCOperationGetValueA),
SetValueA(RPCOperationSetValueA),
WatchValueA(RPCOperationWatchValueA),
@@ -50,6 +48,7 @@ impl RPCAnswerDetail {
match self {
RPCAnswerDetail::StatusA(_) => "StatusA",
RPCAnswerDetail::FindNodeA(_) => "FindNodeA",
RPCAnswerDetail::AppCallA(_) => "AppCallA",
RPCAnswerDetail::GetValueA(_) => "GetValueA",
RPCAnswerDetail::SetValueA(_) => "SetValueA",
RPCAnswerDetail::WatchValueA(_) => "WatchValueA",
@@ -76,6 +75,11 @@ impl RPCAnswerDetail {
let out = RPCOperationFindNodeA::decode(&op_reader)?;
RPCAnswerDetail::FindNodeA(out)
}
veilid_capnp::answer::detail::AppCallA(r) => {
let op_reader = r.map_err(RPCError::protocol)?;
let out = RPCOperationAppCallA::decode(&op_reader)?;
RPCAnswerDetail::AppCallA(out)
}
veilid_capnp::answer::detail::GetValueA(r) => {
let op_reader = r.map_err(RPCError::protocol)?;
let out = RPCOperationGetValueA::decode(&op_reader)?;
@@ -126,6 +130,7 @@ impl RPCAnswerDetail {
match self {
RPCAnswerDetail::StatusA(d) => d.encode(&mut builder.reborrow().init_status_a()),
RPCAnswerDetail::FindNodeA(d) => d.encode(&mut builder.reborrow().init_find_node_a()),
RPCAnswerDetail::AppCallA(d) => d.encode(&mut builder.reborrow().init_app_call_a()),
RPCAnswerDetail::GetValueA(d) => d.encode(&mut builder.reborrow().init_get_value_a()),
RPCAnswerDetail::SetValueA(d) => d.encode(&mut builder.reborrow().init_set_value_a()),
RPCAnswerDetail::WatchValueA(d) => {
veilid-core/src/rpc_processor/coders/operations/mod.rs
+4
View File
@@ -1,5 +1,7 @@
mod answer;
mod operation;
mod operation_app_call;
mod operation_app_message;
mod operation_cancel_tunnel;
mod operation_complete_tunnel;
mod operation_find_block;
@@ -22,6 +24,8 @@ mod statement;
pub use answer::*;
pub use operation::*;
pub use operation_app_call::*;
pub use operation_app_message::*;
pub use operation_cancel_tunnel::*;
pub use operation_complete_tunnel::*;
pub use operation_find_block::*;
veilid-core/src/rpc_processor/coders/operations/operation_app_call.rs
+44
View File
@@ -0,0 +1,44 @@
use crate::*;
use rpc_processor::*;
#[derive(Debug, Clone)]
pub struct RPCOperationAppCallQ {
pub message: Vec<u8>,
}
impl RPCOperationAppCallQ {
pub fn decode(
reader: &veilid_capnp::operation_app_call_q::Reader,
) -> Result<RPCOperationAppCallQ, RPCError> {
let message = reader.get_message().map_err(RPCError::protocol)?.to_vec();
Ok(RPCOperationAppCallQ { message })
}
pub fn encode(
&self,
builder: &mut veilid_capnp::operation_app_call_q::Builder,
) -> Result<(), RPCError> {
builder.set_message(&self.message);
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct RPCOperationAppCallA {
pub message: Vec<u8>,
}
impl RPCOperationAppCallA {
pub fn decode(
reader: &veilid_capnp::operation_app_call_a::Reader,
) -> Result<RPCOperationAppCallA, RPCError> {
let message = reader.get_message().map_err(RPCError::protocol)?.to_vec();
Ok(RPCOperationAppCallA { message })
}
pub fn encode(
&self,
builder: &mut veilid_capnp::operation_app_call_a::Builder,
) -> Result<(), RPCError> {
builder.set_message(&self.message);
Ok(())
}
}
veilid-core/src/rpc_processor/coders/operations/operation_app_message.rs
+23
View File
@@ -0,0 +1,23 @@
use crate::*;
use rpc_processor::*;
#[derive(Debug, Clone)]
pub struct RPCOperationAppMessage {
pub message: Vec<u8>,
}
impl RPCOperationAppMessage {
pub fn decode(
reader: &veilid_capnp::operation_app_message::Reader,
) -> Result<RPCOperationAppMessage, RPCError> {
let message = reader.get_message().map_err(RPCError::protocol)?.to_vec();
Ok(RPCOperationAppMessage { message })
}
pub fn encode(
&self,
builder: &mut veilid_capnp::operation_app_message::Builder,
) -> Result<(), RPCError> {
builder.set_message(&self.message);
Ok(())
}
}
veilid-core/src/rpc_processor/coders/operations/question.rs
+8
View File
@@ -40,6 +40,7 @@ impl RPCQuestion {
pub enum RPCQuestionDetail {
StatusQ(RPCOperationStatusQ),
FindNodeQ(RPCOperationFindNodeQ),
AppCallQ(RPCOperationAppCallQ),
GetValueQ(RPCOperationGetValueQ),
SetValueQ(RPCOperationSetValueQ),
WatchValueQ(RPCOperationWatchValueQ),
@@ -55,6 +56,7 @@ impl RPCQuestionDetail {
match self {
RPCQuestionDetail::StatusQ(_) => "StatusQ",
RPCQuestionDetail::FindNodeQ(_) => "FindNodeQ",
RPCQuestionDetail::AppCallQ(_) => "AppCallQ",
RPCQuestionDetail::GetValueQ(_) => "GetValueQ",
RPCQuestionDetail::SetValueQ(_) => "SetValueQ",
RPCQuestionDetail::WatchValueQ(_) => "WatchValueQ",
@@ -81,6 +83,11 @@ impl RPCQuestionDetail {
let out = RPCOperationFindNodeQ::decode(&op_reader)?;
RPCQuestionDetail::FindNodeQ(out)
}
veilid_capnp::question::detail::Which::AppCallQ(r) => {
let op_reader = r.map_err(RPCError::protocol)?;
let out = RPCOperationAppCallQ::decode(&op_reader)?;
RPCQuestionDetail::AppCallQ(out)
}
veilid_capnp::question::detail::GetValueQ(r) => {
let op_reader = r.map_err(RPCError::protocol)?;
let out = RPCOperationGetValueQ::decode(&op_reader)?;
@@ -131,6 +138,7 @@ impl RPCQuestionDetail {
match self {
RPCQuestionDetail::StatusQ(d) => d.encode(&mut builder.reborrow().init_status_q()),
RPCQuestionDetail::FindNodeQ(d) => d.encode(&mut builder.reborrow().init_find_node_q()),
RPCQuestionDetail::AppCallQ(d) => d.encode(&mut builder.reborrow().init_app_call_q()),
RPCQuestionDetail::GetValueQ(d) => d.encode(&mut builder.reborrow().init_get_value_q()),
RPCQuestionDetail::SetValueQ(d) => d.encode(&mut builder.reborrow().init_set_value_q()),
RPCQuestionDetail::WatchValueQ(d) => {
veilid-core/src/rpc_processor/coders/operations/statement.rs
+10
View File
@@ -42,6 +42,7 @@ pub enum RPCStatementDetail {
ValueChanged(RPCOperationValueChanged),
Signal(RPCOperationSignal),
ReturnReceipt(RPCOperationReturnReceipt),
AppMessage(RPCOperationAppMessage),
}
impl RPCStatementDetail {
@@ -53,6 +54,7 @@ impl RPCStatementDetail {
RPCStatementDetail::ValueChanged(_) => "ValueChanged",
RPCStatementDetail::Signal(_) => "Signal",
RPCStatementDetail::ReturnReceipt(_) => "ReturnReceipt",
RPCStatementDetail::AppMessage(_) => "AppMessage",
}
}
pub fn decode(
@@ -91,6 +93,11 @@ impl RPCStatementDetail {
let out = RPCOperationReturnReceipt::decode(&op_reader)?;
RPCStatementDetail::ReturnReceipt(out)
}
veilid_capnp::statement::detail::AppMessage(r) => {
let op_reader = r.map_err(RPCError::protocol)?;
let out = RPCOperationAppMessage::decode(&op_reader)?;
RPCStatementDetail::AppMessage(out)
}
};
Ok(out)
}
@@ -113,6 +120,9 @@ impl RPCStatementDetail {
RPCStatementDetail::ReturnReceipt(d) => {
d.encode(&mut builder.reborrow().init_return_receipt())
}
RPCStatementDetail::AppMessage(d) => {
d.encode(&mut builder.reborrow().init_app_message())
}
}
}
}
veilid-core/src/rpc_processor/coders/private_safety_route.rs
-74
View File
@@ -2,80 +2,6 @@ use super::*;
////////////////////////////////////////////////////////////////////////////////////////////////////
#[derive(Clone, Debug)]
pub struct RouteHopData {
pub nonce: Nonce,
pub blob: Vec<u8>,
}
#[derive(Clone, Debug)]
pub struct RouteHop {
pub dial_info: NodeDialInfo,
pub next_hop: Option<RouteHopData>,
}
#[derive(Clone, Debug)]
pub struct PrivateRoute {
pub public_key: DHTKey,
pub hop_count: u8,
pub hops: Option<RouteHop>,
}
impl PrivateRoute {
pub fn new_stub(public_key: DHTKey) -> Self {
Self {
public_key,
hop_count: 0,
hops: None,
}
}
}
impl fmt::Display for PrivateRoute {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"PR({:?}+{}{})",
self.public_key,
self.hop_count,
if let Some(hops) = &self.hops {
format!("->{}", hops.dial_info)
} else {
"".to_owned()
}
)
}
}
#[derive(Clone, Debug)]
pub enum SafetyRouteHops {
Data(RouteHopData),
Private(PrivateRoute),
}
#[derive(Clone, Debug)]
pub struct SafetyRoute {
pub public_key: DHTKey,
pub hop_count: u8,
pub hops: SafetyRouteHops,
}
impl fmt::Display for SafetyRoute {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"SR({:?}+{}{})",
self.public_key,
self.hop_count,
match &self.hops {
SafetyRouteHops::Data(_) => "".to_owned(),
SafetyRouteHops::Private(p) => format!("->{}", p),
}
)
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
pub fn encode_route_hop_data(
route_hop_data: &RouteHopData,
builder: &mut veilid_capnp::route_hop_data::Builder,
veilid-core/src/rpc_processor/mod.rs
+147 -172
View File
@@ -1,6 +1,9 @@
mod coders;
mod destination;
mod operation_waiter;
mod private_route;
mod rpc_app_call;
mod rpc_app_message;
mod rpc_cancel_tunnel;
mod rpc_complete_tunnel;
mod rpc_error;
@@ -20,6 +23,7 @@ mod rpc_value_changed;
mod rpc_watch_value;
pub use destination::*;
pub use operation_waiter::*;
pub use private_route::*;
pub use rpc_error::*;
@@ -108,8 +112,7 @@ where
#[derive(Debug)]
struct WaitableReply {
op_id: OperationId,
eventual: EventualValue<(Option<Id>, RPCMessage)>,
handle: OperationWaitHandle<RPCMessage>,
timeout: u64,
node_ref: NodeRef,
send_ts: u64,
@@ -138,62 +141,162 @@ struct RenderedOperation {
/////////////////////////////////////////////////////////////////////
pub struct RPCProcessorInner {
network_manager: NetworkManager,
routing_table: RoutingTable,
node_id: DHTKey,
node_id_secret: DHTKeySecret,
send_channel: Option<flume::Sender<(Option<Id>, RPCMessageEncoded)>>,
timeout: u64,
max_route_hop_count: usize,
waiting_rpc_table: BTreeMap<OperationId, EventualValue<(Option<Id>, RPCMessage)>>,
stop_source: Option<StopSource>,
worker_join_handles: Vec<MustJoinHandle<()>>,
}
pub struct RPCProcessorUnlockedInner {
node_id: DHTKey,
node_id_secret: DHTKeySecret,
timeout: u64,
queue_size: u32,
concurrency: u32,
max_route_hop_count: usize,
validate_dial_info_receipt_time_ms: u32,
update_callback: UpdateCallback,
waiting_rpc_table: OperationWaiter<RPCMessage>,
waiting_app_call_table: OperationWaiter<Vec<u8>>,
}
#[derive(Clone)]
pub struct RPCProcessor {
crypto: Crypto,
config: VeilidConfig,
network_manager: NetworkManager,
routing_table: RoutingTable,
inner: Arc<Mutex<RPCProcessorInner>>,
unlocked_inner: Arc<RPCProcessorUnlockedInner>,
}
impl RPCProcessor {
fn new_inner(network_manager: NetworkManager) -> RPCProcessorInner {
fn new_inner() -> RPCProcessorInner {
RPCProcessorInner {
network_manager: network_manager.clone(),
routing_table: network_manager.routing_table(),
node_id: DHTKey::default(),
node_id_secret: DHTKeySecret::default(),
send_channel: None,
timeout: 10000000,
max_route_hop_count: 7,
waiting_rpc_table: BTreeMap::new(),
stop_source: None,
worker_join_handles: Vec::new(),
}
}
pub fn new(network_manager: NetworkManager) -> Self {
fn new_unlocked_inner(
config: VeilidConfig,
update_callback: UpdateCallback,
) -> RPCProcessorUnlockedInner {
// make local copy of node id for easy access
let c = config.get();
let node_id = c.network.node_id;
let node_id_secret = c.network.node_id_secret;
// set up channel
let mut concurrency = c.network.rpc.concurrency;
let mut queue_size = c.network.rpc.queue_size;
let mut timeout = ms_to_us(c.network.rpc.timeout_ms);
let mut max_route_hop_count = c.network.rpc.max_route_hop_count as usize;
if concurrency == 0 {
concurrency = intf::get_concurrency() / 2;
if concurrency == 0 {
concurrency = 1;
}
}
if queue_size == 0 {
queue_size = 1024;
}
if timeout == 0 {
timeout = 10000000;
}
if max_route_hop_count == 0 {
max_route_hop_count = 7usize;
}
let validate_dial_info_receipt_time_ms = c.network.dht.validate_dial_info_receipt_time_ms;
RPCProcessorUnlockedInner {
node_id,
node_id_secret,
timeout,
queue_size,
concurrency,
max_route_hop_count,
validate_dial_info_receipt_time_ms,
update_callback,
waiting_rpc_table: OperationWaiter::new(),
waiting_app_call_table: OperationWaiter::new(),
}
}
pub fn new(network_manager: NetworkManager, update_callback: UpdateCallback) -> Self {
let config = network_manager.config();
Self {
crypto: network_manager.crypto(),
config: network_manager.config(),
inner: Arc::new(Mutex::new(Self::new_inner(network_manager))),
config: config.clone(),
network_manager: network_manager.clone(),
routing_table: network_manager.routing_table(),
inner: Arc::new(Mutex::new(Self::new_inner())),
unlocked_inner: Arc::new(Self::new_unlocked_inner(config, update_callback)),
}
}
pub fn network_manager(&self) -> NetworkManager {
self.inner.lock().network_manager.clone()
self.network_manager.clone()
}
pub fn routing_table(&self) -> RoutingTable {
self.inner.lock().routing_table.clone()
self.routing_table.clone()
}
pub fn node_id(&self) -> DHTKey {
self.inner.lock().node_id
//////////////////////////////////////////////////////////////////////
#[instrument(level = "debug", skip_all, err)]
pub async fn startup(&self) -> EyreResult<()> {
trace!("startup rpc processor");
let mut inner = self.inner.lock();
let channel = flume::bounded(self.unlocked_inner.queue_size as usize);
inner.send_channel = Some(channel.0.clone());
inner.stop_source = Some(StopSource::new());
// spin up N workers
trace!(
"Spinning up {} RPC workers",
self.unlocked_inner.concurrency
);
for _ in 0..self.unlocked_inner.concurrency {
let this = self.clone();
let receiver = channel.1.clone();
let jh = intf::spawn(Self::rpc_worker(
this,
inner.stop_source.as_ref().unwrap().token(),
receiver,
));
inner.worker_join_handles.push(jh);
}
Ok(())
}
pub fn node_id_secret(&self) -> DHTKeySecret {
self.inner.lock().node_id_secret
#[instrument(level = "debug", skip_all)]
pub async fn shutdown(&self) {
debug!("starting rpc processor shutdown");
// Stop the rpc workers
let mut unord = FuturesUnordered::new();
{
let mut inner = self.inner.lock();
// take the join handles out
for h in inner.worker_join_handles.drain(..) {
unord.push(h);
}
// drop the stop
drop(inner.stop_source.take());
}
debug!("stopping {} rpc worker tasks", unord.len());
// Wait for them to complete
while unord.next().await.is_some() {}
debug!("resetting rpc processor state");
// Release the rpc processor
*self.inner.lock() = Self::new_inner();
debug!("finished rpc processor shutdown");
}
//////////////////////////////////////////////////////////////////////
@@ -278,71 +381,18 @@ impl RPCProcessor {
})
}
// set up wait for reply
fn add_op_id_waiter(&self, op_id: OperationId) -> EventualValue<(Option<Id>, RPCMessage)> {
let mut inner = self.inner.lock();
let e = EventualValue::new();
inner.waiting_rpc_table.insert(op_id, e.clone());
e
}
// remove wait for reply
fn cancel_op_id_waiter(&self, op_id: OperationId) {
let mut inner = self.inner.lock();
inner.waiting_rpc_table.remove(&op_id);
}
// complete the reply
#[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)]
async fn complete_op_id_waiter(&self, msg: RPCMessage) -> Result<(), RPCError> {
let op_id = msg.operation.op_id();
let eventual = {
let mut inner = self.inner.lock();
inner
.waiting_rpc_table
.remove(&op_id)
.ok_or_else(RPCError::else_internal(format!(
"Unmatched operation id: {:#?}",
msg
)))?
};
eventual.resolve((Span::current().id(), msg)).await;
Ok(())
}
// wait for reply
async fn do_wait_for_reply(
&self,
waitable_reply: &WaitableReply,
) -> Result<TimeoutOr<(RPCMessage, u64)>, RPCError> {
let timeout_ms = u32::try_from(waitable_reply.timeout / 1000u64)
.map_err(RPCError::map_internal("invalid timeout"))?;
// wait for eventualvalue
let start_ts = intf::get_timestamp();
let res = intf::timeout(timeout_ms, waitable_reply.eventual.instance())
.await
.into_timeout_or();
Ok(res.map(|res| {
let (_span_id, rpcreader) = res.take_value().unwrap();
let end_ts = intf::get_timestamp();
// fixme: causes crashes? "Missing otel data span extensions"??
//Span::current().follows_from(span_id);
(rpcreader, end_ts - start_ts)
}))
}
#[instrument(level = "trace", skip(self, waitable_reply), err)]
async fn wait_for_reply(
&self,
waitable_reply: WaitableReply,
) -> Result<TimeoutOr<(RPCMessage, u64)>, RPCError> {
let out = self.do_wait_for_reply(&waitable_reply).await;
let out = self
.unlocked_inner
.waiting_rpc_table
.wait_for_op(waitable_reply.handle, waitable_reply.timeout)
.await;
match &out {
Err(_) | Ok(TimeoutOr::Timeout) => {
self.cancel_op_id_waiter(waitable_reply.op_id);
waitable_reply.node_ref.stats_question_lost();
}
Ok(TimeoutOr::Value((rpcreader, _))) => {
@@ -476,7 +526,7 @@ impl RPCProcessor {
}
// Verify hop count isn't larger than out maximum routed hop count
if out_hop_count > self.inner.lock().max_route_hop_count {
if out_hop_count > self.unlocked_inner.max_route_hop_count {
return Err(RPCError::internal("hop count too long for route"))
.map_err(logthru_rpc!(warn));
}
@@ -574,10 +624,10 @@ impl RPCProcessor {
// Calculate answer timeout
// Timeout is number of hops times the timeout per hop
let timeout = self.inner.lock().timeout * (hop_count as u64);
let timeout = self.unlocked_inner.timeout * (hop_count as u64);
// Set up op id eventual
let eventual = self.add_op_id_waiter(op_id);
let handle = self.unlocked_inner.waiting_rpc_table.add_op_waiter(op_id);
// Send question
let bytes = message.len() as u64;
@@ -588,13 +638,11 @@ impl RPCProcessor {
.await
.map_err(|e| {
// If we're returning an error, clean up
self.cancel_op_id_waiter(op_id);
node_ref
.stats_failed_to_send(send_ts, true);
RPCError::network(e)
})? => {
// If we couldn't send we're still cleaning up
self.cancel_op_id_waiter(op_id);
node_ref
.stats_failed_to_send(send_ts, true);
}
@@ -605,8 +653,7 @@ impl RPCProcessor {
// Pass back waitable reply completion
Ok(NetworkResult::value(WaitableReply {
op_id,
eventual,
handle,
timeout,
node_ref,
send_ts,
@@ -794,7 +841,7 @@ impl RPCProcessor {
let mut opt_sender_nr: Option<NodeRef> = None;
if let Some(sender_node_info) = operation.sender_node_info() {
// Sender NodeInfo was specified, update our routing table with it
if !self.filter_node_info(RoutingDomain::PublicInternet, &sender_node_info.node_info) {
if !self.filter_node_info(routing_domain, &sender_node_info.node_info) {
return Err(RPCError::invalid_format(
"sender signednodeinfo has invalid peer scope",
));
@@ -853,6 +900,7 @@ impl RPCProcessor {
RPCOperationKind::Question(q) => match q.detail() {
RPCQuestionDetail::StatusQ(_) => self.process_status_q(msg).await,
RPCQuestionDetail::FindNodeQ(_) => self.process_find_node_q(msg).await,
RPCQuestionDetail::AppCallQ(_) => self.process_app_call_q(msg).await,
RPCQuestionDetail::GetValueQ(_) => self.process_get_value_q(msg).await,
RPCQuestionDetail::SetValueQ(_) => self.process_set_value_q(msg).await,
RPCQuestionDetail::WatchValueQ(_) => self.process_watch_value_q(msg).await,
@@ -871,8 +919,14 @@ impl RPCProcessor {
RPCStatementDetail::ValueChanged(_) => self.process_value_changed(msg).await,
RPCStatementDetail::Signal(_) => self.process_signal(msg).await,
RPCStatementDetail::ReturnReceipt(_) => self.process_return_receipt(msg).await,
RPCStatementDetail::AppMessage(_) => self.process_app_message(msg).await,
},
RPCOperationKind::Answer(_) => self.complete_op_id_waiter(msg).await,
RPCOperationKind::Answer(_) => {
self.unlocked_inner
.waiting_rpc_table
.complete_op_waiter(msg.operation.op_id(), msg)
.await
}
}
}
@@ -908,85 +962,6 @@ impl RPCProcessor {
}
}
#[instrument(level = "debug", skip_all, err)]
pub async fn startup(&self) -> EyreResult<()> {
trace!("startup rpc processor");
let mut inner = self.inner.lock();
// make local copy of node id for easy access
let c = self.config.get();
inner.node_id = c.network.node_id;
inner.node_id_secret = c.network.node_id_secret;
// set up channel
let mut concurrency = c.network.rpc.concurrency;
let mut queue_size = c.network.rpc.queue_size;
let mut timeout = ms_to_us(c.network.rpc.timeout_ms);
let mut max_route_hop_count = c.network.rpc.max_route_hop_count as usize;
if concurrency == 0 {
concurrency = intf::get_concurrency() / 2;
if concurrency == 0 {
concurrency = 1;
}
}
if queue_size == 0 {
queue_size = 1024;
}
if timeout == 0 {
timeout = 10000000;
}
if max_route_hop_count == 0 {
max_route_hop_count = 7usize;
}
inner.timeout = timeout;
inner.max_route_hop_count = max_route_hop_count;
let channel = flume::bounded(queue_size as usize);
inner.send_channel = Some(channel.0.clone());
inner.stop_source = Some(StopSource::new());
// spin up N workers
trace!("Spinning up {} RPC workers", concurrency);
for _ in 0..concurrency {
let this = self.clone();
let receiver = channel.1.clone();
let jh = intf::spawn(Self::rpc_worker(
this,
inner.stop_source.as_ref().unwrap().token(),
receiver,
));
inner.worker_join_handles.push(jh);
}
Ok(())
}
#[instrument(level = "debug", skip_all)]
pub async fn shutdown(&self) {
debug!("starting rpc processor shutdown");
// Stop the rpc workers
let mut unord = FuturesUnordered::new();
{
let mut inner = self.inner.lock();
// take the join handles out
for h in inner.worker_join_handles.drain(..) {
unord.push(h);
}
// drop the stop
drop(inner.stop_source.take());
}
debug!("stopping {} rpc worker tasks", unord.len());
// Wait for them to complete
while unord.next().await.is_some() {}
debug!("resetting rpc processor state");
// Release the rpc processor
*self.inner.lock() = Self::new_inner(self.network_manager());
debug!("finished rpc processor shutdown");
}
#[instrument(level = "trace", skip(self, body), err)]
pub fn enqueue_message(
&self,
veilid-core/src/rpc_processor/operation_waiter.rs
+136
View File
@@ -0,0 +1,136 @@
use super::*;
#[derive(Debug)]
pub struct OperationWaitHandle<T>
where
T: Unpin,
{
waiter: OperationWaiter<T>,
op_id: OperationId,
eventual_instance: Option<EventualValueFuture<(Option<Id>, T)>>,
}
impl<T> Drop for OperationWaitHandle<T>
where
T: Unpin,
{
fn drop(&mut self) {
if self.eventual_instance.is_some() {
self.waiter.cancel_op_waiter(self.op_id);
}
}
}
#[derive(Debug)]
pub struct OperationWaiterInner<T>
where
T: Unpin,
{
waiting_op_table: HashMap<OperationId, EventualValue<(Option<Id>, T)>>,
}
#[derive(Debug)]
pub struct OperationWaiter<T>
where
T: Unpin,
{
inner: Arc<Mutex<OperationWaiterInner<T>>>,
}
impl<T> Clone for OperationWaiter<T>
where
T: Unpin,
{
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
}
}
}
impl<T> OperationWaiter<T>
where
T: Unpin,
{
pub fn new() -> Self {
Self {
inner: Arc::new(Mutex::new(OperationWaiterInner {
waiting_op_table: HashMap::new(),
})),
}
}
// set up wait for op
pub fn add_op_waiter(&self, op_id: OperationId) -> OperationWaitHandle<T> {
let mut inner = self.inner.lock();
let e = EventualValue::new();
if inner.waiting_op_table.insert(op_id, e.clone()).is_some() {
error!(
"add_op_waiter collision should not happen for op_id {}",
op_id
);
}
OperationWaitHandle {
waiter: self.clone(),
op_id,
eventual_instance: Some(e.instance()),
}
}
// remove wait for op
fn cancel_op_waiter(&self, op_id: OperationId) {
let mut inner = self.inner.lock();
inner.waiting_op_table.remove(&op_id);
}
// complete the app call
#[instrument(level = "trace", skip(self, message), err)]
pub async fn complete_op_waiter(&self, op_id: OperationId, message: T) -> Result<(), RPCError> {
let eventual = {
let mut inner = self.inner.lock();
inner
.waiting_op_table
.remove(&op_id)
.ok_or_else(RPCError::else_internal(format!(
"Unmatched app call id, possibly too late for timeout: {}",
op_id
)))?
};
eventual.resolve((Span::current().id(), message)).await;
Ok(())
}
pub async fn wait_for_op(
&self,
mut handle: OperationWaitHandle<T>,
timeout: u64,
) -> Result<TimeoutOr<(T, u64)>, RPCError> {
let timeout_ms = u32::try_from(timeout / 1000u64)
.map_err(|e| RPCError::map_internal("invalid timeout")(e))?;
// Take the instance
// After this, we must manually cancel since the cancel on handle drop is disabled
let eventual_instance = handle.eventual_instance.take().unwrap();
// wait for eventualvalue
let start_ts = intf::get_timestamp();
let res = intf::timeout(timeout_ms, eventual_instance)
.await
.into_timeout_or();
Ok(res
.on_timeout(|| {
log_rpc!(debug "op wait timed out: {}", handle.op_id);
self.cancel_op_waiter(handle.op_id);
})
.map(|res| {
let (_span_id, ret) = res.take_value().unwrap();
let end_ts = intf::get_timestamp();
// fixme: causes crashes? "Missing otel data span extensions"??
//Span::current().follows_from(span_id);
(ret, end_ts - start_ts)
}))
}
}
veilid-core/src/rpc_processor/private_route.rs
+1 -1
View File
@@ -11,7 +11,7 @@ impl RPCProcessor {
let pr_hopcount = private_route.hop_count as usize;
let sr_hopcount = safety_route_spec.hops.len();
let hopcount = 1 + sr_hopcount + pr_hopcount;
if hopcount > self.inner.lock().max_route_hop_count {
if hopcount > self.unlocked_inner.max_route_hop_count {
return Err(RPCError::internal("hop count too long for route"));
}
veilid-core/src/rpc_processor/rpc_app_call.rs
+99
View File
@@ -0,0 +1,99 @@
use super::*;
impl RPCProcessor {
// Sends a high level app request and wait for response
// Can be sent via all methods including relays and routes
#[instrument(level = "trace", skip(self), ret, err)]
pub async fn rpc_call_app_call(
self,
dest: Destination,
message: Vec<u8>,
) -> Result<NetworkResult<Answer<Vec<u8>>>, RPCError> {
let app_call_q = RPCOperationAppCallQ { message };
let question = RPCQuestion::new(RespondTo::Sender, RPCQuestionDetail::AppCallQ(app_call_q));
// Send the app call question
let waitable_reply = network_result_try!(self.question(dest, question).await?);
// Wait for reply
let (msg, latency) = match self.wait_for_reply(waitable_reply).await? {
TimeoutOr::Timeout => return Ok(NetworkResult::Timeout),
TimeoutOr::Value(v) => v,
};
// Get the right answer type
let app_call_a = match msg.operation.into_kind() {
RPCOperationKind::Answer(a) => match a.into_detail() {
RPCAnswerDetail::AppCallA(a) => a,
_ => return Err(RPCError::invalid_format("not an appcall answer")),
},
_ => return Err(RPCError::invalid_format("not an answer")),
};
Ok(NetworkResult::value(Answer::new(
latency,
app_call_a.message,
)))
}
#[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)]
pub(crate) async fn process_app_call_q(&self, msg: RPCMessage) -> Result<(), RPCError> {
// Get the question
let app_call_q = match msg.operation.kind() {
RPCOperationKind::Question(q) => match q.detail() {
RPCQuestionDetail::AppCallQ(q) => q,
_ => panic!("not an appcall question"),
},
_ => panic!("not a question"),
};
// Register a waiter for this app call
let id = msg.operation.op_id();
let handle = self.unlocked_inner.waiting_app_call_table.add_op_waiter(id);
// Pass the call up through the update callback
let sender = msg
.opt_sender_nr
.as_ref()
.map(|nr| NodeId::new(nr.node_id()));
let message = app_call_q.message.clone();
(self.unlocked_inner.update_callback)(VeilidUpdate::AppCall(VeilidAppCall {
sender,
message,
id,
}));
// Wait for an app call answer to come back from the app
let res = self
.unlocked_inner
.waiting_app_call_table
.wait_for_op(handle, self.unlocked_inner.timeout)
.await?;
let (message, _latency) = match res {
TimeoutOr::Timeout => {
// No message sent on timeout, but this isn't an error
log_rpc!(debug "App call timed out for id {}", id);
return Ok(());
}
TimeoutOr::Value(v) => v,
};
// Return the appcall answer
let app_call_a = RPCOperationAppCallA { message };
// Send status answer
let res = self
.answer(msg, RPCAnswer::new(RPCAnswerDetail::AppCallA(app_call_a)))
.await?;
tracing::Span::current().record("res", &tracing::field::display(res));
Ok(())
}
/// Exposed to API for apps to return app call answers
pub async fn app_call_reply(&self, id: u64, message: Vec<u8>) -> Result<(), RPCError> {
self.unlocked_inner
.waiting_app_call_table
.complete_op_waiter(id, message)
.await
}
}
veilid-core/src/rpc_processor/rpc_app_message.rs
+42
View File
@@ -0,0 +1,42 @@
use super::*;
impl RPCProcessor {
// Sends a high level app message
// Can be sent via all methods including relays and routes
#[instrument(level = "trace", skip(self), ret, err)]
pub async fn rpc_call_app_message(
self,
dest: Destination,
message: Vec<u8>,
) -> Result<NetworkResult<()>, RPCError> {
let app_message = RPCOperationAppMessage { message };
let statement = RPCStatement::new(RPCStatementDetail::AppMessage(app_message));
// Send the app message request
network_result_try!(self.statement(dest, statement).await?);
Ok(NetworkResult::value(()))
}
#[instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err)]
pub(crate) async fn process_app_message(&self, msg: RPCMessage) -> Result<(), RPCError> {
// Get the statement
let app_message = match msg.operation.into_kind() {
RPCOperationKind::Statement(s) => match s.into_detail() {
RPCStatementDetail::AppMessage(s) => s,
_ => panic!("not an app message"),
},
_ => panic!("not a statement"),
};
// Pass the message up through the update callback
let sender = msg.opt_sender_nr.map(|nr| NodeId::new(nr.node_id()));
let message = app_message.message;
(self.unlocked_inner.update_callback)(VeilidUpdate::AppMessage(VeilidAppMessage {
sender,
message,
}));
Ok(())
}
}
veilid-core/src/rpc_processor/rpc_error.rs
+12 -5
View File
@@ -3,8 +3,6 @@ use super::*;
#[derive(ThisError, Debug, Clone, PartialOrd, PartialEq, Eq, Ord)]
#[must_use]
pub enum RPCError {
#[error("[RPCError: Unreachable({0})]")]
Unreachable(DHTKey),
#[error("[RPCError: Unimplemented({0})]")]
Unimplemented(String),
#[error("[RPCError: InvalidFormat({0})]")]
@@ -18,9 +16,6 @@ pub enum RPCError {
}
impl RPCError {
pub fn unreachable(key: DHTKey) -> Self {
Self::Unreachable(key)
}
pub fn unimplemented<X: ToString>(x: X) -> Self {
Self::Unimplemented(x.to_string())
}
@@ -52,3 +47,15 @@ impl RPCError {
move |x| Self::Network(format!("{}: {}", message.to_string(), x.to_string()))
}
}
impl From<RPCError> for VeilidAPIError {
fn from(e: RPCError) -> Self {
match e {
RPCError::Unimplemented(message) => VeilidAPIError::Unimplemented { message },
RPCError::InvalidFormat(message) => VeilidAPIError::Generic { message },
RPCError::Protocol(message) => VeilidAPIError::Generic { message },
RPCError::Internal(message) => VeilidAPIError::Internal { message },
RPCError::Network(message) => VeilidAPIError::Generic { message },
}
}
}
veilid-core/src/rpc_processor/rpc_signal.rs
+1 -1
View File
@@ -24,7 +24,7 @@ impl RPCProcessor {
let signal = match msg.operation.into_kind() {
RPCOperationKind::Statement(s) => match s.into_detail() {
RPCStatementDetail::Signal(s) => s,
_ => panic!("not a node info update"),
_ => panic!("not a signal"),
},
_ => panic!("not a statement"),
};
veilid-core/src/rpc_processor/rpc_validate_dial_info.rs
+1 -7
View File
@@ -10,13 +10,7 @@ impl RPCProcessor {
redirect: bool,
) -> Result<bool, RPCError> {
let network_manager = self.network_manager();
let receipt_time = ms_to_us(
self.config
.get()
.network
.dht
.validate_dial_info_receipt_time_ms,
);
let receipt_time = ms_to_us(self.unlocked_inner.validate_dial_info_receipt_time_ms);
// Generate receipt and waitable eventual so we can see if we get the receipt back
let (receipt, eventual_value) = network_manager