error cleanup

veilid-core/src/routing_table/route_spec_store/mod.rs

@@ -228,8 +228,9 @@ impl RouteSpecStore {
 
         // Ensure we have a valid network class so our peer info is useful
         if !rti.has_valid_network_class(RoutingDomain::PublicInternet) {
-            log_rtab!(debug "unable to allocate route until we have a valid PublicInternet network class");
+            apibail_try_again!(
-            apibail_try_again!();
+                "unable to allocate route until we have a valid PublicInternet network class"
+            );
         };
 
         // Get our peer info
@@ -381,8 +382,7 @@ impl RouteSpecStore {
 
         // If we couldn't find enough nodes, wait until we have more nodes in the routing table
         if nodes.len() < hop_count {
-            log_rtab!(debug "not enough nodes to construct route at this time");
+            apibail_try_again!("not enough nodes to construct route at this time");
-            apibail_try_again!();
         }
 
         // Get peer info for everything
@@ -534,8 +534,7 @@ impl RouteSpecStore {
             }
         }
         if route_nodes.is_empty() {
-            log_rtab!(debug "unable to find unique route at this time");
+            apibail_try_again!("unable to find unique route at this time");
-            apibail_try_again!();
         }
 
         drop(perm_func);
@@ -1309,8 +1308,9 @@ impl RouteSpecStore {
 
         // Ensure our network class is valid before attempting to assemble any routes
         if !rti.has_valid_network_class(RoutingDomain::PublicInternet) {
-            log_rtab!(debug "unable to assemble route until we have a valid PublicInternet network class");
+            apibail_try_again!(
-            apibail_try_again!();
+                "unable to assemble route until we have a valid PublicInternet network class"
+            );
         }
 
         // Make innermost route hop to our own node

veilid-core/src/routing_table/tasks/private_route_management.rs

@@ -213,7 +213,9 @@ impl RoutingTable {
                     DirectionSet::all(),
                     &[],
                 ) {
-                    Err(VeilidAPIError::TryAgain) => {}
+                    Err(VeilidAPIError::TryAgain { message }) => {
+                        log_rtab!(debug "Route allocation unavailable: {}", message);
+                    }
                     Err(e) => return Err(e.into()),
                     Ok(v) => {
                         newly_allocated_routes.push(v);

veilid-core/src/rpc_processor/destination.rs

@@ -164,7 +164,7 @@ impl RPCProcessor {
     pub(super) fn get_destination_respond_to(
         &self,
         dest: &Destination,
-    ) -> Result<NetworkResult<RespondTo>, RPCError> {
+    ) -> RPCNetworkResult<RespondTo> {
         let routing_table = self.routing_table();
         let rss = routing_table.route_spec_store();
 
@@ -180,21 +180,13 @@ impl RPCProcessor {
                 SafetySelection::Safe(safety_spec) => {
                     // Sent directly but with a safety route, respond to private route
                     let crypto_kind = target.best_node_id().kind;
-                    let pr_key = match rss.get_private_route_for_safety_spec(
+                    let pr_key = network_result_try!(rss
+                        .get_private_route_for_safety_spec(
                             crypto_kind,
                             safety_spec,
                             &target.node_ids(),
-                    ) {
+                        )
-                        Err(VeilidAPIError::TryAgain) => {
+                        .to_rpc_network_result()?);
-                            return Ok(NetworkResult::no_connection_other(
-                                "no private route for response at this time",
-                            ));
-                        }
-                        Err(e) => {
-                            return Err(RPCError::internal(e));
-                        }
-                        Ok(v) => v,
-                    };
 
                     // Get the assembled route for response
                     let private_route = rss
@@ -219,21 +211,9 @@ impl RPCProcessor {
 
                     let mut avoid_nodes = relay.node_ids();
                     avoid_nodes.add_all(&target.node_ids());
-                    let pr_key = match rss.get_private_route_for_safety_spec(
+                    let pr_key = network_result_try!(rss
-                        crypto_kind,
+                        .get_private_route_for_safety_spec(crypto_kind, safety_spec, &avoid_nodes,)
-                        safety_spec,
+                        .to_rpc_network_result()?);
-                        &avoid_nodes,
-                    ) {
-                        Err(VeilidAPIError::TryAgain) => {
-                            return Ok(NetworkResult::no_connection_other(
-                                "no private route for response at this time",
-                            ));
-                        }
-                        Err(e) => {
-                            return Err(RPCError::internal(e));
-                        }
-                        Ok(v) => v,
-                    };
 
                     // Get the assembled route for response
                     let private_route = rss
@@ -259,7 +239,7 @@ impl RPCProcessor {
                     SafetySelection::Unsafe(_) => {
                         // Sent to a private route with no safety route, use a stub safety route for the response
                         if !routing_table.has_valid_network_class(RoutingDomain::PublicInternet) {
-                            return Ok(NetworkResult::no_connection_other(
+                            return Ok(NetworkResult::service_unavailable(
                                 "Own node info must be valid to use private route",
                             ));
                         }
@@ -292,21 +272,13 @@ impl RPCProcessor {
                             private_route.public_key.value
                         } else {
                             // Get the private route to respond to that matches the safety route spec we sent the request with
-                            match rss.get_private_route_for_safety_spec(
+                            network_result_try!(rss
+                                .get_private_route_for_safety_spec(
                                     crypto_kind,
                                     safety_spec,
                                     &[avoid_node_id],
-                            ) {
+                                )
-                                Err(VeilidAPIError::TryAgain) => {
+                                .to_rpc_network_result()?)
-                                    return Ok(NetworkResult::no_connection_other(
-                                        "no private route for response at this time",
-                                    ));
-                                }
-                                Err(e) => {
-                                    return Err(RPCError::internal(e));
-                                }
-                                Ok(v) => v,
-                            }
                         };
 
                         // Get the assembled route for response

veilid-core/src/rpc_processor/fanout_call.rs

@@ -8,7 +8,7 @@ where
     result: Option<Result<R, RPCError>>,
 }
 
-pub type FanoutCallReturnType = Result<Option<Vec<PeerInfo>>, RPCError>;
+pub type FanoutCallReturnType = RPCNetworkResult<Vec<PeerInfo>>;
 pub type FanoutNodeInfoFilter = Arc<dyn Fn(&[TypedKey], &NodeInfo) -> bool + Send + Sync>;
 
 pub fn empty_fanout_node_info_filter() -> FanoutNodeInfoFilter {
@@ -132,7 +132,7 @@ where
 
             // Do the call for this node
             match (self.call_routine)(next_node.clone()).await {
-                Ok(Some(v)) => {
+                Ok(NetworkResult::Value(v)) => {
                     // Filter returned nodes
                     let filtered_v: Vec<PeerInfo> = v
                         .into_iter()
@@ -155,8 +155,11 @@ where
                         .register_find_node_answer(self.crypto_kind, filtered_v);
                     self.clone().add_to_fanout_queue(&new_nodes);
                 }
-                Ok(None) => {
+                #[allow(unused_variables)]
+                Ok(x) => {
                     // Call failed, node will not be considered again
+                    #[cfg(feature = "network-result-extra")]
+                    log_rpc!(debug "Fanout result {}: {:?}", &next_node, x);
                 }
                 Err(e) => {
                     // Error happened, abort everything and return the error

veilid-core/src/rpc_processor/mod.rs

@@ -469,24 +469,15 @@ impl RPCProcessor {
         let call_routine = |next_node: NodeRef| {
             let this = self.clone();
             async move {
-                match this
+                let v = network_result_try!(this
                     .clone()
                     .rpc_call_find_node(
                         Destination::direct(next_node).with_safety(safety_selection),
                         node_id,
                         vec![],
                     )
-                    .await
+                    .await?);
-                {
+                Ok(NetworkResult::value(v.answer))
-                    Ok(v) => {
-                        let v = network_result_value_or_log!(v => [ format!(": node_id={} count={} fanout={} fanout={} safety_selection={:?}", node_id, count, fanout, timeout_us, safety_selection) ] {
-                            // Any other failures, just try the next node
-                            return Ok(None);
-                        });
-                        Ok(Some(v.answer))
-                    }
-                    Err(e) => Err(e),
-                }
             }
         };
 
@@ -636,7 +627,7 @@ impl RPCProcessor {
         remote_private_route: PrivateRoute,
         reply_private_route: Option<PublicKey>,
         message_data: Vec<u8>,
-    ) -> Result<NetworkResult<RenderedOperation>, RPCError> {
+    ) -> RPCNetworkResult<RenderedOperation> {
         let routing_table = self.routing_table();
         let rss = routing_table.route_spec_store();
 
@@ -650,19 +641,8 @@ impl RPCProcessor {
         };
 
         // Compile the safety route with the private route
-        let compiled_route: CompiledRoute = match rss
+        let compiled_route: CompiledRoute = network_result_try!(rss
-            .compile_safety_route(safety_selection, remote_private_route)
+            .compile_safety_route(safety_selection, remote_private_route).to_rpc_network_result()?);
-        {
-            Err(VeilidAPIError::TryAgain) => {
-                return Ok(NetworkResult::no_connection_other(
-                    "private route could not be compiled at this time",
-                ))
-            }
-            Err(e) => {
-                return Err(RPCError::internal(e));
-            }
-            Ok(v) => v,
-        };
         let sr_is_stub = compiled_route.safety_route.is_stub();
         let sr_pubkey = compiled_route.safety_route.public_key.value;
 
@@ -723,7 +703,7 @@ impl RPCProcessor {
         &self,
         dest: Destination,
         operation: &RPCOperation,
-    ) -> Result<NetworkResult<RenderedOperation>, RPCError> {
+    ) ->RPCNetworkResult<RenderedOperation> {
         let out: NetworkResult<RenderedOperation>;
 
         // Encode message to a builder and make a message reader for it
@@ -1162,7 +1142,7 @@ impl RPCProcessor {
         dest: Destination,
         question: RPCQuestion,
         context: Option<QuestionContext>,
-    ) -> Result<NetworkResult<WaitableReply>, RPCError> {
+    ) ->RPCNetworkResult<WaitableReply> {
         // Get sender peer info if we should send that
         let spi = self.get_sender_peer_info(&dest);
 
@@ -1258,7 +1238,7 @@ impl RPCProcessor {
         &self,
         dest: Destination,
         statement: RPCStatement,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) ->RPCNetworkResult<()> {
         // Get sender peer info if we should send that
         let spi = self.get_sender_peer_info(&dest);
 
@@ -1333,7 +1313,7 @@ impl RPCProcessor {
         &self,
         request: RPCMessage,
         answer: RPCAnswer,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) ->RPCNetworkResult<()> {
         // Extract destination from respond_to
         let dest = network_result_try!(self.get_respond_to_destination(&request));
 
@@ -1459,7 +1439,7 @@ impl RPCProcessor {
     async fn process_rpc_message(
         &self,
         encoded_msg: RPCMessageEncoded,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) ->RPCNetworkResult<()> {
         let address_filter = self.network_manager.address_filter();
 
         // Decode operation appropriately based on header detail

veilid-core/src/rpc_processor/rpc_app_call.rs

@@ -11,7 +11,7 @@ impl RPCProcessor {
         self,
         dest: Destination,
         message: Vec<u8>,
-    ) -> Result<NetworkResult<Answer<Vec<u8>>>, RPCError> {
+    ) -> RPCNetworkResult<Answer<Vec<u8>>> {
         let debug_string = format!("AppCall(message(len)={}) => {}", message.len(), dest);
 
         let app_call_q = RPCOperationAppCallQ::new(message)?;
@@ -49,10 +49,7 @@ impl RPCProcessor {
     }
 
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_app_call_q(
+    pub(crate) async fn process_app_call_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         let routing_table = self.routing_table();
 

veilid-core/src/rpc_processor/rpc_app_message.rs

@@ -11,7 +11,7 @@ impl RPCProcessor {
         self,
         dest: Destination,
         message: Vec<u8>,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         let app_message = RPCOperationAppMessage::new(message)?;
         let statement = RPCStatement::new(RPCStatementDetail::AppMessage(Box::new(app_message)));
 
@@ -20,10 +20,7 @@ impl RPCProcessor {
     }
 
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_app_message(
+    pub(crate) async fn process_app_message(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         let routing_table = self.routing_table();
         let opi = routing_table.get_own_peer_info(msg.header.routing_domain());

veilid-core/src/rpc_processor/rpc_cancel_tunnel.rs

@@ -2,10 +2,7 @@ use super::*;
 
 impl RPCProcessor {
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_cancel_tunnel_q(
+    pub(crate) async fn process_cancel_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         #[cfg(feature = "unstable-tunnels")]
         {

veilid-core/src/rpc_processor/rpc_complete_tunnel.rs

@@ -2,10 +2,7 @@ use super::*;
 
 impl RPCProcessor {
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_complete_tunnel_q(
+    pub(crate) async fn process_complete_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         #[cfg(feature = "unstable-tunnels")]
         {

veilid-core/src/rpc_processor/rpc_error.rs

@@ -13,6 +13,8 @@ pub enum RPCError {
     Internal(String),
     #[error("[RPCError: Network({0})]")]
     Network(String),
+    #[error("[RPCError: TryAgain({0})]")]
+    TryAgain(String),
 }
 
 impl RPCError {
@@ -56,6 +58,25 @@ impl From<RPCError> for VeilidAPIError {
             RPCError::Protocol(message) => VeilidAPIError::Generic { message },
             RPCError::Internal(message) => VeilidAPIError::Internal { message },
             RPCError::Network(message) => VeilidAPIError::Generic { message },
+            RPCError::TryAgain(message) => VeilidAPIError::TryAgain { message },
+        }
+    }
+}
+
+pub(crate) type RPCNetworkResult<T> = Result<NetworkResult<T>, RPCError>;
+
+pub(crate) trait ToRPCNetworkResult<T> {
+    fn to_rpc_network_result(self) -> RPCNetworkResult<T>;
+}
+
+impl<T> ToRPCNetworkResult<T> for VeilidAPIResult<T> {
+    fn to_rpc_network_result(self) -> RPCNetworkResult<T> {
+        match self {
+            Err(VeilidAPIError::TryAgain { message }) => Err(RPCError::TryAgain(message)),
+            Err(VeilidAPIError::Timeout) => Ok(NetworkResult::timeout()),
+            Err(VeilidAPIError::Unimplemented { message }) => Err(RPCError::Unimplemented(message)),
+            Err(e) => Err(RPCError::internal(e)),
+            Ok(v) => Ok(NetworkResult::value(v)),
         }
     }
 }

veilid-core/src/rpc_processor/rpc_find_block.rs

@@ -2,10 +2,7 @@ use super::*;
 
 impl RPCProcessor {
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_find_block_q(
+    pub(crate) async fn process_find_block_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         #[cfg(feature = "unstable-blockstore")]
         {

veilid-core/src/rpc_processor/rpc_find_node.rs

@@ -16,7 +16,7 @@ impl RPCProcessor {
         dest: Destination,
         node_id: TypedKey,
         capabilities: Vec<Capability>,
-    ) -> Result<NetworkResult<Answer<Vec<PeerInfo>>>, RPCError> {
+    ) -> RPCNetworkResult<Answer<Vec<PeerInfo>>> {
         // Ensure destination never has a private route
         if matches!(
             dest,
@@ -78,10 +78,7 @@ impl RPCProcessor {
     }
 
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_find_node_q(
+    pub(crate) async fn process_find_node_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ensure this never came over a private route, safety route is okay though
         match &msg.header.detail {
             RPCMessageHeaderDetail::Direct(_) | RPCMessageHeaderDetail::SafetyRouted(_) => {}

veilid-core/src/rpc_processor/rpc_get_value.rs

@@ -32,7 +32,7 @@ impl RPCProcessor {
         key: TypedKey,
         subkey: ValueSubkey,
         last_descriptor: Option<SignedValueDescriptor>,
-    ) -> Result<NetworkResult<Answer<GetValueAnswer>>, RPCError> {
+    ) ->RPCNetworkResult<Answer<GetValueAnswer>> {
         // Ensure destination never has a private route
         // and get the target noderef so we can validate the response
         let Some(target) = dest.target() else {
@@ -168,7 +168,7 @@ impl RPCProcessor {
     pub(crate) async fn process_get_value_q(
         &self,
         msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) ->RPCNetworkResult<()> {
 
         // Ensure this never came over a private route, safety route is okay though
         match &msg.header.detail {

veilid-core/src/rpc_processor/rpc_return_receipt.rs

@@ -11,7 +11,7 @@ impl RPCProcessor {
         self,
         dest: Destination,
         receipt: D,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         let receipt = receipt.as_ref().to_vec();
 
         let return_receipt = RPCOperationReturnReceipt::new(receipt)?;
@@ -25,10 +25,7 @@ impl RPCProcessor {
     }
 
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_return_receipt(
+    pub(crate) async fn process_return_receipt(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Get the statement
         let (_, _, _, kind) = msg.operation.destructure();
         let receipt = match kind {

veilid-core/src/rpc_processor/rpc_route.rs

@@ -10,7 +10,7 @@ impl RPCProcessor {
         routed_operation: RoutedOperation,
         route_hop: RouteHop,
         safety_route: SafetyRoute,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         // Make sure hop count makes sense
         if safety_route.hop_count as usize > self.unlocked_inner.max_route_hop_count {
             return Ok(NetworkResult::invalid_message(
@@ -69,7 +69,7 @@ impl RPCProcessor {
         next_route_node: RouteNode,
         safety_route_public_key: TypedKey,
         next_private_route: PrivateRoute,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         // Make sure hop count makes sense
         if next_private_route.hop_count as usize > self.unlocked_inner.max_route_hop_count {
             return Ok(NetworkResult::invalid_message(
@@ -122,7 +122,7 @@ impl RPCProcessor {
         vcrypto: CryptoSystemVersion,
         routed_operation: RoutedOperation,
         remote_sr_pubkey: TypedKey,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         // Now that things are valid, decrypt the routed operation with DEC(nonce, DH(the SR's public key, the PR's (or node's) secret)
         // xxx: punish nodes that send messages that fail to decrypt eventually? How to do this for safety routes?
         let node_id_secret = self.routing_table.node_id_secret_key(remote_sr_pubkey.kind);
@@ -170,7 +170,7 @@ impl RPCProcessor {
         routed_operation: RoutedOperation,
         remote_sr_pubkey: TypedKey,
         pr_pubkey: TypedKey,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         // Get sender id of the peer with the crypto kind of the route
         let Some(sender_id) = detail.peer_noderef.node_ids().get(pr_pubkey.kind) else {
             return Ok(NetworkResult::invalid_message(
@@ -246,7 +246,7 @@ impl RPCProcessor {
         routed_operation: RoutedOperation,
         remote_sr_pubkey: TypedKey,
         pr_pubkey: TypedKey,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         // If the private route public key is our node id, then this was sent via safety route to our node directly
         // so there will be no signatures to validate
         if self.routing_table.node_ids().contains(&pr_pubkey) {
@@ -277,7 +277,7 @@ impl RPCProcessor {
         mut routed_operation: RoutedOperation,
         sr_pubkey: TypedKey,
         mut private_route: PrivateRoute,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         let Some(pr_first_hop) = private_route.pop_first_hop() else {
             return Ok(NetworkResult::invalid_message(
                 "switching from safety route to private route requires first hop",
@@ -341,7 +341,7 @@ impl RPCProcessor {
         route_hop_data: &RouteHopData,
         pr_pubkey: &TypedKey,
         route_operation: &mut RoutedOperation,
-    ) -> Result<NetworkResult<RouteHop>, RPCError> {
+    ) -> RPCNetworkResult<RouteHop> {
         // Get crypto kind
         let crypto_kind = pr_pubkey.kind;
         let Some(vcrypto) = self.crypto.get(crypto_kind) else {
@@ -402,10 +402,7 @@ impl RPCProcessor {
         feature = "verbose-tracing",
         instrument(level = "trace", skip(self), ret, err)
     )]
-    pub(crate) async fn process_route(
+    pub(crate) async fn process_route(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         let routing_table = self.routing_table();
         if !routing_table.has_valid_network_class(msg.header.routing_domain()) {

veilid-core/src/rpc_processor/rpc_set_value.rs

@@ -36,7 +36,7 @@ impl RPCProcessor {
         value: SignedValueData,
         descriptor: SignedValueDescriptor,
         send_descriptor: bool,
-    ) -> Result<NetworkResult<Answer<SetValueAnswer>>, RPCError> {
+    ) ->RPCNetworkResult<Answer<SetValueAnswer>> {
         // Ensure destination never has a private route
         // and get the target noderef so we can validate the response
         let Some(target) = dest.target() else {
@@ -182,7 +182,7 @@ impl RPCProcessor {
     pub(crate) async fn process_set_value_q(
         &self,
         msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) ->RPCNetworkResult<()> {
         // Ignore if disabled
         let routing_table = self.routing_table();
         let opi = routing_table.get_own_peer_info(msg.header.routing_domain());

veilid-core/src/rpc_processor/rpc_signal.rs

@@ -11,7 +11,7 @@ impl RPCProcessor {
         self,
         dest: Destination,
         signal_info: SignalInfo,
-    ) -> Result<NetworkResult<()>, RPCError> {
+    ) -> RPCNetworkResult<()> {
         // Ensure destination never has a private route
         if matches!(
             dest,
@@ -33,10 +33,7 @@ impl RPCProcessor {
     }
 
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_signal(
+    pub(crate) async fn process_signal(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         let routing_table = self.routing_table();
         let opi = routing_table.get_own_peer_info(msg.header.routing_domain());

veilid-core/src/rpc_processor/rpc_start_tunnel.rs

@@ -2,10 +2,7 @@ use super::*;
 
 impl RPCProcessor {
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_start_tunnel_q(
+    pub(crate) async fn process_start_tunnel_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         #[cfg(feature = "unstable-tunnels")]
         {

veilid-core/src/rpc_processor/rpc_status.rs

@@ -22,7 +22,7 @@ impl RPCProcessor {
     pub async fn rpc_call_status(
         self,
         dest: Destination,
-    ) -> Result<NetworkResult<Answer<Option<SenderInfo>>>, RPCError> {
+    ) -> RPCNetworkResult<Answer<Option<SenderInfo>>> {
         let (opt_target_nr, routing_domain, node_status) = match dest.get_safety_selection() {
             SafetySelection::Unsafe(_) => {
                 let (opt_target_nr, routing_domain) = match &dest {
@@ -197,10 +197,7 @@ impl RPCProcessor {
     }
 
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_status_q(
+    pub(crate) async fn process_status_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Get the question
         let kind = msg.operation.kind().clone();
         let status_q = match kind {

veilid-core/src/rpc_processor/rpc_supply_block.rs

@@ -2,10 +2,7 @@ use super::*;
 
 impl RPCProcessor {
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_supply_block_q(
+    pub(crate) async fn process_supply_block_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         #[cfg(feature = "unstable-blockstore")]
         {

veilid-core/src/rpc_processor/rpc_validate_dial_info.rs

@@ -56,10 +56,7 @@ impl RPCProcessor {
     }
 
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_validate_dial_info(
+    pub(crate) async fn process_validate_dial_info(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         let routing_table = self.routing_table();
         if !routing_table.has_valid_network_class(msg.header.routing_domain()) {
             return Ok(NetworkResult::service_unavailable(

veilid-core/src/rpc_processor/rpc_value_changed.rs

@@ -2,10 +2,7 @@ use super::*;
 
 impl RPCProcessor {
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), err))]
-    pub(crate) async fn process_value_changed(
+    pub(crate) async fn process_value_changed(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         let routing_table = self.routing_table();
         let opi = routing_table.get_own_peer_info(msg.header.routing_domain());

veilid-core/src/rpc_processor/rpc_watch_value.rs

@@ -2,10 +2,7 @@ use super::*;
 
 impl RPCProcessor {
     #[cfg_attr(feature="verbose-tracing", instrument(level = "trace", skip(self, msg), fields(msg.operation.op_id), ret, err))]
-    pub(crate) async fn process_watch_value_q(
+    pub(crate) async fn process_watch_value_q(&self, msg: RPCMessage) -> RPCNetworkResult<()> {
-        &self,
-        msg: RPCMessage,
-    ) -> Result<NetworkResult<()>, RPCError> {
         // Ignore if disabled
         let routing_table = self.routing_table();
         let opi = routing_table.get_own_peer_info(msg.header.routing_domain());

veilid-core/src/storage_manager/get_value.rs

@@ -54,7 +54,8 @@ impl StorageManager {
             let context = context.clone();
             let last_descriptor = last_subkey_result.descriptor.clone();
             async move {
-                let vres = rpc_processor
+                let gva = network_result_try!(
+                    rpc_processor
                         .clone()
                         .rpc_call_get_value(
                             Destination::direct(next_node.clone()).with_safety(safety_selection),
@@ -62,11 +63,8 @@ impl StorageManager {
                             subkey,
                             last_descriptor,
                         )
-                    .await?;
+                        .await?
-                let gva = network_result_value_or_log!(vres => [ format!(": next_node={} safety_selection={:?} key={} subkey={}", next_node, safety_selection, key, subkey) ] {
+                );
-                    // Any other failures, just try the next node
-                    return Ok(None);
-                });
 
                 // Keep the descriptor if we got one. If we had a last_descriptor it will
                 // already be validated by rpc_call_get_value
@@ -87,8 +85,9 @@ impl StorageManager {
                     let (Some(descriptor), Some(schema)) = (&ctx.descriptor, &ctx.schema) else {
                         // Got a value but no descriptor for it
                         // Move to the next node
-                        log_stor!(debug "Got value with no descriptor");
+                        return Ok(NetworkResult::invalid_message(
-                        return Ok(None);
+                            "Got value with no descriptor",
+                        ));
                     };
 
                     // Validate with schema
@@ -99,8 +98,10 @@ impl StorageManager {
                     ) {
                         // Validation failed, ignore this value
                         // Move to the next node
-                        log_stor!(debug "Schema validation failed on subkey {}", subkey);
+                        return Ok(NetworkResult::invalid_message(format!(
-                        return Ok(None);
+                            "Schema validation failed on subkey {}",
+                            subkey
+                        )));
                     }
 
                     // If we have a prior value, see if this is a newer sequence number
@@ -112,7 +113,7 @@ impl StorageManager {
                             // If sequence number is the same, the data should be the same
                             if prior_value.value_data() != value.value_data() {
                                 // Move to the next node
-                                return Ok(None);
+                                return Ok(NetworkResult::invalid_message("value data mismatch"));
                             }
                             // Increase the consensus count for the existing value
                             ctx.value_count += 1;
@@ -136,7 +137,7 @@ impl StorageManager {
                 #[cfg(feature = "network-result-extra")]
                 log_stor!(debug "GetValue fanout call returned peers {}", gva.answer.peers.len());
 
-                Ok(Some(gva.answer.peers))
+                Ok(NetworkResult::value(gva.answer.peers))
             }
         };
 

veilid-core/src/storage_manager/mod.rs

@@ -209,8 +209,7 @@ impl StorageManager {
 
         // Get rpc processor and drop mutex so we don't block while getting the value from the network
         let Some(rpc_processor) = inner.rpc_processor.clone() else {
-            // Offline, try again later
+            apibail_try_again!("offline, try again later");
-            apibail_try_again!();
         };
 
         // Drop the mutex so we dont block during network access
@@ -310,8 +309,7 @@ impl StorageManager {
 
         // Get rpc processor and drop mutex so we don't block while getting the value from the network
         let Some(rpc_processor) = inner.rpc_processor.clone() else {
-            // Offline, try again later
+            apibail_try_again!("offline, try again later");
-            apibail_try_again!();
         };
 
         // Drop the lock for network access

veilid-core/src/storage_manager/record_store.rs

@@ -337,7 +337,7 @@ where
             .unwrap();
         if !self.total_storage_space.check_limit() {
             self.total_storage_space.rollback();
-            apibail_try_again!();
+            apibail_try_again!("out of storage space");
         }
 
         // Save to record table
@@ -650,7 +650,7 @@ where
             .add(new_subkey_size as u64)
             .unwrap();
         if !self.total_storage_space.check_limit() {
-            apibail_try_again!();
+            apibail_try_again!("out of storage space");
         }
 
         // Write subkey

veilid-core/src/storage_manager/set_value.rs

@@ -60,7 +60,8 @@ impl StorageManager {
                 };
 
                 // send across the wire
-                let vres = rpc_processor
+                let sva = network_result_try!(
+                    rpc_processor
                         .clone()
                         .rpc_call_set_value(
                             Destination::direct(next_node.clone()).with_safety(safety_selection),
@@ -70,11 +71,8 @@ impl StorageManager {
                             descriptor.clone(),
                             send_descriptor,
                         )
-                    .await?;
+                        .await?
-                let sva = network_result_value_or_log!(vres => [ format!(": next_node={} safety_selection={:?} key={} subkey={} send_descriptor={}", next_node, safety_selection, key, subkey, send_descriptor) ] {
+                );
-                    // Any other failures, just try the next node and pretend this one never happened
-                    return Ok(None);
-                });
 
                 // If the node was close enough to possibly set the value
                 if sva.answer.set {
@@ -91,7 +89,7 @@ impl StorageManager {
                             value.value_data(),
                         ) {
                             // Validation failed, ignore this value and pretend we never saw this node
-                            return Ok(None);
+                            return Ok(NetworkResult::invalid_message("Schema validation failed"));
                         }
 
                         // We have a prior value, ensure this is a newer sequence number
@@ -107,7 +105,7 @@ impl StorageManager {
                             // If the sequence number is older, or an equal sequence number,
                             // node should have not returned a value here.
                             // Skip this node and its closer list because it is misbehaving
-                            return Ok(None);
+                            return Ok(NetworkResult::invalid_message("Sequence number is older"));
                         }
                     } else {
                         // It was set on this node and no newer value was found and returned,
@@ -124,7 +122,7 @@ impl StorageManager {
                 #[cfg(feature = "network-result-extra")]
                 log_stor!(debug "SetValue fanout call returned peers {}", sva.answer.peers.len());
 
-                Ok(Some(sva.answer.peers))
+                Ok(NetworkResult::value(sva.answer.peers))
             }
         };
 

veilid-core/src/veilid_api/api.rs

@@ -200,9 +200,11 @@ impl VeilidAPI {
     /// `VLD0:XmnGyJrjMJBRC5ayJZRPXWTBspdX36-pbLb98H3UMeE` but if the prefix is left off
     /// `XmnGyJrjMJBRC5ayJZRPXWTBspdX36-pbLb98H3UMeE` will be parsed with the 'best' cryptosystem
     /// available (at the time of this writing this is `VLD0`)
-    pub async fn parse_as_target<S: AsRef<str>>(&self, s: S) -> VeilidAPIResult<Target> {
+    pub async fn parse_as_target<S: ToString>(&self, s: S) -> VeilidAPIResult<Target> {
+        let s = s.to_string();
+
         // Is this a route id?
-        if let Ok(rrid) = RouteId::from_str(s.as_ref()) {
+        if let Ok(rrid) = RouteId::from_str(&s) {
             let routing_table = self.routing_table()?;
             let rss = routing_table.route_spec_store();
 
@@ -213,11 +215,11 @@ impl VeilidAPI {
         }
 
         // Is this a node id?
-        if let Ok(nid) = TypedKey::from_str(s.as_ref()) {
+        if let Ok(nid) = TypedKey::from_str(&s) {
             return Ok(Target::NodeId(nid));
         }
 
-        Err(VeilidAPIError::invalid_target())
+        Err(VeilidAPIError::parse_error("Unable to parse as target", s))
     }
 
     ////////////////////////////////////////////////////////////////

veilid-core/src/veilid_api/error.rs

@@ -19,8 +19,8 @@ macro_rules! apibail_timeout {
 #[allow(unused_macros)]
 #[macro_export]
 macro_rules! apibail_try_again {
-    () => {
+    ($x:expr) => {
-        return Err(VeilidAPIError::try_again())
+        return Err(VeilidAPIError::try_again($x))
     };
 }
 
@@ -83,8 +83,8 @@ macro_rules! apibail_key_not_found {
 #[allow(unused_macros)]
 #[macro_export]
 macro_rules! apibail_invalid_target {
-    () => {
+    ($x:expr) => {
-        return Err(VeilidAPIError::invalid_target())
+        return Err(VeilidAPIError::invalid_target($x))
     };
 }
 
@@ -116,12 +116,12 @@ pub enum VeilidAPIError {
     AlreadyInitialized,
     #[error("Timeout")]
     Timeout,
-    #[error("TryAgain")]
+    #[error("TryAgain: {message}")]
-    TryAgain,
+    TryAgain { message: String },
     #[error("Shutdown")]
     Shutdown,
-    #[error("Invalid target")]
+    #[error("Invalid target: {message}")]
-    InvalidTarget,
+    InvalidTarget { message: String },
     #[error("No connection: {message}")]
     NoConnection { message: String },
     #[error("Key not found: {key}")]
@@ -158,14 +158,18 @@ impl VeilidAPIError {
     pub fn timeout() -> Self {
         Self::Timeout
     }
-    pub fn try_again() -> Self {
+    pub fn try_again<T: ToString>(msg: T) -> Self {
-        Self::TryAgain
+        Self::TryAgain {
+            message: msg.to_string(),
+        }
     }
     pub fn shutdown() -> Self {
         Self::Shutdown
     }
-    pub fn invalid_target() -> Self {
+    pub fn invalid_target<T: ToString>(msg: T) -> Self {
-        Self::InvalidTarget
+        Self::InvalidTarget {
+            message: msg.to_string(),
+        }
     }
     pub fn no_connection<T: ToString>(msg: T) -> Self {
         Self::NoConnection {
@@ -213,6 +217,21 @@ impl VeilidAPIError {
             message: msg.to_string(),
         }
     }
+
+    pub(crate) fn from_network_result<T>(nr: NetworkResult<T>) -> Result<T, Self> {
+        match nr {
+            NetworkResult::Timeout => Err(VeilidAPIError::timeout()),
+            NetworkResult::ServiceUnavailable(m) => Err(VeilidAPIError::invalid_target(m)),
+            NetworkResult::NoConnection(m) => Err(VeilidAPIError::no_connection(m)),
+            NetworkResult::AlreadyExists(m) => {
+                Err(VeilidAPIError::generic(format!("Already exists: {}", m)))
+            }
+            NetworkResult::InvalidMessage(m) => {
+                Err(VeilidAPIError::parse_error("Invalid message", m))
+            }
+            NetworkResult::Value(v) => Ok(v),
+        }
+    }
 }
 
 pub type VeilidAPIResult<T> = Result<T, VeilidAPIError>;

veilid-core/src/veilid_api/json_api/process.rs

@@ -220,7 +220,7 @@ impl JsonRequestProcessor {
             return Ok(Target::NodeId(nid));
         }
 
-        Err(VeilidAPIError::invalid_target())
+        Err(VeilidAPIError::parse_error("Unable to parse as target", s))
     }
 
     //////////////////////////////////////////////////////////////////////////////////////

veilid-core/src/veilid_api/routing_context.rs

@@ -126,7 +126,7 @@ impl RoutingContext {
                     .await
                 {
                     Ok(Some(nr)) => nr,
-                    Ok(None) => apibail_invalid_target!(),
+                    Ok(None) => apibail_invalid_target!("could not resolve node id"),
                     Err(e) => return Err(e.into()),
                 };
                 // Apply sequencing to match safety selection
@@ -142,7 +142,7 @@ impl RoutingContext {
                 let rss = self.api.routing_table()?.route_spec_store();
 
                 let Some(private_route) = rss.best_remote_private_route(&rsid) else {
-                    apibail_invalid_target!();
+                    apibail_invalid_target!("could not get remote private route");
                 };
 
                 Ok(rpc_processor::Destination::PrivateRoute {
@@ -174,10 +174,7 @@ impl RoutingContext {
         let answer = match rpc_processor.rpc_call_app_call(dest, message).await {
             Ok(NetworkResult::Value(v)) => v,
             Ok(NetworkResult::Timeout) => apibail_timeout!(),
-            Ok(NetworkResult::ServiceUnavailable(e)) => {
+            Ok(NetworkResult::ServiceUnavailable(e)) => apibail_invalid_target!(e),
-                log_network_result!(format!("app_call: ServiceUnavailable({})", e));
-                apibail_try_again!()
-            }
             Ok(NetworkResult::NoConnection(e)) | Ok(NetworkResult::AlreadyExists(e)) => {
                 apibail_no_connection!(e);
             }
@@ -207,10 +204,7 @@ impl RoutingContext {
         match rpc_processor.rpc_call_app_message(dest, message).await {
             Ok(NetworkResult::Value(())) => {}
             Ok(NetworkResult::Timeout) => apibail_timeout!(),
-            Ok(NetworkResult::ServiceUnavailable(e)) => {
+            Ok(NetworkResult::ServiceUnavailable(e)) => apibail_invalid_target!(e),
-                log_network_result!(format!("app_message: ServiceUnavailable({})", e));
-                apibail_try_again!()
-            }
             Ok(NetworkResult::NoConnection(e)) | Ok(NetworkResult::AlreadyExists(e)) => {
                 apibail_no_connection!(e);
             }

veilid-flutter/lib/veilid_api_exception.dart

@@ -22,7 +22,7 @@ abstract class VeilidAPIException implements Exception {
         }
       case 'TryAgain':
         {
-          return VeilidAPIExceptionTryAgain();
+          return VeilidAPIExceptionTryAgain(json['message']! as String);
         }
       case 'Shutdown':
         {
@@ -30,7 +30,7 @@ abstract class VeilidAPIException implements Exception {
         }
       case 'InvalidTarget':
         {
-          return VeilidAPIExceptionInvalidTarget();
+          return VeilidAPIExceptionInvalidTarget(json['message']! as String);
         }
       case 'NoConnection':
         {
@@ -108,11 +108,14 @@ class VeilidAPIExceptionTimeout implements VeilidAPIException {
 
 @immutable
 class VeilidAPIExceptionTryAgain implements VeilidAPIException {
+  //
+  const VeilidAPIExceptionTryAgain(this.message);
+  final String message;
   @override
-  String toString() => 'VeilidAPIException: TryAgain';
+  String toString() => 'VeilidAPIException: TryAgain (message: $message)';
 
   @override
-  String toDisplayError() => 'Try again';
+  String toDisplayError() => 'Try again: (message: $message)';
 }
 
 @immutable
@@ -126,11 +129,15 @@ class VeilidAPIExceptionShutdown implements VeilidAPIException {
 
 @immutable
 class VeilidAPIExceptionInvalidTarget implements VeilidAPIException {
-  @override
+  //
-  String toString() => 'VeilidAPIException: InvalidTarget';
+  const VeilidAPIExceptionInvalidTarget(this.message);
+  final String message;
 
   @override
-  String toDisplayError() => 'Invalid target';
+  String toString() => 'VeilidAPIException: InvalidTarget (message: $message)';
+
+  @override
+  String toDisplayError() => 'Invalid target: (message: $message)';
 }
 
 @immutable

veilid-python/veilid/error.py

@@ -55,6 +55,7 @@ class VeilidAPIErrorTryAgain(VeilidAPIError):
     """Operation could not be performed at this time, retry again later"""
 
     label = "Try again"
+    message: str
 
 
 @dataclass
@@ -69,6 +70,7 @@ class VeilidAPIErrorInvalidTarget(VeilidAPIError):
     """Target of operation is not valid"""
 
     label = "Invalid target"
+    message: str
 
 
 @dataclass

veilid-tools/src/network_result.rs

@@ -218,7 +218,7 @@ impl<T> NetworkResult<T> {
             Self::Value(v) => NetworkResult::<X>::Value(f(v)),
         }
     }
-    pub fn into_result(self) -> Result<T, io::Error> {
+    pub fn into_io_result(self) -> Result<T, io::Error> {
         match self {
             Self::Timeout => Err(io::Error::new(io::ErrorKind::TimedOut, "Timed out")),
             Self::ServiceUnavailable(s) => Err(io::Error::new(

veilid-tools/src/tests/native/test_assembly_buffer.rs

@@ -52,7 +52,7 @@ pub async fn test_single_out_in() {
         // Send to input
         let r_message = assbuf_in
             .insert_frame(&frame, r_remote_addr)
-            .into_result()
+            .into_io_result()
             .expect("should get a value")
             .expect("should get something out");
 
@@ -114,7 +114,7 @@ pub async fn test_one_frag_out_in() {
         // Send to input
         let r_message = assbuf_in
             .insert_frame(&frame, r_remote_addr)
-            .into_result()
+            .into_io_result()
             .expect("should get a value");
 
         // We should have gotten the same message
@@ -179,7 +179,7 @@ pub async fn test_many_frags_out_in() {
         // Send to input
         let r_message = assbuf_in
             .insert_frame(&frame, r_remote_addr)
-            .into_result()
+            .into_io_result()
             .expect("should get a value");
 
         // We should have gotten the same message
@@ -244,7 +244,7 @@ pub async fn test_many_frags_out_in_single_host() {
         // Send to input
         let r_message = assbuf_in
             .insert_frame(&frame, r_remote_addr)
-            .into_result()
+            .into_io_result()
             .expect("should get a value");
 
         // We should have gotten the same message
@@ -322,7 +322,7 @@ pub async fn test_many_frags_with_drops() {
         // Send to input
         let r_message = assbuf_in
             .insert_frame(&frame, r_remote_addr)
-            .into_result()
+            .into_io_result()
             .expect("should get a value");
 
         // We should have gotten the same message
@@ -399,7 +399,7 @@ pub async fn test_many_frags_reordered() {
         // Send to input
         let r_message = assbuf_in
             .insert_frame(&frame, r_remote_addr)
-            .into_result()
+            .into_io_result()
             .expect("should get a value");
 
         // We should have gotten the same message