add more error telemetry
This commit is contained in:
parent
a06f24e287
commit
5977b6a141
@ -807,17 +807,6 @@ impl NetworkManager {
|
|||||||
body: B,
|
body: B,
|
||||||
) -> EyreResult<NetworkResult<SendDataKind>> {
|
) -> EyreResult<NetworkResult<SendDataKind>> {
|
||||||
let destination_node_ref = destination_node_ref.as_ref().unwrap_or(&node_ref).clone();
|
let destination_node_ref = destination_node_ref.as_ref().unwrap_or(&node_ref).clone();
|
||||||
|
|
||||||
if !node_ref.same_entry(&destination_node_ref) {
|
|
||||||
log_net!(
|
|
||||||
"sending envelope to {:?} via {:?}",
|
|
||||||
destination_node_ref,
|
|
||||||
node_ref
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
log_net!("sending envelope to {:?}", node_ref);
|
|
||||||
}
|
|
||||||
|
|
||||||
let best_node_id = destination_node_ref.best_node_id();
|
let best_node_id = destination_node_ref.best_node_id();
|
||||||
|
|
||||||
// Get node's envelope versions and see if we can send to it
|
// Get node's envelope versions and see if we can send to it
|
||||||
@ -832,6 +821,17 @@ impl NetworkManager {
|
|||||||
// Build the envelope to send
|
// Build the envelope to send
|
||||||
let out = self.build_envelope(best_node_id, envelope_version, body)?;
|
let out = self.build_envelope(best_node_id, envelope_version, body)?;
|
||||||
|
|
||||||
|
if !node_ref.same_entry(&destination_node_ref) {
|
||||||
|
log_net!(
|
||||||
|
"sending envelope to {:?} via {:?}, len={}",
|
||||||
|
destination_node_ref,
|
||||||
|
node_ref,
|
||||||
|
out.len()
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
log_net!("sending envelope to {:?}, len={}", node_ref, out.len());
|
||||||
|
}
|
||||||
|
|
||||||
// Send the envelope via whatever means necessary
|
// Send the envelope via whatever means necessary
|
||||||
self.send_data(node_ref, out).await
|
self.send_data(node_ref, out).await
|
||||||
}
|
}
|
||||||
|
@ -31,8 +31,24 @@ impl RawUdpProtocolHandler {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Insert into assembly buffer
|
// Insert into assembly buffer
|
||||||
let Some(message) = self.assembly_buffer.insert_frame(&data[0..size], remote_addr) else {
|
let message = match self
|
||||||
continue;
|
.assembly_buffer
|
||||||
|
.insert_frame(&data[0..size], remote_addr)
|
||||||
|
{
|
||||||
|
NetworkResult::Value(Some(v)) => v,
|
||||||
|
NetworkResult::Value(None) => {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
nres => {
|
||||||
|
#[cfg(feature = "network-result-extra")]
|
||||||
|
log_network_result!(
|
||||||
|
"UDP::recv_message insert_frame failed: {:?} <= size={} remote_addr={}",
|
||||||
|
nres,
|
||||||
|
size,
|
||||||
|
remote_addr
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Check length of reassembled message (same for all protocols)
|
// Check length of reassembled message (same for all protocols)
|
||||||
|
@ -234,23 +234,39 @@ impl AssemblyBuffer {
|
|||||||
|
|
||||||
/// Receive a packet chunk and add to the message assembly
|
/// Receive a packet chunk and add to the message assembly
|
||||||
/// if a message has been completely, return it
|
/// if a message has been completely, return it
|
||||||
pub fn insert_frame(&self, frame: &[u8], remote_addr: SocketAddr) -> Option<Vec<u8>> {
|
pub fn insert_frame(
|
||||||
|
&self,
|
||||||
|
frame: &[u8],
|
||||||
|
remote_addr: SocketAddr,
|
||||||
|
) -> NetworkResult<Option<Vec<u8>>> {
|
||||||
// If we receive a zero length frame, send it
|
// If we receive a zero length frame, send it
|
||||||
if frame.len() == 0 {
|
if frame.len() == 0 {
|
||||||
return Some(frame.to_vec());
|
return NetworkResult::value(Some(frame.to_vec()));
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we receive a frame smaller than or equal to the length of the header, drop it
|
// If we receive a frame smaller than or equal to the length of the header, drop it
|
||||||
// or if this frame is larger than our max message length, then drop it
|
// or if this frame is larger than our max message length, then drop it
|
||||||
if frame.len() <= HEADER_LEN || frame.len() > MAX_LEN {
|
if frame.len() <= HEADER_LEN || frame.len() > MAX_LEN {
|
||||||
return None;
|
#[cfg(feature = "network-result-extra")]
|
||||||
|
return NetworkResult::invalid_message(format!(
|
||||||
|
"invalid header length: frame.len={}",
|
||||||
|
frame.len()
|
||||||
|
));
|
||||||
|
#[cfg(not(feature = "network-result-extra"))]
|
||||||
|
return NetworkResult::invalid_message("invalid header length");
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Decode the header
|
// --- Decode the header
|
||||||
|
|
||||||
// Drop versions we don't understand
|
// Drop versions we don't understand
|
||||||
if frame[0] != VERSION_1 {
|
if frame[0] != VERSION_1 {
|
||||||
return None;
|
#[cfg(feature = "network-result-extra")]
|
||||||
|
return NetworkResult::invalid_message(format!(
|
||||||
|
"invalid frame version: frame[0]={}",
|
||||||
|
frame[0]
|
||||||
|
));
|
||||||
|
#[cfg(not(feature = "network-result-extra"))]
|
||||||
|
return NetworkResult::invalid_message("invalid frame version");
|
||||||
}
|
}
|
||||||
// Version 1 header
|
// Version 1 header
|
||||||
let seq = SequenceType::from_be_bytes(frame[2..4].try_into().unwrap());
|
let seq = SequenceType::from_be_bytes(frame[2..4].try_into().unwrap());
|
||||||
@ -260,16 +276,30 @@ impl AssemblyBuffer {
|
|||||||
|
|
||||||
// See if we have a whole message and not a fragment
|
// See if we have a whole message and not a fragment
|
||||||
if off == 0 && len as usize == chunk.len() {
|
if off == 0 && len as usize == chunk.len() {
|
||||||
return Some(chunk.to_vec());
|
return NetworkResult::value(Some(chunk.to_vec()));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drop fragments with offsets greater than or equal to the message length
|
// Drop fragments with offsets greater than or equal to the message length
|
||||||
if off >= len {
|
if off >= len {
|
||||||
return None;
|
#[cfg(feature = "network-result-extra")]
|
||||||
|
return NetworkResult::invalid_message(format!(
|
||||||
|
"offset greater than length: off={} >= len={}",
|
||||||
|
off, len
|
||||||
|
));
|
||||||
|
#[cfg(not(feature = "network-result-extra"))]
|
||||||
|
return NetworkResult::invalid_message("offset greater than length");
|
||||||
}
|
}
|
||||||
// Drop fragments where the chunk would be applied beyond the message length
|
// Drop fragments where the chunk would be applied beyond the message length
|
||||||
if off as usize + chunk.len() > len as usize {
|
if off as usize + chunk.len() > len as usize {
|
||||||
return None;
|
#[cfg(feature = "network-result-extra")]
|
||||||
|
return NetworkResult::invalid_message(format!(
|
||||||
|
"chunk applied beyond message length: off={} + chunk.len={} > len={}",
|
||||||
|
off,
|
||||||
|
chunk.len(),
|
||||||
|
len
|
||||||
|
));
|
||||||
|
#[cfg(not(feature = "network-result-extra"))]
|
||||||
|
return NetworkResult::invalid_message("chunk applied beyond message length");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get or create the peer message assemblies
|
// Get or create the peer message assemblies
|
||||||
@ -291,19 +321,18 @@ impl AssemblyBuffer {
|
|||||||
e.remove();
|
e.remove();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
NetworkResult::value(out)
|
||||||
out
|
|
||||||
}
|
}
|
||||||
std::collections::hash_map::Entry::Vacant(v) => {
|
std::collections::hash_map::Entry::Vacant(v) => {
|
||||||
// See if we have room for one more
|
// See if we have room for one more
|
||||||
if peer_count == MAX_CONCURRENT_HOSTS {
|
if peer_count == MAX_CONCURRENT_HOSTS {
|
||||||
return None;
|
return NetworkResult::value(None);
|
||||||
}
|
}
|
||||||
// Add the peer
|
// Add the peer
|
||||||
let peer_messages = v.insert(PeerMessages::new());
|
let peer_messages = v.insert(PeerMessages::new());
|
||||||
|
|
||||||
// Insert the fragment and see what comes out
|
// Insert the fragment and see what comes out
|
||||||
peer_messages.insert_fragment(seq, off, len, chunk)
|
NetworkResult::value(peer_messages.insert_fragment(seq, off, len, chunk))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -52,7 +52,9 @@ pub async fn test_single_out_in() {
|
|||||||
// Send to input
|
// Send to input
|
||||||
let r_message = assbuf_in
|
let r_message = assbuf_in
|
||||||
.insert_frame(&frame, r_remote_addr)
|
.insert_frame(&frame, r_remote_addr)
|
||||||
.expect("should get one out");
|
.into_result()
|
||||||
|
.expect("should get a value")
|
||||||
|
.expect("should get something out");
|
||||||
|
|
||||||
// We should have gotten the same message
|
// We should have gotten the same message
|
||||||
assert_eq!(r_message, message);
|
assert_eq!(r_message, message);
|
||||||
@ -110,7 +112,10 @@ pub async fn test_one_frag_out_in() {
|
|||||||
|
|
||||||
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
||||||
// Send to input
|
// Send to input
|
||||||
let r_message = assbuf_in.insert_frame(&frame, r_remote_addr);
|
let r_message = assbuf_in
|
||||||
|
.insert_frame(&frame, r_remote_addr)
|
||||||
|
.into_result()
|
||||||
|
.expect("should get a value");
|
||||||
|
|
||||||
// We should have gotten the same message
|
// We should have gotten the same message
|
||||||
if let Some(r_message) = r_message {
|
if let Some(r_message) = r_message {
|
||||||
@ -172,7 +177,10 @@ pub async fn test_many_frags_out_in() {
|
|||||||
|
|
||||||
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
||||||
// Send to input
|
// Send to input
|
||||||
let r_message = assbuf_in.insert_frame(&frame, r_remote_addr);
|
let r_message = assbuf_in
|
||||||
|
.insert_frame(&frame, r_remote_addr)
|
||||||
|
.into_result()
|
||||||
|
.expect("should get a value");
|
||||||
|
|
||||||
// We should have gotten the same message
|
// We should have gotten the same message
|
||||||
if let Some(r_message) = r_message {
|
if let Some(r_message) = r_message {
|
||||||
@ -234,7 +242,10 @@ pub async fn test_many_frags_out_in_single_host() {
|
|||||||
|
|
||||||
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
||||||
// Send to input
|
// Send to input
|
||||||
let r_message = assbuf_in.insert_frame(&frame, r_remote_addr);
|
let r_message = assbuf_in
|
||||||
|
.insert_frame(&frame, r_remote_addr)
|
||||||
|
.into_result()
|
||||||
|
.expect("should get a value");
|
||||||
|
|
||||||
// We should have gotten the same message
|
// We should have gotten the same message
|
||||||
if let Some(r_message) = r_message {
|
if let Some(r_message) = r_message {
|
||||||
@ -309,7 +320,10 @@ pub async fn test_many_frags_with_drops() {
|
|||||||
|
|
||||||
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
||||||
// Send to input
|
// Send to input
|
||||||
let r_message = assbuf_in.insert_frame(&frame, r_remote_addr);
|
let r_message = assbuf_in
|
||||||
|
.insert_frame(&frame, r_remote_addr)
|
||||||
|
.into_result()
|
||||||
|
.expect("should get a value");
|
||||||
|
|
||||||
// We should have gotten the same message
|
// We should have gotten the same message
|
||||||
if let Some(r_message) = r_message {
|
if let Some(r_message) = r_message {
|
||||||
@ -383,7 +397,10 @@ pub async fn test_many_frags_reordered() {
|
|||||||
|
|
||||||
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
while let Ok((frame, r_remote_addr)) = net_rx.recv_async().await {
|
||||||
// Send to input
|
// Send to input
|
||||||
let r_message = assbuf_in.insert_frame(&frame, r_remote_addr);
|
let r_message = assbuf_in
|
||||||
|
.insert_frame(&frame, r_remote_addr)
|
||||||
|
.into_result()
|
||||||
|
.expect("should get a value");
|
||||||
|
|
||||||
// We should have gotten the same message
|
// We should have gotten the same message
|
||||||
if let Some(r_message) = r_message {
|
if let Some(r_message) = r_message {
|
||||||
|
Loading…
Reference in New Issue
Block a user