From b6e568f6643cb102a18b5c229fab3e8136a279b0 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 28 May 2022 10:07:57 -0400 Subject: [PATCH] receipt rework and discovery rework --- package/linux/veilid-server.conf | 13 +- package/systemd/veilid-server.service | 8 +- veilid-core/proto/veilid.capnp | 20 +- veilid-core/src/dht/envelope.rs | 2 +- veilid-core/src/dht/receipt.rs | 2 +- veilid-core/src/intf/native/network/mod.rs | 8 +- .../native/network/network_class_discovery.rs | 261 +++++++++--------- .../src/intf/wasm/network/protocol/ws.rs | 4 +- veilid-core/src/network_manager.rs | 164 ++++++----- veilid-core/src/receipt_manager.rs | 96 ++++--- veilid-core/src/routing_table/bucket_entry.rs | 4 + veilid-core/src/routing_table/debug.rs | 43 ++- veilid-core/src/routing_table/find_nodes.rs | 2 + veilid-core/src/routing_table/mod.rs | 166 ++++++++--- veilid-core/src/routing_table/node_ref.rs | 8 + .../src/rpc_processor/coders/node_info.rs | 8 + .../src/rpc_processor/coders/signal_info.rs | 58 ++-- veilid-core/src/rpc_processor/mod.rs | 112 ++++---- veilid-core/src/veilid_api/mod.rs | 98 ++++++- veilid-core/src/xx/tools.rs | 8 +- veilid-server/src/server.rs | 12 + veilid-server/src/settings.rs | 129 ++++++--- veilid-server/src/unix.rs | 22 +- 23 files changed, 817 insertions(+), 431 deletions(-) diff --git a/package/linux/veilid-server.conf b/package/linux/veilid-server.conf index b47692c3..cd94b5d7 100644 --- a/package/linux/veilid-server.conf +++ b/package/linux/veilid-server.conf @@ -6,15 +6,16 @@ # ----------------------------------------------------------- --- -daemon: - enabled: true - pid_file: '/run/veilid-server.pid' - working_directory: '/' - user: veilid - group: veilid logging: system: enabled: true level: info terminal: enabled: false +core: + protected_store: + insecure_fallback_directory: '/var/db/veilid-server/protected_store' + table_store: + directory: '/var/db/veilid-server/table_store' + block_store: + directory: '/var/db/veilid-server/block_store' \ No newline at end of file diff --git a/package/systemd/veilid-server.service b/package/systemd/veilid-server.service index ea9e5eeb..04897fdf 100644 --- a/package/systemd/veilid-server.service +++ b/package/systemd/veilid-server.service @@ -6,15 +6,15 @@ Requires=network-online.target After=network-online.target [Service] -Type=forking -PIDFile=/run/veilid-server.pid -ExecStartPre=/usr/bin/rm -f /run/veilid-server.pid +Type=simple ExecStart=/usr/bin/veilid-server ExecReload=/bin/kill -s HUP $MAINPID KillSignal=SIGQUIT TimeoutStopSec=5 -KillMode=mixed PrivateTmp=true +WorkingDirectory=/ +User=veilid +Group=veilid [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 9f42e5da..dd93c3b4 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -111,12 +111,12 @@ struct NodeDialInfo { ############################## struct SignalInfoHolePunch { - receipt @0 :Data; # receipt to return with hole punch + receiptNonce @0 :Nonce; # receipt to return with hole punch peerInfo @1 :PeerInfo; # peer info of the signal sender for hole punch attempt } struct SignalInfoReverseConnect { - receipt @0 :Data; # receipt to return with reverse connect + receiptNonce @0 :Nonce; # receipt to return with reverse connect peerInfo @1 :PeerInfo; # peer info of the signal sender for reverse connect attempt } @@ -217,8 +217,10 @@ struct ProtocolSet { struct NodeInfo { networkClass @0 :NetworkClass; # network class of this node outboundProtocols @1 :ProtocolSet; # protocols that can go outbound - dialInfoDetailList @2 :List(DialInfoDetail); # inbound dial info details for this node - relayPeerInfo @3 :PeerInfo; # (optional) relay peer info for this node + minVersion @2 :UInt8; # minimum protocol version for rpc + maxVersion @3 :UInt8; # maximum protocol version for rpc + dialInfoDetailList @4 :List(DialInfoDetail); # inbound dial info details for this node + relayPeerInfo @5 :PeerInfo; # (optional) relay peer info for this node } struct SignedNodeInfo { @@ -238,13 +240,15 @@ struct OperationStatusA { struct OperationValidateDialInfo { dialInfo @0 :DialInfo; # dial info to use for the receipt - receipt @1 :Data; # receipt to return to dial info to prove it is reachable - redirect @2 :Bool; # request a different node do the validate - alternatePort @3 :Bool; # return receipt from a different source port than the default + minVersion @1 :UInt8; # minimum version for the direct receipt + maxVersion @2 :UInt8; # maximum version for the direct receipt + receiptNonce @3 :Nonce; # receipt to return to dial info to prove it is reachable + redirect @4 :Bool; # request a different node do the validate } struct OperationReturnReceipt { - receipt @0 :Data; # receipt being returned to its origin + receiptNonce @0 :Nonce; # receipt being returned to its origin + extraData @1 :Data; # extra data for receipt } struct OperationFindNodeQ { diff --git a/veilid-core/src/dht/envelope.rs b/veilid-core/src/dht/envelope.rs index 3b6ae51d..7d70a203 100644 --- a/veilid-core/src/dht/envelope.rs +++ b/veilid-core/src/dht/envelope.rs @@ -78,8 +78,8 @@ impl Envelope { pub fn from_signed_data(data: &[u8]) -> Result { // Ensure we are at least the length of the envelope + // Silent drop here, as we use zero length packets as part of the protocol for hole punching if data.len() < MIN_ENVELOPE_SIZE { - trace!("envelope too small: len={}", data.len()); return Err(()); } diff --git a/veilid-core/src/dht/receipt.rs b/veilid-core/src/dht/receipt.rs index e11d2d15..0a6a8554 100644 --- a/veilid-core/src/dht/receipt.rs +++ b/veilid-core/src/dht/receipt.rs @@ -22,7 +22,7 @@ use data_encoding::BASE64URL_NOPAD; // reserved: u8, // 0x05: Reserved for future use // size: u16, // 0x06: Total size of the receipt including the extra data and the signature. Maximum size is 1152 bytes. // nonce: [u8; 24], // 0x08: Randomly chosen bytes that represent a unique receipt. Could be used to encrypt the extra data, but it's not required. -// sender_id: [u8; 32], // 0x20: Node ID of the message source, which is the Ed25519 public key of the sender (must be verified with find_node if this is a new node_id/address combination) +// sender_id: [u8; 32], // 0x20: Node ID of the message source, which is the Ed25519 public key of the sender // extra_data: [u8; ??], // 0x40: Extra data is appended (arbitrary extra data, not encrypted by receipt itself, maximum size is 1024 bytes) // signature: [u8; 64], // 0x?? (end-0x40): Ed25519 signature of the entire receipt including header and extra data is appended to the packet // } diff --git a/veilid-core/src/intf/native/network/mod.rs b/veilid-core/src/intf/native/network/mod.rs index 31ec0587..641a3667 100644 --- a/veilid-core/src/intf/native/network/mod.rs +++ b/veilid-core/src/intf/native/network/mod.rs @@ -534,10 +534,16 @@ impl Network { pub async fn tick(&self) -> Result<(), String> { let network_class = self.get_network_class().unwrap_or(NetworkClass::Invalid); + let routing_table = self.routing_table(); // If we need to figure out our network class, tick the task for it if network_class == NetworkClass::Invalid { - self.unlocked_inner.update_network_class_task.tick().await?; + let rth = routing_table.get_routing_table_health(); + + // Need at least two entries to do this + if rth.unreliable_entry_count + rth.reliable_entry_count >= 2 { + self.unlocked_inner.update_network_class_task.tick().await?; + } } Ok(()) diff --git a/veilid-core/src/intf/native/network/network_class_discovery.rs b/veilid-core/src/intf/native/network/network_class_discovery.rs index fbfe70ee..ab4853a7 100644 --- a/veilid-core/src/intf/native/network/network_class_discovery.rs +++ b/veilid-core/src/intf/native/network/network_class_discovery.rs @@ -16,9 +16,10 @@ struct DiscoveryContextInner { intf_addrs: Option>, protocol_type: Option, address_type: Option, - external1_dial_info: Option, - external1: Option, - node_b: Option, + // first node contacted + external_1_dial_info: Option, + external_1_address: Option, + node_1: Option, // detected public dialinfo detected_network_class: Option, detected_public_dial_info: Option, @@ -40,9 +41,9 @@ impl DiscoveryContext { intf_addrs: None, protocol_type: None, address_type: None, - external1_dial_info: None, - external1: None, - node_b: None, + external_1_dial_info: None, + external_1_address: None, + node_1: None, detected_network_class: None, detected_public_dial_info: None, })), @@ -64,6 +65,7 @@ impl DiscoveryContext { } // Ask for a public address check from a particular noderef + // This is done over the normal port using RPC async fn request_public_address(&self, node_ref: NodeRef) -> Option { let rpc = self.routing_table.rpc_processor(); rpc.rpc_call_status(node_ref.clone()) @@ -81,6 +83,7 @@ impl DiscoveryContext { } // find fast peers with a particular address type, and ask them to tell us what our external address is + // This is done over the normal port using RPC async fn discover_external_address( &self, protocol_type: ProtocolType, @@ -109,6 +112,7 @@ impl DiscoveryContext { None } + // This pulls the already-detected local interface dial info from the routing table fn get_local_addresses( &self, protocol_type: ProtocolType, @@ -135,10 +139,9 @@ impl DiscoveryContext { node_ref: NodeRef, dial_info: DialInfo, redirect: bool, - alternate_port: bool, ) -> bool { let rpc = self.routing_table.rpc_processor(); - rpc.rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect, alternate_port) + rpc.rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect) .await .map_err(logthru_net!( "failed to send validate_dial_info to {:?}", @@ -179,19 +182,20 @@ impl DiscoveryContext { inner.intf_addrs = Some(intf_addrs); inner.protocol_type = Some(protocol_type); inner.address_type = Some(address_type); - inner.external1_dial_info = None; - inner.external1 = None; - inner.node_b = None; + inner.external_1_dial_info = None; + inner.external_1_address = None; + inner.node_1 = None; } + // Get our first node's view of our external IP address via normal RPC pub async fn protocol_get_external_address_1(&self) -> bool { let (protocol_type, address_type) = { let inner = self.inner.lock(); (inner.protocol_type.unwrap(), inner.address_type.unwrap()) }; - // Get our external address from some fast node, call it node B - let (external1, node_b) = match self + // Get our external address from some fast node, call it node 1 + let (external_1, node_1) = match self .discover_external_address(protocol_type, address_type, None) .await { @@ -202,54 +206,56 @@ impl DiscoveryContext { } Some(v) => v, }; - let external1_dial_info = self.make_dial_info(external1, protocol_type); + let external_1_dial_info = self.make_dial_info(external_1, protocol_type); let mut inner = self.inner.lock(); - inner.external1_dial_info = Some(external1_dial_info); - inner.external1 = Some(external1); - inner.node_b = Some(node_b); + inner.external_1_dial_info = Some(external_1_dial_info); + inner.external_1_address = Some(external_1); + inner.node_1 = Some(node_1); - log_net!(debug "external1_dial_info: {:?}\nexternal1: {:?}\nnode_b: {:?}", inner.external1_dial_info, inner.external1, inner.node_b); + log_net!(debug "external_1_dial_info: {:?}\nexternal_1_address: {:?}\nnode_1: {:?}", inner.external_1_dial_info, inner.external_1_address, inner.node_1); true } + // If we know we are not behind NAT, check our firewall status pub async fn protocol_process_no_nat(&self) -> Result<(), String> { - let (node_b, external1_dial_info) = { + let (node_b, external_1_dial_info) = { let inner = self.inner.lock(); ( - inner.node_b.as_ref().unwrap().clone(), - inner.external1_dial_info.as_ref().unwrap().clone(), + inner.node_1.as_ref().unwrap().clone(), + inner.external_1_dial_info.as_ref().unwrap().clone(), ) }; // Do a validate_dial_info on the external address from a redirected node if self - .validate_dial_info(node_b.clone(), external1_dial_info.clone(), true, false) + .validate_dial_info(node_b.clone(), external_1_dial_info.clone(), true) .await { // Add public dial info with Direct dialinfo class - self.set_detected_public_dial_info(external1_dial_info, DialInfoClass::Direct); + self.set_detected_public_dial_info(external_1_dial_info, DialInfoClass::Direct); } - // Attempt a UDP port mapping via all available and enabled mechanisms + // Attempt a port mapping via all available and enabled mechanisms else if let Some(external_mapped_dial_info) = self.try_port_mapping().await { // Got a port mapping, let's use it self.set_detected_public_dial_info(external_mapped_dial_info, DialInfoClass::Mapped); } else { // Add public dial info with Blocked dialinfo class - self.set_detected_public_dial_info(external1_dial_info, DialInfoClass::Blocked); + self.set_detected_public_dial_info(external_1_dial_info, DialInfoClass::Blocked); } self.set_detected_network_class(NetworkClass::InboundCapable); Ok(()) } + // If we know we are behind NAT check what kind pub async fn protocol_process_nat(&self) -> Result { - let (node_b, external1_dial_info, external1, protocol_type, address_type) = { + let (node_1, external_1_dial_info, external_1_address, protocol_type, address_type) = { let inner = self.inner.lock(); ( - inner.node_b.as_ref().unwrap().clone(), - inner.external1_dial_info.as_ref().unwrap().clone(), - inner.external1.unwrap(), + inner.node_1.as_ref().unwrap().clone(), + inner.external_1_dial_info.as_ref().unwrap().clone(), + inner.external_1_address.unwrap(), inner.protocol_type.unwrap(), inner.address_type.unwrap(), ) @@ -267,14 +273,14 @@ impl DiscoveryContext { // Port mapping was not possible, let's see what kind of NAT we have - // Does a redirected dial info validation find us? + // Does a redirected dial info validation from a different address and a random port find us? if self - .validate_dial_info(node_b.clone(), external1_dial_info.clone(), true, false) + .validate_dial_info(node_1.clone(), external_1_dial_info.clone(), true) .await { // Yes, another machine can use the dial info directly, so Full Cone // Add public dial info with full cone NAT network class - self.set_detected_public_dial_info(external1_dial_info, DialInfoClass::FullConeNAT); + self.set_detected_public_dial_info(external_1_dial_info, DialInfoClass::FullConeNAT); self.set_detected_network_class(NetworkClass::InboundCapable); // No more retries @@ -283,9 +289,9 @@ impl DiscoveryContext { // No, we are restricted, determine what kind of restriction - // Get our external address from some fast node, that is not node B, call it node D - let (external2, node_d) = match self - .discover_external_address(protocol_type, address_type, Some(node_b.node_id())) + // Get our external address from some fast node, that is not node 1, call it node 2 + let (external_2_address, node_2) = match self + .discover_external_address(protocol_type, address_type, Some(node_1.node_id())) .await { None => { @@ -296,7 +302,7 @@ impl DiscoveryContext { }; // If we have two different external addresses, then this is a symmetric NAT - if external2 != external1 { + if external_2_address != external_1_address { // Symmetric NAT is outbound only, no public dial info will work self.set_detected_network_class(NetworkClass::OutboundOnly); @@ -305,23 +311,22 @@ impl DiscoveryContext { } // If we're going to end up as a restricted NAT of some sort - // Address is the same, so it's address or port restricted - let external2_dial_info = DialInfo::udp(external2); - // Do a validate_dial_info on the external address from a routed node + + // Do a validate_dial_info on the external address from a random port if self - .validate_dial_info(node_d.clone(), external2_dial_info.clone(), false, true) + .validate_dial_info(node_2.clone(), external_1_dial_info.clone(), false) .await { // Got a reply from a non-default port, which means we're only address restricted self.set_detected_public_dial_info( - external1_dial_info, + external_1_dial_info, DialInfoClass::AddressRestrictedNAT, ); } else { // Didn't get a reply from a non-default port, which means we are also port restricted self.set_detected_public_dial_info( - external1_dial_info, + external_1_dial_info, DialInfoClass::PortRestrictedNAT, ); } @@ -348,13 +353,13 @@ impl Network { // Loop for restricted NAT retries loop { - // Get our external address from some fast node, call it node B + // Get our external address from some fast node, call it node 1 if !context.protocol_get_external_address_1().await { // If we couldn't get an external address, then we should just try the whole network class detection again later return Ok(()); } - // If our local interface list contains external1 then there is no NAT in place + // If our local interface list contains external_1 then there is no NAT in place { let res = { let inner = context.inner.lock(); @@ -362,7 +367,7 @@ impl Network { .intf_addrs .as_ref() .unwrap() - .contains(inner.external1.as_ref().unwrap()) + .contains(inner.external_1_address.as_ref().unwrap()) }; if res { // No NAT @@ -397,25 +402,25 @@ impl Network { // Start doing ipv6 protocol context.protocol_begin(protocol_type, AddressType::IPV6); - // Get our external address from some fast node, call it node B + // Get our external address from some fast node, call it node 1 if !context.protocol_get_external_address_1().await { // If we couldn't get an external address, then we should just try the whole network class detection again later return Ok(()); } - // If our local interface list doesn't contain external1 then there is an Ipv6 NAT in place + // If our local interface list doesn't contain external_1 then there is an Ipv6 NAT in place { let inner = context.inner.lock(); if !inner .intf_addrs .as_ref() .unwrap() - .contains(inner.external1.as_ref().unwrap()) + .contains(inner.external_1_address.as_ref().unwrap()) { // IPv6 NAT is not supported today log_net!(warn "IPv6 NAT is not supported for external address: {}", - inner.external1.unwrap() + inner.external_1_address.unwrap() ); return Ok(()); } @@ -454,90 +459,90 @@ impl Network { .boxed(), ); - // UDPv6 - unord.push( - async { - let udpv6_context = DiscoveryContext::new(self.routing_table(), self.clone()); - if let Err(e) = self - .update_ipv6_protocol_dialinfo(&udpv6_context, ProtocolType::UDP) - .await - { - log_net!(debug "Failed UDPv6 dialinfo discovery: {}", e); - return None; - } - Some(udpv6_context) - } - .boxed(), - ); + // // UDPv6 + // unord.push( + // async { + // let udpv6_context = DiscoveryContext::new(self.routing_table(), self.clone()); + // if let Err(e) = self + // .update_ipv6_protocol_dialinfo(&udpv6_context, ProtocolType::UDP) + // .await + // { + // log_net!(debug "Failed UDPv6 dialinfo discovery: {}", e); + // return None; + // } + // Some(udpv6_context) + // } + // .boxed(), + // ); } - if protocol_config.inbound.contains(ProtocolType::TCP) { - // TCPv4 - unord.push( - async { - let tcpv4_context = DiscoveryContext::new(self.routing_table(), self.clone()); - if let Err(e) = self - .update_ipv4_protocol_dialinfo(&tcpv4_context, ProtocolType::TCP) - .await - { - log_net!(debug "Failed TCPv4 dialinfo discovery: {}", e); - return None; - } - Some(tcpv4_context) - } - .boxed(), - ); + // if protocol_config.inbound.contains(ProtocolType::TCP) { + // // TCPv4 + // unord.push( + // async { + // let tcpv4_context = DiscoveryContext::new(self.routing_table(), self.clone()); + // if let Err(e) = self + // .update_ipv4_protocol_dialinfo(&tcpv4_context, ProtocolType::TCP) + // .await + // { + // log_net!(debug "Failed TCPv4 dialinfo discovery: {}", e); + // return None; + // } + // Some(tcpv4_context) + // } + // .boxed(), + // ); - // TCPv6 - unord.push( - async { - let tcpv6_context = DiscoveryContext::new(self.routing_table(), self.clone()); - if let Err(e) = self - .update_ipv6_protocol_dialinfo(&tcpv6_context, ProtocolType::TCP) - .await - { - log_net!(debug "Failed TCPv6 dialinfo discovery: {}", e); - return None; - } - Some(tcpv6_context) - } - .boxed(), - ); - } + // // TCPv6 + // unord.push( + // async { + // let tcpv6_context = DiscoveryContext::new(self.routing_table(), self.clone()); + // if let Err(e) = self + // .update_ipv6_protocol_dialinfo(&tcpv6_context, ProtocolType::TCP) + // .await + // { + // log_net!(debug "Failed TCPv6 dialinfo discovery: {}", e); + // return None; + // } + // Some(tcpv6_context) + // } + // .boxed(), + // ); + // } - if protocol_config.inbound.contains(ProtocolType::WS) { - // WS4 - unord.push( - async { - let wsv4_context = DiscoveryContext::new(self.routing_table(), self.clone()); - if let Err(e) = self - .update_ipv4_protocol_dialinfo(&wsv4_context, ProtocolType::WS) - .await - { - log_net!(debug "Failed WSv4 dialinfo discovery: {}", e); - return None; - } - Some(wsv4_context) - } - .boxed(), - ); + // if protocol_config.inbound.contains(ProtocolType::WS) { + // // WS4 + // unord.push( + // async { + // let wsv4_context = DiscoveryContext::new(self.routing_table(), self.clone()); + // if let Err(e) = self + // .update_ipv4_protocol_dialinfo(&wsv4_context, ProtocolType::WS) + // .await + // { + // log_net!(debug "Failed WSv4 dialinfo discovery: {}", e); + // return None; + // } + // Some(wsv4_context) + // } + // .boxed(), + // ); - // WSv6 - unord.push( - async { - let wsv6_context = DiscoveryContext::new(self.routing_table(), self.clone()); - if let Err(e) = self - .update_ipv6_protocol_dialinfo(&wsv6_context, ProtocolType::TCP) - .await - { - log_net!(debug "Failed WSv6 dialinfo discovery: {}", e); - return None; - } - Some(wsv6_context) - } - .boxed(), - ); - } + // // WSv6 + // unord.push( + // async { + // let wsv6_context = DiscoveryContext::new(self.routing_table(), self.clone()); + // if let Err(e) = self + // .update_ipv6_protocol_dialinfo(&wsv6_context, ProtocolType::TCP) + // .await + // { + // log_net!(debug "Failed WSv6 dialinfo discovery: {}", e); + // return None; + // } + // Some(wsv6_context) + // } + // .boxed(), + // ); + // } // Wait for all discovery futures to complete and collect contexts let mut contexts = Vec::::new(); diff --git a/veilid-core/src/intf/wasm/network/protocol/ws.rs b/veilid-core/src/intf/wasm/network/protocol/ws.rs index 8f9af7ed..767ced1a 100644 --- a/veilid-core/src/intf/wasm/network/protocol/ws.rs +++ b/veilid-core/src/intf/wasm/network/protocol/ws.rs @@ -111,11 +111,13 @@ impl WebsocketProtocolHandler { data.len(), dial_info, ); - + + // Make the real connection let conn = Self::connect(None, dial_info) .await .map_err(|e| format!("failed to connect websocket for unbound message: {}", e))?; conn.send(data).await + } } diff --git a/veilid-core/src/network_manager.rs b/veilid-core/src/network_manager.rs index f1230cbe..55b28449 100644 --- a/veilid-core/src/network_manager.rs +++ b/veilid-core/src/network_manager.rs @@ -421,96 +421,80 @@ impl NetworkManager { } } - // Generates an out-of-band receipt - pub fn generate_receipt>( + // Generates a multi-shot/normal receipt + pub fn generate_receipt( &self, expiration_us: u64, expected_returns: u32, - extra_data: D, callback: impl ReceiptCallback, - ) -> Result, String> { + ) -> Result { let receipt_manager = self.receipt_manager(); - let routing_table = self.routing_table(); - // Generate receipt and serialized form to return - let nonce = Crypto::get_random_nonce(); - let receipt = Receipt::try_new(0, nonce, routing_table.node_id(), extra_data)?; - let out = receipt - .to_signed_data(&routing_table.node_id_secret()) - .map_err(|_| "failed to generate signed receipt".to_owned())?; + // Generate receipt nonce + let receipt_nonce = Crypto::get_random_nonce(); // Record the receipt for later let exp_ts = intf::get_timestamp() + expiration_us; - receipt_manager.record_receipt(receipt, exp_ts, expected_returns, callback); + receipt_manager.record_receipt(receipt_nonce, exp_ts, expected_returns, callback); - Ok(out) + Ok(receipt_nonce) } - pub fn generate_single_shot_receipt>( + // Generates a single-shot/normal receipt + pub fn generate_single_shot_receipt( &self, expiration_us: u64, - extra_data: D, - ) -> Result<(Vec, EventualValueFuture), String> { + ) -> Result<(ReceiptNonce, EventualValueFuture), String> { let receipt_manager = self.receipt_manager(); - let routing_table = self.routing_table(); - // Generate receipt and serialized form to return - let nonce = Crypto::get_random_nonce(); - let receipt = Receipt::try_new(0, nonce, routing_table.node_id(), extra_data)?; - let out = receipt - .to_signed_data(&routing_table.node_id_secret()) - .map_err(|_| "failed to generate signed receipt".to_owned())?; + // Generate receipt nonce + let receipt_nonce = Crypto::get_random_nonce(); // Record the receipt for later let exp_ts = intf::get_timestamp() + expiration_us; let eventual = SingleShotEventual::new(Some(ReceiptEvent::Cancelled)); let instance = eventual.instance(); - receipt_manager.record_single_shot_receipt(receipt, exp_ts, eventual); + receipt_manager.record_single_shot_receipt(receipt_nonce, exp_ts, eventual); - Ok((out, instance)) + Ok((receipt_nonce, instance)) } // Process a received out-of-band receipt pub async fn handle_out_of_band_receipt>( &self, receipt_data: R, - descriptor: ConnectionDescriptor, ) -> Result<(), String> { - let routing_table = self.routing_table(); let receipt_manager = self.receipt_manager(); - let ts = intf::get_timestamp(); let receipt = Receipt::from_signed_data(receipt_data.as_ref()) .map_err(|_| "failed to parse signed receipt".to_owned())?; - // Cache the receipt information in the routing table - let source_noderef = routing_table - .register_node_with_existing_connection(receipt.get_sender_id(), descriptor, ts) - .map_err(|e| format!("node id registration from receipt failed: {}", e))?; - receipt_manager - .handle_receipt(source_noderef, receipt) + .handle_receipt(receipt.get_nonce(), receipt.get_extra_data().to_vec(), None) .await } // Process a received in-band receipt - pub async fn handle_in_band_receipt>( + pub async fn handle_in_band_receipt( &self, - receipt_data: R, + receipt_nonce: ReceiptNonce, + extra_data: Vec, inbound_nr: NodeRef, ) -> Result<(), String> { let receipt_manager = self.receipt_manager(); - let receipt = Receipt::from_signed_data(receipt_data.as_ref()) - .map_err(|_| "failed to parse signed receipt".to_owned())?; - - receipt_manager.handle_receipt(inbound_nr, receipt).await + receipt_manager + .handle_receipt(receipt_nonce, extra_data, Some(inbound_nr)) + .await } // Process a received signal pub async fn handle_signal(&self, signal_info: SignalInfo) -> Result<(), String> { match signal_info { - SignalInfo::ReverseConnect { receipt, peer_info } => { + SignalInfo::ReverseConnect { + receipt_nonce, + peer_info, + } => { let routing_table = self.routing_table(); let rpc = self.rpc_processor(); @@ -521,12 +505,16 @@ impl NetworkManager { )?; // Make a reverse connection to the peer and send the receipt to it - rpc.rpc_call_return_receipt(Destination::Direct(peer_nr), None, receipt) + rpc.rpc_call_return_receipt(Destination::Direct(peer_nr), None, receipt_nonce, []) .await .map_err(map_to_string)?; } - SignalInfo::HolePunch { receipt, peer_info } => { + SignalInfo::HolePunch { + receipt_nonce, + peer_info, + } => { let routing_table = self.routing_table(); + let rpc = self.rpc_processor(); // Add the peer info to our routing table let mut peer_nr = routing_table.register_node_with_signed_node_info( @@ -540,6 +528,12 @@ impl NetworkManager { .first_filtered_dial_info_detail(Some(RoutingDomain::PublicInternet)) .ok_or_else(|| "No hole punch capable dialinfo found for node".to_owned())?; + // Now that we picked a specific dialinfo, further restrict the noderef to the specific address type + let mut filter = peer_nr.take_filter().unwrap(); + filter.peer_scope = PeerScope::Global; + filter.address_type = Some(hole_punch_dial_info_detail.dial_info.address_type()); + peer_nr.set_filter(Some(filter)); + // Do our half of the hole punch by sending an empty packet // Both sides will do this and then the receipt will get sent over the punched hole self.net() @@ -551,8 +545,8 @@ impl NetworkManager { // XXX: do we need a delay here? or another hole punch packet? - // Return the receipt over the direct channel since we want to use exactly the same dial info - self.send_direct_receipt(hole_punch_dial_info_detail.dial_info, receipt, false) + // Return the receipt using the same dial info send the receipt to it + rpc.rpc_call_return_receipt(Destination::Direct(peer_nr), None, receipt_nonce, []) .await .map_err(map_to_string)?; } @@ -639,26 +633,26 @@ impl NetworkManager { } // Called by the RPC handler when we want to issue an direct receipt - pub async fn send_direct_receipt>( + pub async fn send_out_of_band_receipt>( &self, dial_info: DialInfo, - rcpt_data: B, - alternate_port: bool, + version: u8, + receipt_nonce: Nonce, + extra_data: D, ) -> Result<(), String> { - // Validate receipt before we send it, otherwise this may be arbitrary data! - let _ = Receipt::from_signed_data(rcpt_data.as_ref()) - .map_err(|_| "failed to validate direct receipt".to_owned())?; + let routing_table = self.routing_table(); + let node_id = routing_table.node_id(); + let node_id_secret = routing_table.node_id_secret(); + + let receipt = Receipt::try_new(version, receipt_nonce, node_id, extra_data)?; + let rcpt_data = receipt + .to_signed_data(&node_id_secret) + .map_err(|_| "failed to sign receipt".to_owned())?; // Send receipt directly - if alternate_port { - self.net() - .send_data_unbound_to_dial_info(dial_info, rcpt_data.as_ref().to_vec()) - .await - } else { - self.net() - .send_data_to_dial_info(dial_info, rcpt_data.as_ref().to_vec()) - .await - } + self.net() + .send_data_unbound_to_dial_info(dial_info, rcpt_data) + .await } // Figure out how to reach a node @@ -790,8 +784,8 @@ impl NetworkManager { // Build a return receipt for the signal let receipt_timeout = ms_to_us(self.config.get().network.reverse_connection_receipt_time_ms); - let (receipt, eventual_value) = self - .generate_single_shot_receipt(receipt_timeout, []) + let (receipt_nonce, eventual_value) = self + .generate_single_shot_receipt(receipt_timeout) .map_err(map_to_string)?; // Get our peer info @@ -802,14 +796,23 @@ impl NetworkManager { rpc.rpc_call_signal( Destination::Relay(relay_nr.clone(), target_nr.node_id()), None, - SignalInfo::ReverseConnect { receipt, peer_info }, + SignalInfo::ReverseConnect { + receipt_nonce, + peer_info, + }, ) .await .map_err(logthru_net!("failed to send signal to {:?}", relay_nr)) .map_err(map_to_string)?; // Wait for the return receipt let inbound_nr = match eventual_value.await.take_value().unwrap() { - ReceiptEvent::Returned(inbound_nr) => inbound_nr, + ReceiptEvent::ReturnedOutOfBand { extra_data: _ } => { + return Err("reverse connect receipt should be returned in-band".to_owned()); + } + ReceiptEvent::ReturnedInBand { + inbound_noderef, + extra_data: _, + } => inbound_noderef, ReceiptEvent::Expired => { return Err(format!( "reverse connect receipt expired from {:?}", @@ -867,8 +870,8 @@ impl NetworkManager { // Build a return receipt for the signal let receipt_timeout = ms_to_us(self.config.get().network.reverse_connection_receipt_time_ms); - let (receipt, eventual_value) = self - .generate_single_shot_receipt(receipt_timeout, []) + let (receipt_nonce, eventual_value) = self + .generate_single_shot_receipt(receipt_timeout) .map_err(map_to_string)?; // Get our peer info @@ -890,7 +893,10 @@ impl NetworkManager { rpc.rpc_call_signal( Destination::Relay(relay_nr.clone(), target_nr.node_id()), None, - SignalInfo::HolePunch { receipt, peer_info }, + SignalInfo::HolePunch { + receipt_nonce, + peer_info, + }, ) .await .map_err(logthru_net!("failed to send signal to {:?}", relay_nr)) @@ -898,19 +904,28 @@ impl NetworkManager { // Wait for the return receipt let inbound_nr = match eventual_value.await.take_value().unwrap() { - ReceiptEvent::Returned(inbound_nr) => inbound_nr, + ReceiptEvent::ReturnedOutOfBand { extra_data: _ } => { + return Err("hole punch receipt should be returned in-band".to_owned()); + } + ReceiptEvent::ReturnedInBand { + inbound_noderef, + extra_data: _, + } => inbound_noderef, ReceiptEvent::Expired => { - return Err(format!("hole punch receipt expired from {:?}", target_nr)); + return Err(format!("hole punch receipt expired from {}", target_nr)); } ReceiptEvent::Cancelled => { - return Err(format!("hole punch receipt cancelled from {:?}", target_nr)); + return Err(format!("hole punch receipt cancelled from {}", target_nr)); } }; // We expect the inbound noderef to be the same as the target noderef // if they aren't the same, we should error on this and figure out what then hell is up if target_nr != inbound_nr { - error!("unexpected noderef mismatch on hole punch"); + return Err(format!( + "unexpected noderef mismatch on hole punch {}, expected {}", + inbound_nr, target_nr + )); } // And now use the existing connection to send over @@ -1020,9 +1035,14 @@ impl NetworkManager { // Network accounting self.stats_packet_rcvd(descriptor.remote.to_socket_addr().ip(), data.len() as u64); + // Ensure we can read the magic number + if data.len() < 4 { + return Err("short packet".to_owned()); + } + // Is this an out-of-band receipt instead of an envelope? if data[0..4] == *RECEIPT_MAGIC { - self.handle_out_of_band_receipt(data, descriptor).await?; + self.handle_out_of_band_receipt(data).await?; return Ok(true); } diff --git a/veilid-core/src/receipt_manager.rs b/veilid-core/src/receipt_manager.rs index 4b0bf061..4375c84a 100644 --- a/veilid-core/src/receipt_manager.rs +++ b/veilid-core/src/receipt_manager.rs @@ -8,7 +8,13 @@ use xx::*; #[derive(Clone, Debug, PartialEq, Eq)] pub enum ReceiptEvent { - Returned(NodeRef), + ReturnedOutOfBand { + extra_data: Vec, + }, + ReturnedInBand { + inbound_noderef: NodeRef, + extra_data: Vec, + }, Expired, Cancelled, } @@ -108,29 +114,29 @@ impl fmt::Debug for ReceiptRecord { } impl ReceiptRecord { - pub fn from_receipt( - receipt: &Receipt, + pub fn new( + receipt_nonce: ReceiptNonce, expiration_ts: u64, expected_returns: u32, receipt_callback: impl ReceiptCallback, ) -> Self { Self { expiration_ts, - nonce: receipt.get_nonce(), + nonce: receipt_nonce, expected_returns, returns_so_far: 0u32, receipt_callback: ReceiptRecordCallbackType::Normal(Box::new(receipt_callback)), } } - pub fn from_single_shot_receipt( - receipt: &Receipt, + pub fn new_single_shot( + receipt_nonce: ReceiptNonce, expiration_ts: u64, eventual: ReceiptSingleShotType, ) -> Self { Self { expiration_ts, - nonce: receipt.get_nonce(), + nonce: receipt_nonce, returns_so_far: 0u32, expected_returns: 1u32, receipt_callback: ReceiptRecordCallbackType::SingleShot(Some(eventual)), @@ -167,7 +173,7 @@ impl PartialOrd for ReceiptRecordTimestampSort { pub struct ReceiptManagerInner { network_manager: NetworkManager, - receipts_by_nonce: BTreeMap>>, + records_by_nonce: BTreeMap>>, next_oldest_ts: Option, timeout_task: SingleFuture<()>, } @@ -181,7 +187,7 @@ impl ReceiptManager { fn new_inner(network_manager: NetworkManager) -> ReceiptManagerInner { ReceiptManagerInner { network_manager, - receipts_by_nonce: BTreeMap::new(), + records_by_nonce: BTreeMap::new(), next_oldest_ts: None, timeout_task: SingleFuture::new(), } @@ -240,7 +246,7 @@ impl ReceiptManager { { let mut inner = self.inner.lock(); let mut expired_nonces = Vec::new(); - for (k, v) in &inner.receipts_by_nonce { + for (k, v) in &inner.records_by_nonce { let receipt_inner = v.lock(); if receipt_inner.expiration_ts <= now { // Expire this receipt @@ -257,10 +263,7 @@ impl ReceiptManager { } // Now remove the expired receipts for e in expired_nonces { - let expired_record = inner - .receipts_by_nonce - .remove(&e) - .expect("key should exist"); + let expired_record = inner.records_by_nonce.remove(&e).expect("key should exist"); expired_records.push(expired_record); } // Update the next oldest timestamp @@ -305,37 +308,39 @@ impl ReceiptManager { pub fn record_receipt( &self, - receipt: Receipt, + receipt_nonce: ReceiptNonce, expiration: u64, expected_returns: u32, callback: impl ReceiptCallback, ) { - log_rpc!(debug "== New Multiple Receipt ({}) {} ", expected_returns, receipt.get_nonce().encode()); - let record = Arc::new(Mutex::new(ReceiptRecord::from_receipt( - &receipt, + log_rpc!(debug "== New Multiple Receipt ({}) {} ", expected_returns, receipt_nonce.encode()); + let record = Arc::new(Mutex::new(ReceiptRecord::new( + receipt_nonce, expiration, expected_returns, callback, ))); let mut inner = self.inner.lock(); - inner.receipts_by_nonce.insert(receipt.get_nonce(), record); + inner.records_by_nonce.insert(receipt_nonce, record); Self::update_next_oldest_timestamp(&mut *inner); } pub fn record_single_shot_receipt( &self, - receipt: Receipt, + receipt_nonce: ReceiptNonce, expiration: u64, eventual: ReceiptSingleShotType, ) { - log_rpc!(debug "== New SingleShot Receipt {}", receipt.get_nonce().encode()); + log_rpc!(debug "== New SingleShot Receipt {}", receipt_nonce.encode()); - let record = Arc::new(Mutex::new(ReceiptRecord::from_single_shot_receipt( - &receipt, expiration, eventual, + let record = Arc::new(Mutex::new(ReceiptRecord::new_single_shot( + receipt_nonce, + expiration, + eventual, ))); let mut inner = self.inner.lock(); - inner.receipts_by_nonce.insert(receipt.get_nonce(), record); + inner.records_by_nonce.insert(receipt_nonce, record); Self::update_next_oldest_timestamp(&mut *inner); } @@ -343,7 +348,7 @@ impl ReceiptManager { fn update_next_oldest_timestamp(inner: &mut ReceiptManagerInner) { // Update the next oldest timestamp let mut new_next_oldest_ts: Option = None; - for v in inner.receipts_by_nonce.values() { + for v in inner.records_by_nonce.values() { let receipt_inner = v.lock(); if new_next_oldest_ts.is_none() || receipt_inner.expiration_ts < new_next_oldest_ts.unwrap() @@ -362,7 +367,7 @@ impl ReceiptManager { // Remove the record let record = { let mut inner = self.inner.lock(); - let record = match inner.receipts_by_nonce.remove(nonce) { + let record = match inner.records_by_nonce.remove(nonce) { Some(r) => r, None => { return Err("receipt not recorded".to_owned()); @@ -386,14 +391,31 @@ impl ReceiptManager { Ok(()) } - pub async fn handle_receipt(&self, node_ref: NodeRef, receipt: Receipt) -> Result<(), String> { - log_rpc!(debug "<<== RECEIPT {} <- {}", receipt.get_nonce().encode(), node_ref); + pub async fn handle_receipt( + &self, + receipt_nonce: ReceiptNonce, + extra_data: Vec, + inbound_noderef: Option, + ) -> Result<(), String> { + log_rpc!(debug "<<== RECEIPT {} <- {}{}", + receipt_nonce.encode(), + if let Some(nr) = &inbound_noderef { + nr.to_string() + } else { + "DIRECT".to_owned() + }, + if extra_data.is_empty() { + "".to_owned() + } else { + format!("[{} extra]", extra_data.len()) + } + ); // Increment return count let callback_future = { // Look up the receipt record from the nonce let mut inner = self.inner.lock(); - let record = match inner.receipts_by_nonce.get(&receipt.get_nonce()) { + let record = match inner.records_by_nonce.get(&receipt_nonce) { Some(r) => r.clone(), None => { return Err("receipt not recorded".to_owned()); @@ -402,12 +424,22 @@ impl ReceiptManager { // Generate the callback future let mut record_mut = record.lock(); record_mut.returns_so_far += 1; - let callback_future = - Self::perform_callback(ReceiptEvent::Returned(node_ref), &mut record_mut); + + // Get the receipt event to return + let receipt_event = if let Some(inbound_noderef) = inbound_noderef { + ReceiptEvent::ReturnedInBand { + inbound_noderef, + extra_data, + } + } else { + ReceiptEvent::ReturnedOutOfBand { extra_data } + }; + + let callback_future = Self::perform_callback(receipt_event, &mut record_mut); // Remove the record if we're done if record_mut.returns_so_far == record_mut.expected_returns { - inner.receipts_by_nonce.remove(&receipt.get_nonce()); + inner.records_by_nonce.remove(&receipt_nonce); Self::update_next_oldest_timestamp(&mut *inner); } diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index bd121c34..4e11bc78 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -143,6 +143,10 @@ impl BucketEntry { return; } } + self.min_max_version = Some(( + signed_node_info.node_info.min_version, + signed_node_info.node_info.max_version, + )); self.opt_signed_node_info = Some(signed_node_info); } pub fn update_local_node_info(&mut self, local_node_info: LocalNodeInfo) { diff --git a/veilid-core/src/routing_table/debug.rs b/veilid-core/src/routing_table/debug.rs index e0d8a019..d90dd8ec 100644 --- a/veilid-core/src/routing_table/debug.rs +++ b/veilid-core/src/routing_table/debug.rs @@ -30,18 +30,41 @@ impl RoutingTable { if gdis.is_empty() { out += "No TXT Record\n"; } else { - out += "TXT Record:\n"; - out += &self.node_id().encode(); - - let mut urls = Vec::new(); + let mut short_urls = Vec::new(); + let mut some_hostname = Option::::None; for gdi in gdis { - urls.push(gdi.dial_info.to_url().await); - } - urls.sort(); - urls.dedup(); + let (short_url, hostname) = gdi.dial_info.to_short().await; + if let Some(h) = &some_hostname { + if h != &hostname { + return format!( + "Inconsistent hostnames for dial info: {} vs {}", + some_hostname.unwrap(), + hostname + ); + } + } else { + some_hostname = Some(hostname); + } - for url in urls { - out += &format!(",{}", url); + short_urls.push(short_url); + } + if some_hostname.is_none() || short_urls.is_empty() { + return "No dial info for bootstrap host".to_owned(); + } + short_urls.sort(); + short_urls.dedup(); + + out += "TXT Record:\n"; + out += &format!( + "{},{},{},{},{}", + BOOTSTRAP_TXT_VERSION, + MIN_VERSION, + MAX_VERSION, + self.node_id().encode(), + some_hostname.unwrap() + ); + for short_url in short_urls { + out += &format!(",{}", short_url); } out += "\n"; } diff --git a/veilid-core/src/routing_table/find_nodes.rs b/veilid-core/src/routing_table/find_nodes.rs index e2f01d2e..1e44ab9b 100644 --- a/veilid-core/src/routing_table/find_nodes.rs +++ b/veilid-core/src/routing_table/find_nodes.rs @@ -67,6 +67,8 @@ impl RoutingTable { NodeInfo { network_class: netman.get_network_class().unwrap_or(NetworkClass::Invalid), outbound_protocols: netman.get_protocol_config().unwrap_or_default().outbound, + min_version: MIN_VERSION, + max_version: MAX_VERSION, dial_info_detail_list: self.dial_info_details(RoutingDomain::PublicInternet), relay_peer_info: relay_node.and_then(|rn| rn.peer_info().map(Box::new)), } diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index f3f9f512..28bf493a 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -22,6 +22,16 @@ pub use stats_accounting::*; ////////////////////////////////////////////////////////////////////////// +pub const BOOTSTRAP_TXT_VERSION: u8 = 0; + +#[derive(Clone, Debug)] +pub struct BootstrapRecord { + min_version: u8, + max_version: u8, + dial_info_details: Vec, +} +pub type BootstrapRecordMap = BTreeMap; + #[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Ord, Eq)] pub enum RoutingDomain { PublicInternet, @@ -817,9 +827,10 @@ impl RoutingTable { } // Bootstrap lookup process - async fn resolve_bootstrap(&self, bootstrap: Vec) -> Result, String> { - let mut out = Vec::::new(); - + async fn resolve_bootstrap( + &self, + bootstrap: Vec, + ) -> Result { // Resolve from bootstrap root to bootstrap hostnames let mut bsnames = Vec::::new(); for bh in bootstrap { @@ -856,20 +867,64 @@ impl RoutingTable { } Ok(v) => v, }; - // for each record resolve into node dial info strings - let mut nodedialinfos: Vec = Vec::new(); + // for each record resolve into key/bootstraprecord pairs + let mut bootstrap_records: Vec<(DHTKey, BootstrapRecord)> = Vec::new(); for bsnirecord in bsnirecords { - // split bootstrap node record by commas. example: - // 7lxDEabK_qgjbe38RtBa3IZLrud84P6NhGP-pRTZzdQ,tcp://bootstrap-dev-alpha.veilid.net:5150,udp://bootstrap-dev-alpha.veilid.net:5150,ws://bootstrap-dev-alpha.veilid.net:5150/ws - let mut records = bsnirecord.split(',').map(|x| x.trim()); - let node_id_str = match records.next() { - Some(v) => v, - None => { - warn!("no node id specified in bootstrap node txt record"); + // Bootstrap TXT Record Format Version 0: + // txt_version,min_version,max_version,nodeid,hostname,dialinfoshort* + // + // Split bootstrap node record by commas. Example: + // 0,0,0,7lxDEabK_qgjbe38RtBa3IZLrud84P6NhGP-pRTZzdQ,bootstrap-dev-alpha.veilid.net,T5150,U5150,W5150/ws + let records: Vec = bsnirecord + .trim() + .split(',') + .map(|x| x.trim().to_owned()) + .collect(); + if records.len() < 6 { + warn!("invalid number of fields in bootstrap txt record"); + continue; + } + + // Bootstrap TXT record version + let txt_version: u8 = match records[0].parse::() { + Ok(v) => v, + Err(e) => { + warn!( + "invalid txt_version specified in bootstrap node txt record: {}", + e + ); continue; } }; - // Decode the node id + if txt_version != BOOTSTRAP_TXT_VERSION { + warn!("unsupported bootstrap txt record version"); + continue; + } + + // Min/Max wire protocol version + let min_version: u8 = match records[1].parse::() { + Ok(v) => v, + Err(e) => { + warn!( + "invalid min_version specified in bootstrap node txt record: {}", + e + ); + continue; + } + }; + let max_version: u8 = match records[2].parse::() { + Ok(v) => v, + Err(e) => { + warn!( + "invalid max_version specified in bootstrap node txt record: {}", + e + ); + continue; + } + }; + + // Node Id + let node_id_str = &records[3]; let node_id_key = match DHTKey::try_decode(node_id_str) { Ok(v) => v, Err(e) => { @@ -881,16 +936,23 @@ impl RoutingTable { } }; + // Hostname + let hostname_str = &records[4]; + // If this is our own node id, then we skip it for bootstrap, in case we are a bootstrap node if self.node_id() == node_id_key { continue; } // Resolve each record and store in node dial infos list - let node_id = NodeId::new(node_id_key); - for rec in records { + let mut bootstrap_record = BootstrapRecord { + min_version, + max_version, + dial_info_details: Vec::new(), + }; + for rec in &records[5..] { let rec = rec.trim(); - let dial_infos = match DialInfo::try_vec_from_url(rec) { + let dial_infos = match DialInfo::try_vec_from_short(rec, hostname_str) { Ok(dis) => dis, Err(e) => { warn!("Couldn't resolve bootstrap node dial info {}: {}", rec, e); @@ -898,24 +960,34 @@ impl RoutingTable { } }; - for dial_info in dial_infos { - nodedialinfos.push(NodeDialInfo { - node_id: node_id.clone(), - dial_info, - }) + for di in dial_infos { + bootstrap_record.dial_info_details.push(DialInfoDetail { + dial_info: di, + class: DialInfoClass::Direct, + }); } } + bootstrap_records.push((node_id_key, bootstrap_record)); } - Some(nodedialinfos) + Some(bootstrap_records) }); } - while let Some(ndis) = unord.next().await { - if let Some(mut ndis) = ndis { - out.append(&mut ndis); + + let mut bsmap = BootstrapRecordMap::new(); + while let Some(bootstrap_records) = unord.next().await { + if let Some(bootstrap_records) = bootstrap_records { + for (bskey, mut bsrec) in bootstrap_records { + let rec = bsmap.entry(bskey).or_insert_with(|| BootstrapRecord { + min_version: bsrec.min_version, + max_version: bsrec.max_version, + dial_info_details: Vec::new(), + }); + rec.dial_info_details.append(&mut bsrec.dial_info_details); + } } } - Ok(out) + Ok(bsmap) } async fn bootstrap_task_routine(self) -> Result<(), String> { @@ -930,8 +1002,10 @@ impl RoutingTable { log_rtab!(debug "--- bootstrap_task"); // If we aren't specifying a bootstrap node list explicitly, then pull from the bootstrap server(s) - let bootstrap_node_dial_infos = if !bootstrap_nodes.is_empty() { - let mut bsnvec = Vec::new(); + + let bsmap: BootstrapRecordMap = if !bootstrap_nodes.is_empty() { + let mut bsmap = BootstrapRecordMap::new(); + let mut bootstrap_node_dial_infos = Vec::new(); for b in bootstrap_nodes { let ndis = NodeDialInfo::from_str(b.as_str()) .map_err(map_to_string) @@ -939,26 +1013,30 @@ impl RoutingTable { "Invalid node dial info in bootstrap entry: {}", b ))?; - bsnvec.push(ndis); + bootstrap_node_dial_infos.push(ndis); } - bsnvec + for ndi in bootstrap_node_dial_infos { + let node_id = ndi.node_id.key; + bsmap + .entry(node_id) + .or_insert_with(|| BootstrapRecord { + min_version: MIN_VERSION, + max_version: MAX_VERSION, + dial_info_details: Vec::new(), + }) + .dial_info_details + .push(DialInfoDetail { + dial_info: ndi.dial_info, + class: DialInfoClass::Direct, // Bootstraps are always directly reachable + }); + } + bsmap } else { // Resolve bootstrap servers and recurse their TXT entries self.resolve_bootstrap(bootstrap).await? }; // Map all bootstrap entries to a single key with multiple dialinfo - let mut bsmap: BTreeMap> = BTreeMap::new(); - for ndi in bootstrap_node_dial_infos { - let node_id = ndi.node_id.key; - bsmap - .entry(node_id) - .or_insert_with(Vec::new) - .push(DialInfoDetail { - dial_info: ndi.dial_info, - class: DialInfoClass::Direct, // Bootstraps are always directly reachable - }); - } // Run all bootstrap operations concurrently let mut unord = FuturesUnordered::new(); @@ -972,8 +1050,10 @@ impl RoutingTable { SignedNodeInfo::with_no_signature(NodeInfo { network_class: NetworkClass::InboundCapable, // Bootstraps are always inbound capable outbound_protocols: ProtocolSet::empty(), // Bootstraps do not participate in relaying and will not make outbound requests - dial_info_detail_list: v, // Dial info is as specified in the bootstrap list - relay_peer_info: None, // Bootstraps never require a relay themselves + min_version: v.min_version, // Minimum protocol version specified in txt record + max_version: v.max_version, // Maximum protocol version specified in txt record + dial_info_detail_list: v.dial_info_details, // Dial info is as specified in the bootstrap list + relay_peer_info: None, // Bootstraps never require a relay themselves }), ) .map_err(logthru_rtab!(error "Couldn't add bootstrap node: {}", k))?; diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 8ef3dc99..54082a13 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -228,6 +228,14 @@ impl NodeRef { None } })?; + + // Verify this connection matches the noderef filter + if let Some(filter) = &self.filter { + if !last_connection.matches_filter(filter) { + return None; + } + } + // Should we check the connection table? if last_connection.protocol_type().is_connection_oriented() { // Look the connection up in the connection manager and see if it's still there diff --git a/veilid-core/src/rpc_processor/coders/node_info.rs b/veilid-core/src/rpc_processor/coders/node_info.rs index 46dc3097..f338f473 100644 --- a/veilid-core/src/rpc_processor/coders/node_info.rs +++ b/veilid-core/src/rpc_processor/coders/node_info.rs @@ -10,6 +10,9 @@ pub fn encode_node_info( let mut ps_builder = builder.reborrow().init_outbound_protocols(); encode_protocol_set(&node_info.outbound_protocols, &mut ps_builder)?; + builder.set_min_version(node_info.min_version); + builder.set_max_version(node_info.max_version); + let mut didl_builder = builder.reborrow().init_dial_info_detail_list( node_info .dial_info_detail_list @@ -51,6 +54,9 @@ pub fn decode_node_info( .map_err(map_error_capnp_error!())?, )?; + let min_version = reader.reborrow().get_min_version(); + let max_version = reader.reborrow().get_max_version(); + let didl_reader = reader .reborrow() .get_dial_info_detail_list() @@ -84,6 +90,8 @@ pub fn decode_node_info( Ok(NodeInfo { network_class, outbound_protocols, + min_version, + max_version, dial_info_detail_list, relay_peer_info, }) diff --git a/veilid-core/src/rpc_processor/coders/signal_info.rs b/veilid-core/src/rpc_processor/coders/signal_info.rs index c87454b9..b3301545 100644 --- a/veilid-core/src/rpc_processor/coders/signal_info.rs +++ b/veilid-core/src/rpc_processor/coders/signal_info.rs @@ -6,28 +6,24 @@ pub fn encode_signal_info( builder: &mut veilid_capnp::operation_signal::Builder, ) -> Result<(), RPCError> { match signal_info { - SignalInfo::HolePunch { receipt, peer_info } => { + SignalInfo::HolePunch { + receipt_nonce, + peer_info, + } => { let mut hp_builder = builder.reborrow().init_hole_punch(); - let rcpt_builder = - hp_builder - .reborrow() - .init_receipt(receipt.len().try_into().map_err(map_error_protocol!( - "invalid receipt length in hole punch signal info" - ))?); - rcpt_builder.copy_from_slice(receipt.as_slice()); + let mut rn_builder = hp_builder.reborrow().init_receipt_nonce(); + encode_nonce(receipt_nonce, &mut rn_builder); let mut pi_builder = hp_builder.init_peer_info(); encode_peer_info(peer_info, &mut pi_builder)?; } - SignalInfo::ReverseConnect { receipt, peer_info } => { - let mut hp_builder = builder.reborrow().init_reverse_connect(); - let rcpt_builder = - hp_builder - .reborrow() - .init_receipt(receipt.len().try_into().map_err(map_error_protocol!( - "invalid receipt length in reverse connect signal info" - ))?); - rcpt_builder.copy_from_slice(receipt.as_slice()); - let mut pi_builder = hp_builder.init_peer_info(); + SignalInfo::ReverseConnect { + receipt_nonce, + peer_info, + } => { + let mut rc_builder = builder.reborrow().init_reverse_connect(); + let mut rn_builder = rc_builder.reborrow().init_receipt_nonce(); + encode_nonce(receipt_nonce, &mut rn_builder); + let mut pi_builder = rc_builder.init_peer_info(); encode_peer_info(peer_info, &mut pi_builder)?; } } @@ -49,18 +45,17 @@ pub fn decode_signal_info( Ok(r) => r, Err(_) => return Err(rpc_error_internal("invalid hole punch")), }; - let receipt = r - .get_receipt() - .map_err(map_error_protocol!( - "invalid receipt in hole punch signal info" - ))? - .to_vec(); + let receipt_nonce = + decode_nonce(&r.get_receipt_nonce().map_err(map_error_capnp_error!())?); let pi_reader = r.get_peer_info().map_err(map_error_protocol!( "invalid peer info in hole punch signal info" ))?; let peer_info = decode_peer_info(&pi_reader, true)?; - SignalInfo::HolePunch { receipt, peer_info } + SignalInfo::HolePunch { + receipt_nonce, + peer_info, + } } veilid_capnp::operation_signal::ReverseConnect(r) => { // Extract reverse connect reader @@ -68,18 +63,17 @@ pub fn decode_signal_info( Ok(r) => r, Err(_) => return Err(rpc_error_internal("invalid reverse connect")), }; - let receipt = r - .get_receipt() - .map_err(map_error_protocol!( - "invalid receipt in reverse connect signal info" - ))? - .to_vec(); + let receipt_nonce = + decode_nonce(&r.get_receipt_nonce().map_err(map_error_capnp_error!())?); let pi_reader = r.get_peer_info().map_err(map_error_protocol!( "invalid peer info in reverse connect signal info" ))?; let peer_info = decode_peer_info(&pi_reader, true)?; - SignalInfo::ReverseConnect { receipt, peer_info } + SignalInfo::ReverseConnect { + receipt_nonce, + peer_info, + } } }, ) diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 5444b0eb..5d6e3b6a 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -872,7 +872,7 @@ impl RPCProcessor { rpcreader: RPCMessageReader, ) -> Result<(), RPCError> { // - let (alternate_port, redirect, dial_info, rcpt_data) = { + let (redirect, dial_info, receipt_nonce, min_version, max_version) = { let operation = rpcreader .reader .get_root::() @@ -893,18 +893,20 @@ impl RPCProcessor { }; // Parse out fields - let alternate_port = vdi_reader.get_alternate_port(); + let min_version = vdi_reader.get_min_version(); + let max_version = vdi_reader.get_max_version(); let redirect = vdi_reader.get_redirect(); let dial_info = decode_dial_info( &vdi_reader .get_dial_info() .map_err(map_error_internal!("no valid dial info"))?, )?; - let rcpt_data = vdi_reader - .get_receipt() - .map_err(map_error_internal!("no valid receipt"))?; + let rn_reader = vdi_reader + .get_receipt_nonce() + .map_err(map_error_internal!("no valid receipt nonce"))?; + let receipt_nonce = decode_nonce(&rn_reader); - (alternate_port, redirect, dial_info, rcpt_data) + (redirect, dial_info, receipt_nonce, min_version, max_version) }; // Redirect this request if we are asked to @@ -946,18 +948,13 @@ impl RPCProcessor { respond_to.set_none(()); let detail = question.reborrow().init_detail(); let mut vdi_builder = detail.init_validate_dial_info(); - vdi_builder.set_alternate_port(alternate_port); vdi_builder.set_redirect(false); + vdi_builder.set_min_version(min_version); + vdi_builder.set_max_version(max_version); let mut di_builder = vdi_builder.reborrow().init_dial_info(); encode_dial_info(&dial_info, &mut di_builder)?; - let r_builder = vdi_builder.reborrow().init_receipt( - rcpt_data - .len() - .try_into() - .map_err(map_error_internal!("receipt too large"))?, - ); - r_builder.copy_from_slice(rcpt_data); - + let mut rn_builder = vdi_builder.reborrow().init_receipt_nonce(); + encode_nonce(&receipt_nonce, &mut rn_builder); vdi_msg.into_reader() }; @@ -970,13 +967,25 @@ impl RPCProcessor { // Otherwise send a return receipt directly // Possibly from an alternate port + let version = { + #[allow(clippy::absurd_extreme_comparisons)] + if min_version > MAX_VERSION || max_version < MIN_VERSION { + return Err(rpc_error_protocol(format!( + "can't send direct receipt to {} because version is unsupported: ({},{})", + dial_info, min_version, max_version + ))) + .map_err( + logthru_rpc!(debug)); + } + cmp::min(max_version, MAX_VERSION) + }; let network_manager = self.network_manager(); network_manager - .send_direct_receipt(dial_info.clone(), rcpt_data, alternate_port) + .send_out_of_band_receipt(dial_info.clone(), version, receipt_nonce, []) .await .map_err(map_error_string!()) .map_err( - logthru_net!(error "failed to send direct receipt to dial info: {}, alternate_port={}", dial_info, alternate_port), + logthru_net!(error "failed to send direct receipt to dial info: {}, version={}", dial_info, version), )?; Ok(()) @@ -1157,7 +1166,7 @@ impl RPCProcessor { } async fn process_return_receipt(&self, rpcreader: RPCMessageReader) -> Result<(), RPCError> { - let rcpt_data = { + let (receipt_nonce, extra_data) = { let operation = rpcreader .reader .get_root::() @@ -1177,18 +1186,24 @@ impl RPCProcessor { _ => panic!("invalid operation type in process_return_receipt"), }; - // Get receipt data - let rcpt_data = rr_reader - .get_receipt() - .map_err(map_error_internal!("no valid receipt"))?; + // Get receipt nonce + let rn_reader = rr_reader + .get_receipt_nonce() + .map_err(map_error_internal!("no valid receipt_nonce"))?; + let receipt_nonce = decode_nonce(&rn_reader); - rcpt_data.to_vec() + // Get receipt extra data + let extra_data = rr_reader + .get_extra_data() + .map_err(map_error_internal!("no valid extra data"))?; + + (receipt_nonce, extra_data.to_vec()) }; // Handle it let network_manager = self.network_manager(); network_manager - .handle_in_band_receipt(rcpt_data, rpcreader.header.peer_noderef) + .handle_in_band_receipt(receipt_nonce, extra_data, rpcreader.header.peer_noderef) .await .map_err(map_error_string!()) } @@ -1567,7 +1582,6 @@ impl RPCProcessor { peer: NodeRef, dial_info: DialInfo, redirect: bool, - alternate_port: bool, ) -> Result { let network_manager = self.network_manager(); let receipt_time = ms_to_us( @@ -1588,21 +1602,18 @@ impl RPCProcessor { let mut vdi_builder = detail.init_validate_dial_info(); // Generate receipt and waitable eventual so we can see if we get the receipt back - let (rcpt_data, eventual_value) = network_manager - .generate_single_shot_receipt(receipt_time, []) + let (receipt_nonce, eventual_value) = network_manager + .generate_single_shot_receipt(receipt_time) .map_err(map_error_string!())?; vdi_builder.set_redirect(redirect); - vdi_builder.set_alternate_port(alternate_port); + vdi_builder.set_min_version(MIN_VERSION); + vdi_builder.set_max_version(MAX_VERSION); let mut di_builder = vdi_builder.reborrow().init_dial_info(); encode_dial_info(&dial_info, &mut di_builder)?; - let r_builder = vdi_builder.reborrow().init_receipt( - rcpt_data - .len() - .try_into() - .map_err(map_error_internal!("receipt too large"))?, - ); - r_builder.copy_from_slice(rcpt_data.as_slice()); + let mut rn_builder = vdi_builder.reborrow().init_receipt_nonce(); + encode_nonce(&receipt_nonce, &mut rn_builder); + (vdi_msg.into_reader(), eventual_value) }; @@ -1614,7 +1625,15 @@ impl RPCProcessor { log_net!(debug "waiting for validate_dial_info receipt"); // Wait for receipt match eventual_value.await.take_value().unwrap() { - ReceiptEvent::Returned(_) => { + ReceiptEvent::ReturnedInBand { + inbound_noderef: _, + extra_data:_ + } => { + Err(rpc_error_internal("validate_dial_info receipt should be returned out-of-band")) + } + ReceiptEvent::ReturnedOutOfBand { + extra_data:_ + } => { log_net!(debug "validate_dial_info receipt returned"); Ok(true) } @@ -1764,12 +1783,10 @@ impl RPCProcessor { &self, dest: Destination, safety_route: Option<&SafetyRouteSpec>, - rcpt_data: B, - ) -> Result<(), RPCError> { - // Validate receipt before we send it, otherwise this may be arbitrary data! - let _ = Receipt::from_signed_data(rcpt_data.as_ref()) - .map_err(|_| "failed to validate direct receipt".to_owned()) - .map_err(map_error_string!())?; + receipt_nonce: Nonce, + extra_data: B, + ) -> Result<(), RPCError> { + let extra_data = extra_data.as_ref(); let rr_msg = { let mut rr_msg = ::capnp::message::Builder::new_default(); @@ -1778,12 +1795,13 @@ impl RPCProcessor { let mut respond_to = question.reborrow().init_respond_to(); respond_to.set_none(()); let detail = question.reborrow().init_detail(); - let rr_builder = detail.init_return_receipt(); - - let r_builder = rr_builder.init_receipt(rcpt_data.as_ref().len().try_into().map_err( - map_error_protocol!("invalid receipt length in return receipt"), + let mut rr_builder = detail.init_return_receipt(); + let mut rn_builder = rr_builder.reborrow().init_receipt_nonce(); + encode_nonce(&receipt_nonce, &mut rn_builder); + let ed_builder = rr_builder.init_extra_data(extra_data.len().try_into().map_err( + map_error_protocol!("invalid extra data length in return receipt"), )?); - r_builder.copy_from_slice(rcpt_data.as_ref()); + ed_builder.copy_from_slice(extra_data); rr_msg.into_reader() }; diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 1efa902a..8ccfe5ff 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -14,6 +14,7 @@ pub use attachment_manager::AttachmentManager; pub use core::str::FromStr; pub use dht::crypto::Crypto; pub use dht::key::{generate_secret, sign, verify, DHTKey, DHTKeySecret, DHTSignature}; +pub use dht::receipt::ReceiptNonce; pub use intf::BlockStore; pub use intf::ProtectedStore; pub use intf::TableStore; @@ -337,6 +338,8 @@ pub struct NodeStatus { pub struct NodeInfo { pub network_class: NetworkClass, pub outbound_protocols: ProtocolSet, + pub min_version: u8, + pub max_version: u8, pub dial_info_detail_list: Vec, pub relay_peer_info: Option>, } @@ -1039,6 +1042,36 @@ impl DialInfo { } } + pub fn try_vec_from_short, H: AsRef>( + short: S, + hostname: H, + ) -> Result, VeilidAPIError> { + let short = short.as_ref(); + let hostname = hostname.as_ref(); + + if short.len() < 2 { + return Err(parse_error!("invalid short url length", short)); + } + let url = match &short[0..1] { + "U" => { + format!("udp://{}:{}", hostname, &short[1..]) + } + "T" => { + format!("tcp://{}:{}", hostname, &short[1..]) + } + "W" => { + format!("ws://{}:{}", hostname, &short[1..]) + } + "S" => { + format!("wss://{}:{}", hostname, &short[1..]) + } + _ => { + return Err(parse_error!("invalid short url type", short)); + } + }; + Self::try_vec_from_url(url) + } + pub fn try_vec_from_url>(url: S) -> Result, VeilidAPIError> { let url = url.as_ref(); let split_url = SplitUrl::from_str(url) @@ -1088,6 +1121,60 @@ impl DialInfo { Ok(out) } + pub async fn to_short(&self) -> (String, String) { + match self { + DialInfo::UDP(di) => ( + format!("U{}", di.socket_address.port()), + intf::ptr_lookup(di.socket_address.to_ip_addr()) + .await + .unwrap_or_else(|_| di.socket_address.to_string()), + ), + DialInfo::TCP(di) => ( + format!("T{}", di.socket_address.port()), + intf::ptr_lookup(di.socket_address.to_ip_addr()) + .await + .unwrap_or_else(|_| di.socket_address.to_string()), + ), + DialInfo::WS(di) => { + let mut split_url = SplitUrl::from_str(&format!("ws://{}", di.request)).unwrap(); + if let SplitUrlHost::IpAddr(a) = split_url.host { + if let Ok(host) = intf::ptr_lookup(a).await { + split_url.host = SplitUrlHost::Hostname(host); + } + } + ( + format!( + "W{}{}", + split_url.port.unwrap_or(80), + split_url + .path + .map(|p| format!("/{}", p)) + .unwrap_or_default() + ), + split_url.host.to_string(), + ) + } + DialInfo::WSS(di) => { + let mut split_url = SplitUrl::from_str(&format!("wss://{}", di.request)).unwrap(); + if let SplitUrlHost::IpAddr(a) = split_url.host { + if let Ok(host) = intf::ptr_lookup(a).await { + split_url.host = SplitUrlHost::Hostname(host); + } + } + ( + format!( + "S{}{}", + split_url.port.unwrap_or(443), + split_url + .path + .map(|p| format!("/{}", p)) + .unwrap_or_default() + ), + split_url.host.to_string(), + ) + } + } + } pub async fn to_url(&self) -> String { match self { DialInfo::UDP(di) => intf::ptr_lookup(di.socket_address.to_ip_addr()) @@ -1395,13 +1482,13 @@ cfg_if! { pub enum SignalInfo { HolePunch { // UDP Hole Punch Request - receipt: Vec, // Receipt to be returned after the hole punch - peer_info: PeerInfo, // Sender's peer info + receipt_nonce: ReceiptNonce, // Receipt to be returned after the hole punch + peer_info: PeerInfo, // Sender's peer info }, ReverseConnect { // Reverse Connection Request - receipt: Vec, // Receipt to be returned by the reverse connection - peer_info: PeerInfo, // Sender's peer info + receipt_nonce: ReceiptNonce, // Receipt to be returned by the reverse connection + peer_info: PeerInfo, // Sender's peer info }, // XXX: WebRTC // XXX: App-level signalling @@ -1734,7 +1821,6 @@ impl VeilidAPI { node_id: NodeId, dial_info: DialInfo, redirect: bool, - alternate_port: bool, ) -> Result { let rpc = self.rpc_processor()?; let routing_table = rpc.routing_table(); @@ -1742,7 +1828,7 @@ impl VeilidAPI { None => return Err(VeilidAPIError::NodeNotFound { node_id }), Some(nr) => nr, }; - rpc.rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect, alternate_port) + rpc.rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect) .await .map_err(map_rpc_error!()) } diff --git a/veilid-core/src/xx/tools.rs b/veilid-core/src/xx/tools.rs index 6609c51e..232f4c30 100644 --- a/veilid-core/src/xx/tools.rs +++ b/veilid-core/src/xx/tools.rs @@ -183,7 +183,7 @@ cfg_if::cfg_if! { if #[cfg(unix)] { use std::os::unix::fs::MetadataExt; use std::os::unix::prelude::PermissionsExt; - use nix::unistd::{chown, Uid, Gid}; + use nix::unistd::{Uid, Gid}; pub fn ensure_file_private_owner>(path: P) -> Result<(), String> { @@ -200,7 +200,7 @@ cfg_if::cfg_if! { std::fs::set_permissions(path,std::fs::Permissions::from_mode(0o600)).map_err(|e| format!("unable to set correct permissions on path '{:?}': {}", path, e))?; } if meta.uid() != uid.as_raw() || meta.gid() != gid.as_raw() { - chown(path, Some(uid), Some(gid)).map_err(|e| format!("unable to set correct owner on path '{:?}': {}", path, e))?; + return Err(format!("path has incorrect owner/group: {:?}", path)); } Ok(()) } @@ -223,9 +223,9 @@ cfg_if::cfg_if! { std::fs::set_permissions(path,std::fs::Permissions::from_mode(0o600)).map_err(|e| format!("unable to set correct permissions on path '{:?}': {}", path, e))?; } - // if meta.uid() != uid.as_raw() || meta.gid() != gid.as_raw() { + //if meta.uid() != uid.as_raw() || meta.gid() != gid.as_raw() { // chown(path, Some(uid), Some(gid)).map_err(|e| format!("unable to set correct owner on path '{:?}': {}", path, e))?; - // } + //} Ok(()) } } else { diff --git a/veilid-server/src/server.rs b/veilid-server/src/server.rs index 216a12ef..6b123e23 100644 --- a/veilid-server/src/server.rs +++ b/veilid-server/src/server.rs @@ -32,6 +32,18 @@ pub async fn run_veilid_server( settings: Settings, logs: VeilidLogs, server_mode: ServerMode, +) -> Result<(), String> { + run_veilid_server_internal(settings, logs, server_mode) + .await + .map_err(|e| { + error!("{}", e); + e + }) +} +pub async fn run_veilid_server_internal( + settings: Settings, + logs: VeilidLogs, + server_mode: ServerMode, ) -> Result<(), String> { let settingsr = settings.read(); diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index 7ec12752..69e36739 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -89,7 +89,7 @@ core: set_value_timeout: set_value_count: 20 set_value_fanout: 5 - min_peer_count: 20 + min_peer_count: 1 # 20 min_peer_refresh_time_ms: 2000 validate_dial_info_receipt_time_ms: 5000 upnp: false @@ -97,8 +97,8 @@ core: enable_local_peer_scope: false restricted_nat_retries: 3 tls: - certificate_path: '/etc/veilid-server/server.crt' - private_key_path: '/etc/veilid-server/private/server.key' + certificate_path: '%CERTIFICATE_DIRECTORY%/server.crt' + private_key_path: '%PRIVATE_KEY_DIRECTORY%/server.key' connection_initial_timeout_ms: 2000 application: https: @@ -150,6 +150,14 @@ core: .replace( "%INSECURE_FALLBACK_DIRECTORY%", &Settings::get_default_protected_store_insecure_fallback_directory().to_string_lossy(), + ) + .replace( + "%CERTIFICATE_DIRECTORY%", + &Settings::get_default_certificate_directory().to_string_lossy(), + ) + .replace( + "%PRIVATE_KEY_DIRECTORY%", + &Settings::get_default_private_key_directory().to_string_lossy(), ); config::Config::builder() .add_source(config::File::from_str( @@ -723,68 +731,123 @@ impl Settings { Ok(()) } - fn is_root() -> bool { - cfg_if::cfg_if! { - if #[cfg(unix)] { - use nix::unistd::Uid; - Uid::effective().is_root() - } else { - false + pub fn get_default_config_path() -> PathBuf { + #[cfg(unix)] + { + let globalpath = PathBuf::from("/etc/veilid-server/veilid-server.conf"); + if globalpath.exists() { + return globalpath; } } - } - pub fn get_default_config_path() -> PathBuf { - let mut default_config_path = if Self::is_root() { - PathBuf::from("/etc/veilid-server") - } else if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { + let mut cfg_path = if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") + { PathBuf::from(my_proj_dirs.config_dir()) } else { PathBuf::from("./") }; + cfg_path.push("veilid-server.conf"); - default_config_path.push("veilid-server.conf"); - - default_config_path + cfg_path } pub fn get_default_table_store_path() -> PathBuf { - let mut default_db_path = if Self::is_root() { - PathBuf::from("/var/db/veilid-server") - } else if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { + #[cfg(unix)] + { + let globalpath = PathBuf::from("/var/db/veilid-server/table_store"); + if globalpath.exists() { + return globalpath; + } + } + + let mut ts_path = if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { PathBuf::from(my_proj_dirs.data_local_dir()) } else { PathBuf::from("./") }; - default_db_path.push("table_store"); + ts_path.push("table_store"); - default_db_path + ts_path } pub fn get_default_block_store_path() -> PathBuf { - let mut default_db_path = if Self::is_root() { - PathBuf::from("/var/db/veilid-server") - } else if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { + #[cfg(unix)] + { + let globalpath = PathBuf::from("/var/db/veilid-server/block_store"); + if globalpath.exists() { + return globalpath; + } + } + + let mut bs_path = if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { PathBuf::from(my_proj_dirs.data_local_dir()) } else { PathBuf::from("./") }; - default_db_path.push("block_store"); + bs_path.push("block_store"); - default_db_path + bs_path } pub fn get_default_protected_store_insecure_fallback_directory() -> PathBuf { - let mut default_db_path = if Self::is_root() { - PathBuf::from("/var/db/veilid-server") - } else if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { + #[cfg(unix)] + { + let globalpath = PathBuf::from("/var/db/veilid-server/protected_store"); + if globalpath.exists() { + return globalpath; + } + } + + let mut ps_path = if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { PathBuf::from(my_proj_dirs.data_local_dir()) } else { PathBuf::from("./") }; - default_db_path.push("protected_store"); + ps_path.push("protected_store"); - default_db_path + ps_path + } + + pub fn get_default_certificate_directory() -> PathBuf { + #[cfg(unix)] + { + let mut globalpath = PathBuf::from("/etc/veilid-server"); + if globalpath.exists() { + globalpath.push("ssl"); + globalpath.push("certs"); + return globalpath; + } + } + + let mut c_path = if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { + PathBuf::from(my_proj_dirs.data_local_dir()) + } else { + PathBuf::from("./") + }; + c_path.push("ssl"); + c_path.push("certs"); + c_path + } + + pub fn get_default_private_key_directory() -> PathBuf { + #[cfg(unix)] + { + let mut globalpath = PathBuf::from("/etc/veilid-server"); + if globalpath.exists() { + globalpath.push("ssl"); + globalpath.push("keys"); + return globalpath; + } + } + + let mut pk_path = if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { + PathBuf::from(my_proj_dirs.data_local_dir()) + } else { + PathBuf::from("./") + }; + pk_path.push("ssl"); + pk_path.push("keys"); + pk_path } pub fn get_core_config_callback(&self) -> veilid_core::ConfigCallback { diff --git a/veilid-server/src/unix.rs b/veilid-server/src/unix.rs index 62efddda..d2ca5e85 100644 --- a/veilid-server/src/unix.rs +++ b/veilid-server/src/unix.rs @@ -6,6 +6,7 @@ use async_std::task; use clap::ArgMatches; use signal_hook::consts::signal::*; use signal_hook_async_std::Signals; +use std::io::Read; async fn handle_signals(mut signals: Signals) { while let Some(signal) = signals.next().await { @@ -26,8 +27,25 @@ pub fn run_daemon(settings: Settings, _matches: ArgMatches) -> Result<(), String let daemon = { let mut daemon = daemonize::Daemonize::new(); let s = settings.read(); - if let Some(pid_file) = &s.daemon.pid_file { - daemon = daemon.pid_file(pid_file); //.chown_pid_file(true); + if let Some(pid_file) = s.daemon.pid_file.clone() { + daemon = daemon.pid_file(pid_file.clone()); //.chown_pid_file(true); + daemon = daemon.exit_action(move || { + // wait for pid file to exist before exiting parent + let pid_path = std::path::Path::new(&pid_file); + loop { + if let Ok(mut f) = std::fs::File::open(pid_path) { + let mut s = String::new(); + if f.read_to_string(&mut s).is_ok() + && !s.is_empty() + && s.parse::().is_ok() + { + println!("pidfile found"); + break; + } + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + }) } if let Some(chroot) = &s.daemon.chroot { daemon = daemon.chroot(chroot);