From 6a86f2265ac7b316e3f6dad92d639ea8d6f18a52 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 3 Jun 2023 18:33:27 -0400 Subject: [PATCH 01/23] json api work --- Cargo.lock | 42 ++++ veilid-core/Cargo.toml | 1 + veilid-core/proto/veilid.capnp | 14 +- veilid-core/src/crypto/mod.rs | 3 +- veilid-core/src/lib.rs | 1 + veilid-core/src/rpc_processor/coders/mod.rs | 2 + .../rpc_processor/coders/operations/answer.rs | 15 ++ .../rpc_processor/coders/operations/mod.rs | 20 +- .../operations/operation_cancel_tunnel.rs | 2 + .../operations/operation_complete_tunnel.rs | 2 + .../operations/operation_start_tunnel.rs | 2 + .../coders/operations/question.rs | 15 ++ veilid-core/src/rpc_processor/mod.rs | 13 +- veilid-core/src/veilid_api/api.rs | 3 + veilid-core/src/veilid_api/error.rs | 6 +- .../src/veilid_api/json_api/crypto_system.rs | 66 ++++++ veilid-core/src/veilid_api/json_api/mod.rs | 203 ++++++++++++++++++ .../veilid_api/json_api/routing_context.rs | 52 +++++ .../src/veilid_api/json_api/table_db.rs | 26 +++ veilid-core/src/veilid_api/mod.rs | 2 + .../serialize_range_set_blaze.rs | 10 +- .../src/veilid_api/types/aligned_u64.rs | 11 +- .../src/veilid_api/types/app_message_call.rs | 55 ++++- .../types/dht/dht_record_descriptor.rs | 4 + .../src/veilid_api/types/dht/schema/dflt.rs | 1 + .../src/veilid_api/types/dht/schema/mod.rs | 1 + .../src/veilid_api/types/dht/schema/smpl.rs | 3 + .../src/veilid_api/types/dht/value_data.rs | 12 ++ .../types/dht/value_subkey_range_set.rs | 7 +- veilid-core/src/veilid_api/types/fourcc.rs | 16 ++ veilid-core/src/veilid_api/types/mod.rs | 2 + veilid-core/src/veilid_api/types/safety.rs | 5 + veilid-core/src/veilid_api/types/stats.rs | 20 +- veilid-core/src/veilid_api/types/tunnel.rs | 34 ++- .../src/veilid_api/types/veilid_log.rs | 12 +- .../src/veilid_api/types/veilid_state.rs | 81 ++++++- veilid-core/src/veilid_config.rs | 22 +- veilid-server/src/cmdline.rs | 8 + veilid-server/src/main.rs | 19 ++ 39 files changed, 751 insertions(+), 62 deletions(-) create mode 100644 veilid-core/src/veilid_api/json_api/crypto_system.rs create mode 100644 veilid-core/src/veilid_api/json_api/mod.rs create mode 100644 veilid-core/src/veilid_api/json_api/routing_context.rs create mode 100644 veilid-core/src/veilid_api/json_api/table_db.rs diff --git a/Cargo.lock b/Cargo.lock index 047c015a..9866cef7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1678,6 +1678,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" +[[package]] +name = "dyn-clone" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" + [[package]] name = "ed25519" version = "1.5.3" @@ -4689,6 +4695,30 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schemars" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -4854,6 +4884,17 @@ dependencies = [ "syn 2.0.18", ] +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "serde_json" version = "1.0.96" @@ -6212,6 +6253,7 @@ dependencies = [ "rusqlite", "rustls 0.19.1", "rustls-pemfile 0.2.1", + "schemars", "secrecy", "send_wrapper 0.6.0", "serde", diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 6eb99b12..e73f3db9 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -72,6 +72,7 @@ data-encoding = { version = "^2" } weak-table = "0.3.2" range-set-blaze = "0.1.5" argon2 = "0.5.0" +schemars = "0.8.12" # Dependencies for native builds only # Linux, Windows, Mac, iOS, Android diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 2fa813b6..7e0ba4fb 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -500,9 +500,10 @@ struct Question @0xd8510bc33492ef70 { findBlockQ @9 :OperationFindBlockQ; # Tunnel operations - startTunnelQ @10 :OperationStartTunnelQ; - completeTunnelQ @11 :OperationCompleteTunnelQ; - cancelTunnelQ @12 :OperationCancelTunnelQ; + # #[cfg(feature="unstable-tunnels")] + # startTunnelQ @10 :OperationStartTunnelQ; + # completeTunnelQ @11 :OperationCompleteTunnelQ; + # cancelTunnelQ @12 :OperationCancelTunnelQ; } } @@ -537,9 +538,10 @@ struct Answer @0xacacb8b6988c1058 { findBlockA @7 :OperationFindBlockA; # Tunnel operations - startTunnelA @8 :OperationStartTunnelA; - completeTunnelA @9 :OperationCompleteTunnelA; - cancelTunnelA @10 :OperationCancelTunnelA; + # #[cfg(feature="unstable-tunnels")] + # startTunnelA @8 :OperationStartTunnelA; + # completeTunnelA @9 :OperationCompleteTunnelA; + # cancelTunnelA @10 :OperationCancelTunnelA; } } diff --git a/veilid-core/src/crypto/mod.rs b/veilid-core/src/crypto/mod.rs index 822687eb..966633dd 100644 --- a/veilid-core/src/crypto/mod.rs +++ b/veilid-core/src/crypto/mod.rs @@ -25,11 +25,10 @@ pub use none::*; #[cfg(feature = "enable-crypto-vld0")] pub use vld0::*; -use crate::*; +use super::*; use core::convert::TryInto; use hashlink::linked_hash_map::Entry; use hashlink::LruCache; -use serde::{Deserialize, Serialize}; /// Handle to a particular cryptosystem pub type CryptoSystemVersion = Arc; diff --git a/veilid-core/src/lib.rs b/veilid-core/src/lib.rs index bca8a6fe..ff7a311d 100644 --- a/veilid-core/src/lib.rs +++ b/veilid-core/src/lib.rs @@ -47,6 +47,7 @@ use rkyv::{ Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, }; type RkyvDefaultValidator<'t> = rkyv::validation::validators::DefaultValidator<'t>; +use schemars::{schema_for, JsonSchema}; use serde::*; pub mod veilid_capnp { diff --git a/veilid-core/src/rpc_processor/coders/mod.rs b/veilid-core/src/rpc_processor/coders/mod.rs index 34bc7913..4f76486a 100644 --- a/veilid-core/src/rpc_processor/coders/mod.rs +++ b/veilid-core/src/rpc_processor/coders/mod.rs @@ -22,6 +22,7 @@ mod signed_relayed_node_info; mod signed_value_data; mod signed_value_descriptor; mod socket_address; +#[cfg(feature = "unstable-tunnels")] mod tunnel; mod typed_key; mod typed_signature; @@ -50,6 +51,7 @@ pub use signed_relayed_node_info::*; pub use signed_value_data::*; pub use signed_value_descriptor::*; pub use socket_address::*; +#[cfg(feature = "unstable-tunnels")] pub use tunnel::*; pub use typed_key::*; pub use typed_signature::*; diff --git a/veilid-core/src/rpc_processor/coders/operations/answer.rs b/veilid-core/src/rpc_processor/coders/operations/answer.rs index 9f01da68..c1aeb4c4 100644 --- a/veilid-core/src/rpc_processor/coders/operations/answer.rs +++ b/veilid-core/src/rpc_processor/coders/operations/answer.rs @@ -39,8 +39,11 @@ pub enum RPCAnswerDetail { WatchValueA(RPCOperationWatchValueA), SupplyBlockA(RPCOperationSupplyBlockA), FindBlockA(RPCOperationFindBlockA), + #[cfg(feature = "unstable-tunnels")] StartTunnelA(RPCOperationStartTunnelA), + #[cfg(feature = "unstable-tunnels")] CompleteTunnelA(RPCOperationCompleteTunnelA), + #[cfg(feature = "unstable-tunnels")] CancelTunnelA(RPCOperationCancelTunnelA), } @@ -55,8 +58,11 @@ impl RPCAnswerDetail { RPCAnswerDetail::WatchValueA(_) => "WatchValueA", RPCAnswerDetail::SupplyBlockA(_) => "SupplyBlockA", RPCAnswerDetail::FindBlockA(_) => "FindBlockA", + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::StartTunnelA(_) => "StartTunnelA", + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::CompleteTunnelA(_) => "CompleteTunnelA", + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::CancelTunnelA(_) => "CancelTunnelA", } } @@ -70,8 +76,11 @@ impl RPCAnswerDetail { RPCAnswerDetail::WatchValueA(r) => r.validate(validate_context), RPCAnswerDetail::SupplyBlockA(r) => r.validate(validate_context), RPCAnswerDetail::FindBlockA(r) => r.validate(validate_context), + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::StartTunnelA(r) => r.validate(validate_context), + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::CompleteTunnelA(r) => r.validate(validate_context), + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::CancelTunnelA(r) => r.validate(validate_context), } } @@ -120,16 +129,19 @@ impl RPCAnswerDetail { let out = RPCOperationFindBlockA::decode(&op_reader)?; RPCAnswerDetail::FindBlockA(out) } + #[cfg(feature = "unstable-tunnels")] veilid_capnp::answer::detail::StartTunnelA(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationStartTunnelA::decode(&op_reader)?; RPCAnswerDetail::StartTunnelA(out) } + #[cfg(feature = "unstable-tunnels")] veilid_capnp::answer::detail::CompleteTunnelA(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationCompleteTunnelA::decode(&op_reader)?; RPCAnswerDetail::CompleteTunnelA(out) } + #[cfg(feature = "unstable-tunnels")] veilid_capnp::answer::detail::CancelTunnelA(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationCancelTunnelA::decode(&op_reader)?; @@ -155,12 +167,15 @@ impl RPCAnswerDetail { d.encode(&mut builder.reborrow().init_supply_block_a()) } RPCAnswerDetail::FindBlockA(d) => d.encode(&mut builder.reborrow().init_find_block_a()), + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::StartTunnelA(d) => { d.encode(&mut builder.reborrow().init_start_tunnel_a()) } + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::CompleteTunnelA(d) => { d.encode(&mut builder.reborrow().init_complete_tunnel_a()) } + #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::CancelTunnelA(d) => { d.encode(&mut builder.reborrow().init_cancel_tunnel_a()) } diff --git a/veilid-core/src/rpc_processor/coders/operations/mod.rs b/veilid-core/src/rpc_processor/coders/operations/mod.rs index 7caf0fb2..db390b9f 100644 --- a/veilid-core/src/rpc_processor/coders/operations/mod.rs +++ b/veilid-core/src/rpc_processor/coders/operations/mod.rs @@ -2,8 +2,6 @@ mod answer; mod operation; mod operation_app_call; mod operation_app_message; -mod operation_cancel_tunnel; -mod operation_complete_tunnel; mod operation_find_block; mod operation_find_node; mod operation_get_value; @@ -11,7 +9,6 @@ mod operation_return_receipt; mod operation_route; mod operation_set_value; mod operation_signal; -mod operation_start_tunnel; mod operation_status; mod operation_supply_block; mod operation_validate_dial_info; @@ -21,12 +18,17 @@ mod question; mod respond_to; mod statement; +#[cfg(feature = "unstable-tunnels")] +mod operation_cancel_tunnel; +#[cfg(feature = "unstable-tunnels")] +mod operation_complete_tunnel; +#[cfg(feature = "unstable-tunnels")] +mod operation_start_tunnel; + pub use answer::*; pub use operation::*; pub use operation_app_call::*; pub use operation_app_message::*; -pub use operation_cancel_tunnel::*; -pub use operation_complete_tunnel::*; pub use operation_find_block::*; pub use operation_find_node::*; pub use operation_get_value::*; @@ -34,7 +36,6 @@ pub use operation_return_receipt::*; pub use operation_route::*; pub use operation_set_value::*; pub use operation_signal::*; -pub use operation_start_tunnel::*; pub use operation_status::*; pub use operation_supply_block::*; pub use operation_validate_dial_info::*; @@ -44,4 +45,11 @@ pub use question::*; pub use respond_to::*; pub use statement::*; +#[cfg(feature = "unstable-tunnels")] +pub use operation_cancel_tunnel::*; +#[cfg(feature = "unstable-tunnels")] +pub use operation_complete_tunnel::*; +#[cfg(feature = "unstable-tunnels")] +pub use operation_start_tunnel::*; + use super::*; diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs b/veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs index 4bb6ac06..c7254421 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_cancel_tunnel.rs @@ -1,5 +1,6 @@ use super::*; +#[cfg(feature = "unstable-tunnels")] #[derive(Debug, Clone)] pub struct RPCOperationCancelTunnelQ { id: TunnelId, @@ -37,6 +38,7 @@ impl RPCOperationCancelTunnelQ { } } +#[cfg(feature = "unstable-tunnels")] #[derive(Debug, Clone)] pub enum RPCOperationCancelTunnelA { Tunnel(TunnelId), diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs b/veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs index 46b0258a..f699d613 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_complete_tunnel.rs @@ -1,5 +1,6 @@ use super::*; +#[cfg(feature = "unstable-tunnels")] #[derive(Debug, Clone)] pub struct RPCOperationCompleteTunnelQ { id: TunnelId, @@ -74,6 +75,7 @@ impl RPCOperationCompleteTunnelQ { } } +#[cfg(feature = "unstable-tunnels")] #[derive(Debug, Clone)] pub enum RPCOperationCompleteTunnelA { Tunnel(FullTunnel), diff --git a/veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs b/veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs index b3741462..5df5d78a 100644 --- a/veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs +++ b/veilid-core/src/rpc_processor/coders/operations/operation_start_tunnel.rs @@ -1,5 +1,6 @@ use super::*; +#[cfg(feature = "unstable-tunnels")] #[derive(Debug, Clone)] pub struct RPCOperationStartTunnelQ { id: TunnelId, @@ -64,6 +65,7 @@ impl RPCOperationStartTunnelQ { } } +#[cfg(feature = "unstable-tunnels")] #[derive(Debug, Clone)] pub enum RPCOperationStartTunnelA { Partial(PartialTunnel), diff --git a/veilid-core/src/rpc_processor/coders/operations/question.rs b/veilid-core/src/rpc_processor/coders/operations/question.rs index f0f2c056..03debcb5 100644 --- a/veilid-core/src/rpc_processor/coders/operations/question.rs +++ b/veilid-core/src/rpc_processor/coders/operations/question.rs @@ -51,8 +51,11 @@ pub enum RPCQuestionDetail { WatchValueQ(RPCOperationWatchValueQ), SupplyBlockQ(RPCOperationSupplyBlockQ), FindBlockQ(RPCOperationFindBlockQ), + #[cfg(feature = "unstable-tunnels")] StartTunnelQ(RPCOperationStartTunnelQ), + #[cfg(feature = "unstable-tunnels")] CompleteTunnelQ(RPCOperationCompleteTunnelQ), + #[cfg(feature = "unstable-tunnels")] CancelTunnelQ(RPCOperationCancelTunnelQ), } @@ -67,8 +70,11 @@ impl RPCQuestionDetail { RPCQuestionDetail::WatchValueQ(_) => "WatchValueQ", RPCQuestionDetail::SupplyBlockQ(_) => "SupplyBlockQ", RPCQuestionDetail::FindBlockQ(_) => "FindBlockQ", + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::StartTunnelQ(_) => "StartTunnelQ", + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::CompleteTunnelQ(_) => "CompleteTunnelQ", + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::CancelTunnelQ(_) => "CancelTunnelQ", } } @@ -82,8 +88,11 @@ impl RPCQuestionDetail { RPCQuestionDetail::WatchValueQ(r) => r.validate(validate_context), RPCQuestionDetail::SupplyBlockQ(r) => r.validate(validate_context), RPCQuestionDetail::FindBlockQ(r) => r.validate(validate_context), + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::StartTunnelQ(r) => r.validate(validate_context), + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::CompleteTunnelQ(r) => r.validate(validate_context), + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::CancelTunnelQ(r) => r.validate(validate_context), } } @@ -133,16 +142,19 @@ impl RPCQuestionDetail { let out = RPCOperationFindBlockQ::decode(&op_reader)?; RPCQuestionDetail::FindBlockQ(out) } + #[cfg(feature = "unstable-tunnels")] veilid_capnp::question::detail::StartTunnelQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationStartTunnelQ::decode(&op_reader)?; RPCQuestionDetail::StartTunnelQ(out) } + #[cfg(feature = "unstable-tunnels")] veilid_capnp::question::detail::CompleteTunnelQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationCompleteTunnelQ::decode(&op_reader)?; RPCQuestionDetail::CompleteTunnelQ(out) } + #[cfg(feature = "unstable-tunnels")] veilid_capnp::question::detail::CancelTunnelQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationCancelTunnelQ::decode(&op_reader)?; @@ -170,12 +182,15 @@ impl RPCQuestionDetail { RPCQuestionDetail::FindBlockQ(d) => { d.encode(&mut builder.reborrow().init_find_block_q()) } + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::StartTunnelQ(d) => { d.encode(&mut builder.reborrow().init_start_tunnel_q()) } + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::CompleteTunnelQ(d) => { d.encode(&mut builder.reborrow().init_complete_tunnel_q()) } + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::CancelTunnelQ(d) => { d.encode(&mut builder.reborrow().init_cancel_tunnel_q()) } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index c526c77f..d76d31fe 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -4,8 +4,6 @@ mod fanout_call; mod operation_waiter; mod rpc_app_call; mod rpc_app_message; -mod rpc_cancel_tunnel; -mod rpc_complete_tunnel; mod rpc_error; mod rpc_find_block; mod rpc_find_node; @@ -14,13 +12,19 @@ mod rpc_return_receipt; mod rpc_route; mod rpc_set_value; mod rpc_signal; -mod rpc_start_tunnel; mod rpc_status; mod rpc_supply_block; mod rpc_validate_dial_info; mod rpc_value_changed; mod rpc_watch_value; +#[cfg(feature = "unstable-tunnels")] +mod rpc_cancel_tunnel; +#[cfg(feature = "unstable-tunnels")] +mod rpc_complete_tunnel; +#[cfg(feature = "unstable-tunnels")] +mod rpc_start_tunnel; + pub use coders::*; pub use destination::*; pub use fanout_call::*; @@ -1410,8 +1414,11 @@ impl RPCProcessor { RPCQuestionDetail::WatchValueQ(_) => self.process_watch_value_q(msg).await, RPCQuestionDetail::SupplyBlockQ(_) => self.process_supply_block_q(msg).await, RPCQuestionDetail::FindBlockQ(_) => self.process_find_block_q(msg).await, + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::StartTunnelQ(_) => self.process_start_tunnel_q(msg).await, + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::CompleteTunnelQ(_) => self.process_complete_tunnel_q(msg).await, + #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::CancelTunnelQ(_) => self.process_cancel_tunnel_q(msg).await, }, RPCOperationKind::Statement(s) => match s.detail() { diff --git a/veilid-core/src/veilid_api/api.rs b/veilid-core/src/veilid_api/api.rs index ab40f11f..59a9f2db 100644 --- a/veilid-core/src/veilid_api/api.rs +++ b/veilid-core/src/veilid_api/api.rs @@ -268,6 +268,7 @@ impl VeilidAPI { //////////////////////////////////////////////////////////////// // Tunnel Building + #[cfg(feature = "unstable-tunnels")] #[instrument(level = "debug", err, skip(self))] pub async fn start_tunnel( &self, @@ -277,6 +278,7 @@ impl VeilidAPI { panic!("unimplemented"); } + #[cfg(feature = "unstable-tunnels")] #[instrument(level = "debug", err, skip(self))] pub async fn complete_tunnel( &self, @@ -287,6 +289,7 @@ impl VeilidAPI { panic!("unimplemented"); } + #[cfg(feature = "unstable-tunnels")] #[instrument(level = "debug", err, skip(self))] pub async fn cancel_tunnel(&self, _tunnel_id: TunnelId) -> VeilidAPIResult { panic!("unimplemented"); diff --git a/veilid-core/src/veilid_api/error.rs b/veilid-core/src/veilid_api/error.rs index 778b4830..0eb7b791 100644 --- a/veilid-core/src/veilid_api/error.rs +++ b/veilid-core/src/veilid_api/error.rs @@ -117,6 +117,7 @@ macro_rules! apibail_already_initialized { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] #[serde(tag = "kind")] @@ -136,7 +137,10 @@ pub enum VeilidAPIError { #[error("No connection: {message}")] NoConnection { message: String }, #[error("Key not found: {key}")] - KeyNotFound { key: TypedKey }, + KeyNotFound { + #[schemars(with="String")] + key: TypedKey + }, #[error("Internal: {message}")] Internal { message: String }, #[error("Unimplemented: {message}")] diff --git a/veilid-core/src/veilid_api/json_api/crypto_system.rs b/veilid-core/src/veilid_api/json_api/crypto_system.rs new file mode 100644 index 00000000..73f08923 --- /dev/null +++ b/veilid-core/src/veilid_api/json_api/crypto_system.rs @@ -0,0 +1,66 @@ +use super::*; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct CryptoSystemRequest { + cs_id: String, + #[serde(flatten)] + cs_op: CryptoSystemRequestOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct CryptoSystemResponse { + cs_id: String, + #[serde(flatten)] + cs_op: CryptoSystemResponseOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "cs_op")] +pub enum CryptoSystemRequestOp { + Release, + CachedDh, + ComputeDh, + RandomBytes, + DefaultSaltLength, + HashPassword, + VerifyPassword, + DeriveSharedSecret, + RandomNonce, + RandomSharedSecret, + GenerateKeyPair, + GenerateHash, + ValidateKeyPair, + ValidateHash, + Distance, + Sign, + Verify, + AeadOverhead, + DecryptAead, + EncryptAead, + CryptNoAuth, +} +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "cs_op")] +pub enum CryptoSystemResponseOp { + Release, + CachedDh, + ComputeDh, + RandomBytes, + DefaultSaltLength, + HashPassword, + VerifyPassword, + DeriveSharedSecret, + RandomNonce, + RandomSharedSecret, + GenerateKeyPair, + GenerateHash, + ValidateKeyPair, + ValidateHash, + Distance, + Sign, + Verify, + AeadOverhead, + DecryptAead, + EncryptAead, + CryptNoAuth, +} diff --git a/veilid-core/src/veilid_api/json_api/mod.rs b/veilid-core/src/veilid_api/json_api/mod.rs new file mode 100644 index 00000000..5fb4b07b --- /dev/null +++ b/veilid-core/src/veilid_api/json_api/mod.rs @@ -0,0 +1,203 @@ +use super::*; + +mod routing_context; +pub use routing_context::*; + +mod table_db; +pub use table_db::*; + +mod crypto_system; +pub use crypto_system::*; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct Request { + /// Operation Id (pairs with Response) + id: String, + /// The request operation variant + #[serde(flatten)] + op: RequestOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct Response { + /// Operation Id (pairs with Request) + id: String, + /// The response operation variant + #[serde(flatten)] + op: ResponseOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "op")] +pub enum RequestOp { + GetState, + Attach, + Detach, + NewPrivateRoute, + NewCustomPrivateRoute { + #[schemars(with = "Vec")] + crypto_kinds: Vec, + #[serde(default)] + stability: Stability, + #[serde(default)] + sequencing: Sequencing, + }, + ImportRemotePrivateRoute { + #[serde( + serialize_with = "json_as_base64::serialize", + deserialize_with = "json_as_base64::deserialize" + )] + #[schemars(with = "String")] + blob: Vec, + }, + ReleasePrivateRoute { + #[schemars(with = "String")] + route_id: RouteId, + }, + AppCallReply { + #[schemars(with = "String")] + call_id: OperationId, + #[serde( + serialize_with = "json_as_base64::serialize", + deserialize_with = "json_as_base64::deserialize" + )] + #[schemars(with = "String")] + message: Vec, + }, + // Routing Context + NewRoutingContext, + RoutingContext(RoutingContextRequest), + // TableDb + OpenTableDb { + name: String, + column_count: u32, + }, + DeleteTableDb { + name: String, + }, + TableDb(TableDbRequest), + // Crypto + GetCryptoSystem { + #[schemars(with = "String")] + crypto_kind: CryptoKind, + }, + BestCryptoSystem, + CryptoSystem(CryptoSystemRequest), + VerifySignatures { + #[schemars(with = "Vec")] + node_ids: Vec, + #[serde( + serialize_with = "json_as_base64::serialize", + deserialize_with = "json_as_base64::deserialize" + )] + #[schemars(with = "String")] + data: Vec, + #[schemars(with = "Vec")] + signatures: Vec, + }, + GenerateSignatures { + #[serde( + serialize_with = "json_as_base64::serialize", + deserialize_with = "json_as_base64::deserialize" + )] + #[schemars(with = "String")] + data: Vec, + #[schemars(with = "Vec")] + key_pairs: Vec, + }, + GenerateKeyPair { + #[schemars(with = "String")] + crypto_kind: CryptoKind, + }, + // Misc + Now, + Debug { + command: String, + }, + VeilidVersionString, + VeilidVersion, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct NewPrivateRouteResult { + #[schemars(with = "String")] + route_id: RouteId, + #[serde( + serialize_with = "json_as_base64::serialize", + deserialize_with = "json_as_base64::deserialize" + )] + #[schemars(with = "String")] + blob: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "op")] +pub enum ResponseOp { + GetState { + #[serde(flatten)] + result: ApiResult, + }, + Attach { + #[serde(flatten)] + result: ApiResult<()>, + }, + Detach { + #[serde(flatten)] + result: ApiResult<()>, + }, + NewPrivateRoute { + #[serde(flatten)] + result: ApiResult, + }, + NewCustomPrivateRoute { + #[serde(flatten)] + result: ApiResult, + }, + ImportRemotePrivateRoute, + ReleasePrivateRoute, + AppCallReply, + // Routing Context + NewRoutingContext, + RoutingContext(RoutingContextResponse), + // TableDb + OpenTableDb, + DeleteTableDb, + TableDb(TableDbResponse), + // Crypto + GetCryptoSystem, + BestCryptoSystem, + CryptoSystem(CryptoSystemResponse), + VerifySignatures, + GenerateSignatures, + GenerateKeyPair, + // Misc + Now, + Debug, + VeilidVersionString, + VeilidVersion, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum ApiResult +where + T: Clone + fmt::Debug + JsonSchema, +{ + Ok { value: T }, + Err { error: VeilidAPIError }, +} + +pub fn emit_schemas(out: &mut HashMap) { + let schema_request = schema_for!(Request); + let schema_response = schema_for!(Response); + + out.insert( + "Request".to_owned(), + serde_json::to_string_pretty(&schema_request).unwrap(), + ); + + out.insert( + "Response".to_owned(), + serde_json::to_string_pretty(&schema_response).unwrap(), + ); +} diff --git a/veilid-core/src/veilid_api/json_api/routing_context.rs b/veilid-core/src/veilid_api/json_api/routing_context.rs new file mode 100644 index 00000000..c4a9405c --- /dev/null +++ b/veilid-core/src/veilid_api/json_api/routing_context.rs @@ -0,0 +1,52 @@ +use super::*; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct RoutingContextRequest { + rc_id: String, + #[serde(flatten)] + rc_op: RoutingContextRequestOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct RoutingContextResponse { + rc_id: String, + #[serde(flatten)] + rc_op: RoutingContextResponseOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "rc_op")] +pub enum RoutingContextRequestOp { + Release, + WithPrivacy, + WithCustomPrivacy, + WithSequencing, + AppCall, + AppMessage, + CreateDhtRecord, + OpenDhtRecord, + CloseDhtRecord, + DeleteDhtRecord, + GetDhtValue, + SetDhtValue, + WatchDhtValues, + CancelDhtWatch, +} +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "rc_op")] +pub enum RoutingContextResponseOp { + Release, + WithPrivacy, + WithCustomPrivacy, + WithSequencing, + AppCall, + AppMessage, + CreateDhtRecord, + OpenDhtRecord, + CloseDhtRecord, + DeleteDhtRecord, + GetDhtValue, + SetDhtValue, + WatchDhtValues, + CancelDhtWatch, +} diff --git a/veilid-core/src/veilid_api/json_api/table_db.rs b/veilid-core/src/veilid_api/json_api/table_db.rs new file mode 100644 index 00000000..62190e83 --- /dev/null +++ b/veilid-core/src/veilid_api/json_api/table_db.rs @@ -0,0 +1,26 @@ +use super::*; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct TableDbRequest { + db_id: String, + #[serde(flatten)] + db_op: TableDbRequestOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct TableDbResponse { + db_id: String, + #[serde(flatten)] + db_op: TableDbResponseOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "db_op")] +pub enum TableDbRequestOp { + Release, +} +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "db_op")] +pub enum TableDbResponseOp { + Release, +} diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index d046b71f..50afee6f 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -3,6 +3,7 @@ mod api; mod debug; mod error; +mod json_api; mod routing_context; mod serialize_helpers; mod types; @@ -12,6 +13,7 @@ pub mod tests; pub use api::*; pub use debug::*; pub use error::*; +pub use json_api::*; pub use routing_context::*; pub use serialize_helpers::*; pub use types::*; diff --git a/veilid-core/src/veilid_api/serialize_helpers/serialize_range_set_blaze.rs b/veilid-core/src/veilid_api/serialize_helpers/serialize_range_set_blaze.rs index 5ef5fa46..2b98a3e3 100644 --- a/veilid-core/src/veilid_api/serialize_helpers/serialize_range_set_blaze.rs +++ b/veilid-core/src/veilid_api/serialize_helpers/serialize_range_set_blaze.rs @@ -9,11 +9,10 @@ pub fn serialize( v: &RangeSetBlaze, s: S, ) -> Result { - let cnt = v.ranges_len() * 2; + let cnt = v.ranges_len(); let mut seq = s.serialize_seq(Some(cnt))?; for range in v.ranges() { - seq.serialize_element(range.start())?; - seq.serialize_element(range.end())?; + seq.serialize_element(&(range.start(), range.end()))?; } seq.end() } @@ -41,10 +40,7 @@ pub fn deserialize<'de, T: Integer + Deserialize<'de>, D: Deserializer<'de>>( { let mut values = RangeSetBlaze::::new(); - while let Some(start) = seq.next_element()? { - let Some(end) = seq.next_element()? else { - break; - }; + while let Some((start, end)) = seq.next_element()? { values.ranges_insert(start..=end); } diff --git a/veilid-core/src/veilid_api/types/aligned_u64.rs b/veilid-core/src/veilid_api/types/aligned_u64.rs index 6a64b86d..11ee022d 100644 --- a/veilid-core/src/veilid_api/types/aligned_u64.rs +++ b/veilid-core/src/veilid_api/types/aligned_u64.rs @@ -18,10 +18,19 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[repr(C, align(8))] #[archive_attr(repr(C, align(8)), derive(CheckBytes))] -pub struct AlignedU64(u64); +#[serde(transparent)] +pub struct AlignedU64( + #[serde( + serialize_with = "json_as_string::serialize", + deserialize_with = "json_as_string::deserialize" + )] + #[schemars(with = "String")] + u64, +); impl From for AlignedU64 { fn from(v: u64) -> Self { diff --git a/veilid-core/src/veilid_api/types/app_message_call.rs b/veilid-core/src/veilid_api/types/app_message_call.rs index 6f0805fe..2f3c77be 100644 --- a/veilid-core/src/veilid_api/types/app_message_call.rs +++ b/veilid-core/src/veilid_api/types/app_message_call.rs @@ -2,15 +2,33 @@ use super::*; /// Direct statement blob passed to hosting application for processing #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidAppMessage { /// Some(sender) if the message was sent directly, None if received via a private/safety route - #[serde(with = "opt_json_as_string")] + #[serde( + serialize_with = "opt_json_as_string::serialize", + deserialize_with = "opt_json_as_string::deserialize" + )] + #[schemars(with = "Option")] sender: Option, + /// The content of the message to deliver to the application - #[serde(with = "json_as_base64")] + #[serde( + serialize_with = "json_as_base64::serialize", + deserialize_with = "json_as_base64::deserialize" + )] + #[schemars(with = "String")] message: Vec, } @@ -29,18 +47,41 @@ impl VeilidAppMessage { /// Direct question blob passed to hosting application for processing to send an eventual AppReply #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidAppCall { /// Some(sender) if the request was sent directly, None if received via a private/safety route - #[serde(with = "opt_json_as_string")] + #[serde( + serialize_with = "opt_json_as_string::serialize", + deserialize_with = "opt_json_as_string::deserialize" + )] + #[schemars(with = "Option")] sender: Option, + /// The content of the request to deliver to the application - #[serde(with = "json_as_base64")] + #[serde( + serialize_with = "json_as_base64::serialize", + deserialize_with = "json_as_base64::deserialize" + )] + #[schemars(with = "String")] message: Vec, + /// The id to reply to - #[serde(with = "json_as_string")] + #[serde( + serialize_with = "json_as_string::serialize", + deserialize_with = "json_as_string::deserialize" + )] + #[schemars(with = "String")] id: OperationId, } diff --git a/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs b/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs index 1c4d114e..6162fd9f 100644 --- a/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs +++ b/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs @@ -13,15 +13,19 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct DHTRecordDescriptor { /// DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ] + #[schemars(with = "String")] key: TypedKey, /// The public key of the owner + #[schemars(with = "String")] owner: PublicKey, /// If this key is being created: Some(the secret key of the owner) /// If this key is just being opened: None + #[schemars(with = "Option")] owner_secret: Option, /// The schema in use associated with the key schema: DHTSchema, diff --git a/veilid-core/src/veilid_api/types/dht/schema/dflt.rs b/veilid-core/src/veilid_api/types/dht/schema/dflt.rs index bd9c0858..a742bdec 100644 --- a/veilid-core/src/veilid_api/types/dht/schema/dflt.rs +++ b/veilid-core/src/veilid_api/types/dht/schema/dflt.rs @@ -13,6 +13,7 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct DHTSchemaDFLT { diff --git a/veilid-core/src/veilid_api/types/dht/schema/mod.rs b/veilid-core/src/veilid_api/types/dht/schema/mod.rs index b043afcc..2361e32c 100644 --- a/veilid-core/src/veilid_api/types/dht/schema/mod.rs +++ b/veilid-core/src/veilid_api/types/dht/schema/mod.rs @@ -19,6 +19,7 @@ pub use smpl::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] #[serde(tag = "kind")] diff --git a/veilid-core/src/veilid_api/types/dht/schema/smpl.rs b/veilid-core/src/veilid_api/types/dht/schema/smpl.rs index 90b20b86..9b2c7df1 100644 --- a/veilid-core/src/veilid_api/types/dht/schema/smpl.rs +++ b/veilid-core/src/veilid_api/types/dht/schema/smpl.rs @@ -13,10 +13,12 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct DHTSchemaSMPLMember { /// Member key + #[schemars(with = "String")] pub m_key: PublicKey, /// Member subkey count pub m_cnt: u16, @@ -35,6 +37,7 @@ pub struct DHTSchemaSMPLMember { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct DHTSchemaSMPL { diff --git a/veilid-core/src/veilid_api/types/dht/value_data.rs b/veilid-core/src/veilid_api/types/dht/value_data.rs index 734e735d..eae2e2c4 100644 --- a/veilid-core/src/veilid_api/types/dht/value_data.rs +++ b/veilid-core/src/veilid_api/types/dht/value_data.rs @@ -13,11 +13,23 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct ValueData { + /// An increasing sequence number to time-order the DHT record changes seq: ValueSeqNum, + + /// The contents of a DHT Record + #[serde( + serialize_with = "json_as_base64::serialize", + deserialize_with = "json_as_base64::deserialize" + )] + #[schemars(with = "String")] data: Vec, + + /// The public identity key of the writer of the data + #[schemars(with = "String")] writer: PublicKey, } impl ValueData { diff --git a/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs b/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs index 3dd40f67..955c9d01 100644 --- a/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs +++ b/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs @@ -15,11 +15,16 @@ use range_set_blaze::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct ValueSubkeyRangeSet { #[with(RkyvRangeSetBlaze)] - #[serde(with = "serialize_range_set_blaze")] + #[serde( + serialize_with = "serialize_range_set_blaze::serialize", + deserialize_with = "serialize_range_set_blaze::deserialize" + )] + #[schemars(with = "Vec<(u32,u32)>")] data: RangeSetBlaze, } diff --git a/veilid-core/src/veilid_api/types/fourcc.rs b/veilid-core/src/veilid_api/types/fourcc.rs index edce2b03..1d56da33 100644 --- a/veilid-core/src/veilid_api/types/fourcc.rs +++ b/veilid-core/src/veilid_api/types/fourcc.rs @@ -15,8 +15,11 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes, PartialOrd, Ord, PartialEq, Eq, Hash))] +#[serde(try_from = "String")] +#[serde(into = "String")] pub struct FourCC(pub [u8; 4]); impl From<[u8; 4]> for FourCC { @@ -37,6 +40,12 @@ impl From for u32 { } } +impl From for String { + fn from(u: FourCC) -> Self { + String::from_utf8_lossy(&u.0).to_string() + } +} + impl TryFrom<&[u8]> for FourCC { type Error = VeilidAPIError; fn try_from(b: &[u8]) -> Result { @@ -44,6 +53,13 @@ impl TryFrom<&[u8]> for FourCC { } } +impl TryFrom for FourCC { + type Error = VeilidAPIError; + fn try_from(s: String) -> Result { + Self::from_str(s.as_str()) + } +} + impl fmt::Display for FourCC { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { write!(f, "{}", String::from_utf8_lossy(&self.0)) diff --git a/veilid-core/src/veilid_api/types/mod.rs b/veilid-core/src/veilid_api/types/mod.rs index 6dea3db5..d3c2d9dc 100644 --- a/veilid-core/src/veilid_api/types/mod.rs +++ b/veilid-core/src/veilid_api/types/mod.rs @@ -4,6 +4,7 @@ mod dht; mod fourcc; mod safety; mod stats; +#[cfg(feature = "unstable-tunnels")] mod tunnel; mod veilid_log; mod veilid_state; @@ -16,6 +17,7 @@ pub use dht::*; pub use fourcc::*; pub use safety::*; pub use stats::*; +#[cfg(feature = "unstable-tunnels")] pub use tunnel::*; pub use veilid_log::*; pub use veilid_state::*; diff --git a/veilid-core/src/veilid_api/types/safety.rs b/veilid-core/src/veilid_api/types/safety.rs index 27cb46ba..970619c2 100644 --- a/veilid-core/src/veilid_api/types/safety.rs +++ b/veilid-core/src/veilid_api/types/safety.rs @@ -15,6 +15,7 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] pub enum Sequencing { @@ -44,6 +45,7 @@ impl Default for Sequencing { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] pub enum Stability { @@ -72,6 +74,7 @@ impl Default for Stability { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] pub enum SafetySelection { @@ -111,10 +114,12 @@ impl Default for SafetySelection { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct SafetySpec { /// preferred safety route set id if it still exists + #[schemars(with = "Option")] pub preferred_route: Option, /// must be greater than 0 pub hop_count: usize, diff --git a/veilid-core/src/veilid_api/types/stats.rs b/veilid-core/src/veilid_api/types/stats.rs index 8ea79e11..a7bbce14 100644 --- a/veilid-core/src/veilid_api/types/stats.rs +++ b/veilid-core/src/veilid_api/types/stats.rs @@ -11,14 +11,12 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct LatencyStats { - #[serde(with = "json_as_string")] pub fastest: TimestampDuration, // fastest latency in the ROLLING_LATENCIES_SIZE last latencies - #[serde(with = "json_as_string")] pub average: TimestampDuration, // average latency over the ROLLING_LATENCIES_SIZE last latencies - #[serde(with = "json_as_string")] pub slowest: TimestampDuration, // slowest latency in the ROLLING_LATENCIES_SIZE last latencies } @@ -33,16 +31,13 @@ pub struct LatencyStats { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct TransferStats { - #[serde(with = "json_as_string")] - pub total: ByteCount, // total amount transferred ever - #[serde(with = "json_as_string")] + pub total: ByteCount, // total amount transferred ever pub maximum: ByteCount, // maximum rate over the ROLLING_TRANSFERS_SIZE last amounts - #[serde(with = "json_as_string")] pub average: ByteCount, // average rate over the ROLLING_TRANSFERS_SIZE last amounts - #[serde(with = "json_as_string")] pub minimum: ByteCount, // minimum rate over the ROLLING_TRANSFERS_SIZE last amounts } @@ -57,6 +52,7 @@ pub struct TransferStats { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct TransferStatsDownUp { @@ -75,17 +71,15 @@ pub struct TransferStatsDownUp { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct RPCStats { pub messages_sent: u32, // number of rpcs that have been sent in the total_time range pub messages_rcvd: u32, // number of rpcs that have been received in the total_time range pub questions_in_flight: u32, // number of questions issued that have yet to be answered - #[serde(with = "opt_json_as_string")] pub last_question_ts: Option, // when the peer was last questioned (either successfully or not) and we wanted an answer - #[serde(with = "opt_json_as_string")] pub last_seen_ts: Option, // when the peer was last seen for any reason, including when we first attempted to reach out to it - #[serde(with = "opt_json_as_string")] pub first_consecutive_seen_ts: Option, // the timestamp of the first consecutive proof-of-life for this node (an answer or received question) pub recent_lost_answers: u32, // number of answers that have been lost since we lost reliability pub failed_to_send: u32, // number of messages that have failed to send since we last successfully sent one @@ -102,12 +96,12 @@ pub struct RPCStats { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct PeerStats { - #[serde(with = "json_as_string")] pub time_added: Timestamp, // when the peer was added to the routing table - pub rpc_stats: RPCStats, // information about RPCs + pub rpc_stats: RPCStats, // information about RPCs pub latency: Option, // latencies for communications with the peer pub transfer: TransferStatsDownUp, // Stats for communications with the peer } diff --git a/veilid-core/src/veilid_api/types/tunnel.rs b/veilid-core/src/veilid_api/types/tunnel.rs index 968c7695..d7dd0326 100644 --- a/veilid-core/src/veilid_api/types/tunnel.rs +++ b/veilid-core/src/veilid_api/types/tunnel.rs @@ -1,8 +1,11 @@ +#[cfg(feature = "unstable-tunnels")] use super::*; /// Tunnel identifier +#[cfg(feature = "unstable-tunnels")] pub type TunnelId = AlignedU64; +#[cfg(feature = "unstable-tunnels")] #[derive( Copy, Clone, @@ -16,6 +19,7 @@ pub type TunnelId = AlignedU64; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] pub enum TunnelMode { @@ -23,6 +27,7 @@ pub enum TunnelMode { Turn, } +#[cfg(feature = "unstable-tunnels")] #[derive( Copy, Clone, @@ -36,6 +41,7 @@ pub enum TunnelMode { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] pub enum TunnelError { @@ -45,13 +51,17 @@ pub enum TunnelError { NoCapacity, // Endpoint is full } -#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[cfg(feature = "unstable-tunnels")] +#[derive( + Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, JsonSchema, +)] #[archive_attr(repr(C), derive(CheckBytes))] pub struct TunnelEndpoint { pub mode: TunnelMode, pub description: String, // XXX: TODO } +#[cfg(feature = "unstable-tunnels")] impl Default for TunnelEndpoint { fn default() -> Self { Self { @@ -61,8 +71,17 @@ impl Default for TunnelEndpoint { } } +#[cfg(feature = "unstable-tunnels")] #[derive( - Clone, Debug, Default, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Clone, + Debug, + Default, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct FullTunnel { @@ -72,8 +91,17 @@ pub struct FullTunnel { pub remote: TunnelEndpoint, } +#[cfg(feature = "unstable-tunnels")] #[derive( - Clone, Debug, Default, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Clone, + Debug, + Default, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct PartialTunnel { diff --git a/veilid-core/src/veilid_api/types/veilid_log.rs b/veilid-core/src/veilid_api/types/veilid_log.rs index 5eab945c..66852e14 100644 --- a/veilid-core/src/veilid_api/types/veilid_log.rs +++ b/veilid-core/src/veilid_api/types/veilid_log.rs @@ -14,6 +14,7 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] pub enum VeilidLogLevel { @@ -78,7 +79,16 @@ impl fmt::Display for VeilidLogLevel { /// A VeilidCore log message with optional backtrace #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidLog { diff --git a/veilid-core/src/veilid_api/types/veilid_state.rs b/veilid-core/src/veilid_api/types/veilid_state.rs index 09f21a24..01902e2a 100644 --- a/veilid-core/src/veilid_api/types/veilid_state.rs +++ b/veilid-core/src/veilid_api/types/veilid_state.rs @@ -12,6 +12,7 @@ use super::*; RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(u8), derive(CheckBytes))] pub enum AttachmentState { @@ -60,7 +61,16 @@ impl TryFrom for AttachmentState { } #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidStateAttachment { @@ -70,39 +80,76 @@ pub struct VeilidStateAttachment { } #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct PeerTableData { + #[schemars(with = "Vec")] pub node_ids: Vec, pub peer_address: String, pub peer_stats: PeerStats, } #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidStateNetwork { pub started: bool, - #[serde(with = "json_as_string")] pub bps_down: ByteCount, - #[serde(with = "json_as_string")] pub bps_up: ByteCount, pub peers: Vec, } #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidRouteChange { + #[schemars(with = "Vec")] pub dead_routes: Vec, + #[schemars(with = "Vec")] pub dead_remote_routes: Vec, } #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidStateConfig { @@ -110,17 +157,29 @@ pub struct VeilidStateConfig { } #[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, + JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidValueChange { + #[schemars(with = "String")] key: TypedKey, subkeys: Vec, count: u32, value: ValueData, } -#[derive(Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[derive( + Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, JsonSchema, +)] #[archive_attr(repr(u8), derive(CheckBytes))] #[serde(tag = "kind")] pub enum VeilidUpdate { @@ -135,7 +194,9 @@ pub enum VeilidUpdate { Shutdown, } -#[derive(Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[derive( + Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, JsonSchema, +)] #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidState { pub attachment: VeilidStateAttachment, diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index 68c06e6f..8559ba47 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -25,6 +25,7 @@ pub type ConfigCallback = Arc ConfigCallbackReturn + Send + Sy RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigHTTPS { pub enabled: bool, @@ -54,6 +55,7 @@ pub struct VeilidConfigHTTPS { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigHTTP { pub enabled: bool, @@ -79,6 +81,7 @@ pub struct VeilidConfigHTTP { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigApplication { pub https: VeilidConfigHTTPS, @@ -106,6 +109,7 @@ pub struct VeilidConfigApplication { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigUDP { pub enabled: bool, @@ -135,6 +139,7 @@ pub struct VeilidConfigUDP { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigTCP { pub connect: bool, @@ -166,6 +171,7 @@ pub struct VeilidConfigTCP { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigWS { pub connect: bool, @@ -198,6 +204,7 @@ pub struct VeilidConfigWS { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigWSS { pub connect: bool, @@ -226,6 +233,7 @@ pub struct VeilidConfigWSS { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigProtocol { pub udp: VeilidConfigUDP, @@ -253,6 +261,7 @@ pub struct VeilidConfigProtocol { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigTLS { pub certificate_path: String, @@ -273,6 +282,7 @@ pub struct VeilidConfigTLS { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigDHT { pub max_find_node_count: u32, @@ -309,6 +319,7 @@ pub struct VeilidConfigDHT { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigRPC { pub concurrency: u32, @@ -333,9 +344,12 @@ pub struct VeilidConfigRPC { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigRoutingTable { + #[schemars(with = "Vec")] pub node_id: TypedKeySet, + #[schemars(with = "Vec")] pub node_id_secret: TypedSecretSet, pub bootstrap: Vec, pub limit_over_attached: u32, @@ -358,6 +372,7 @@ pub struct VeilidConfigRoutingTable { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigNetwork { pub connection_initial_timeout_ms: u32, @@ -391,6 +406,7 @@ pub struct VeilidConfigNetwork { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigTableStore { pub directory: String, @@ -408,6 +424,7 @@ pub struct VeilidConfigTableStore { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigBlockStore { pub directory: String, @@ -425,6 +442,7 @@ pub struct VeilidConfigBlockStore { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigProtectedStore { pub allow_insecure_fallback: bool, @@ -446,6 +464,7 @@ pub struct VeilidConfigProtectedStore { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigCapabilities { pub protocol_udp: bool, @@ -468,6 +487,7 @@ pub struct VeilidConfigCapabilities { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub enum VeilidConfigLogLevel { Off, @@ -537,6 +557,7 @@ impl Default for VeilidConfigLogLevel { RkyvArchive, RkyvSerialize, RkyvDeserialize, + JsonSchema, )] pub struct VeilidConfigInner { pub program_name: String, @@ -737,7 +758,6 @@ impl VeilidConfig { safe_cfg.protected_store.device_encryption_key_password = "".to_owned(); safe_cfg.protected_store.new_device_encryption_key_password = None; - safe_cfg } diff --git a/veilid-server/src/cmdline.rs b/veilid-server/src/cmdline.rs index 3d4fe163..cdc4158a 100644 --- a/veilid-server/src/cmdline.rs +++ b/veilid-server/src/cmdline.rs @@ -132,6 +132,14 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result EyreResult<()> { bail!("missing crypto kind"); } } + // -- Emit JSON-Schema -- + if matches.occurrences_of("emit-schema") != 0 { + if let Some(esstr) = matches.value_of("emit-schema") { + let mut schemas = HashMap::::new(); + veilid_core::emit_schemas(&mut schemas); + + if let Some(schema) = schemas.get(esstr) { + println!("{}", schema); + } else { + println!("Valid schemas:"); + for s in schemas.keys() { + println!(" {}", s); + } + } + + return Ok(()); + } + } // See if we're just running a quick command let (server_mode, success, failure) = if matches.occurrences_of("set-node-id") != 0 { From 06081df22a43be49ba85478c8cedcee061691b87 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 4 Jun 2023 21:22:55 -0400 Subject: [PATCH 02/23] json schema generation --- veilid-core/proto/veilid.capnp | 10 +- veilid-core/src/attachment_manager.rs | 6 +- veilid-core/src/core_context.rs | 28 ++- veilid-core/src/intf/native/mod.rs | 4 + veilid-core/src/intf/wasm/mod.rs | 4 + veilid-core/src/network_manager/mod.rs | 6 + .../src/routing_table/tests/test_serialize.rs | 3 + .../rpc_processor/coders/operations/answer.rs | 10 + .../rpc_processor/coders/operations/mod.rs | 15 +- .../coders/operations/question.rs | 10 + veilid-core/src/rpc_processor/mod.rs | 9 +- veilid-core/src/storage_manager/mod.rs | 7 +- veilid-core/src/veilid_api/api.rs | 1 + .../src/veilid_api/json_api/crypto_system.rs | 231 +++++++++++++++--- veilid-core/src/veilid_api/json_api/mod.rs | 159 +++++++++--- .../veilid_api/json_api/routing_context.rs | 138 +++++++++-- veilid-core/src/veilid_api/mod.rs | 1 + veilid-core/src/veilid_api/routing_context.rs | 2 + .../serialize_helpers/serialize_json.rs | 21 ++ .../src/veilid_api/types/aligned_u64.rs | 5 +- .../src/veilid_api/types/app_message_call.rs | 25 +- .../src/veilid_api/types/dht/value_data.rs | 5 +- .../types/dht/value_subkey_range_set.rs | 6 +- 23 files changed, 552 insertions(+), 154 deletions(-) diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 7e0ba4fb..b7f1a359 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -496,8 +496,9 @@ struct Question @0xd8510bc33492ef70 { getValueQ @5 :OperationGetValueQ; setValueQ @6 :OperationSetValueQ; watchValueQ @7 :OperationWatchValueQ; - supplyBlockQ @8 :OperationSupplyBlockQ; - findBlockQ @9 :OperationFindBlockQ; + # #[cfg(feature="unstable-blockstore")] + # supplyBlockQ @8 :OperationSupplyBlockQ; + # findBlockQ @9 :OperationFindBlockQ; # Tunnel operations # #[cfg(feature="unstable-tunnels")] @@ -534,8 +535,9 @@ struct Answer @0xacacb8b6988c1058 { getValueA @3 :OperationGetValueA; setValueA @4 :OperationSetValueA; watchValueA @5 :OperationWatchValueA; - supplyBlockA @6 :OperationSupplyBlockA; - findBlockA @7 :OperationFindBlockA; + # #[cfg(feature="unstable-blockstore")] + #supplyBlockA @6 :OperationSupplyBlockA; + #findBlockA @7 :OperationFindBlockA; # Tunnel operations # #[cfg(feature="unstable-tunnels")] diff --git a/veilid-core/src/attachment_manager.rs b/veilid-core/src/attachment_manager.rs index fca5650d..a373bea2 100644 --- a/veilid-core/src/attachment_manager.rs +++ b/veilid-core/src/attachment_manager.rs @@ -30,7 +30,7 @@ impl AttachmentManager { storage_manager: StorageManager, protected_store: ProtectedStore, table_store: TableStore, - block_store: BlockStore, + #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, crypto: Crypto, ) -> AttachmentManagerUnlockedInner { AttachmentManagerUnlockedInner { @@ -40,6 +40,7 @@ impl AttachmentManager { storage_manager, protected_store, table_store, + #[cfg(feature = "unstable-blockstore")] block_store, crypto, ), @@ -60,7 +61,7 @@ impl AttachmentManager { storage_manager: StorageManager, protected_store: ProtectedStore, table_store: TableStore, - block_store: BlockStore, + #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, crypto: Crypto, ) -> Self { Self { @@ -70,6 +71,7 @@ impl AttachmentManager { storage_manager, protected_store, table_store, + #[cfg(feature = "unstable-blockstore")] block_store, crypto, )), diff --git a/veilid-core/src/core_context.rs b/veilid-core/src/core_context.rs index 555bdcf2..630f7375 100644 --- a/veilid-core/src/core_context.rs +++ b/veilid-core/src/core_context.rs @@ -17,6 +17,7 @@ struct ServicesContext { pub protected_store: Option, pub table_store: Option, + #[cfg(feature = "unstable-blockstore")] pub block_store: Option, pub crypto: Option, pub attachment_manager: Option, @@ -30,6 +31,7 @@ impl ServicesContext { update_callback, protected_store: None, table_store: None, + #[cfg(feature = "unstable-blockstore")] block_store: None, crypto: None, attachment_manager: None, @@ -42,7 +44,7 @@ impl ServicesContext { update_callback: UpdateCallback, protected_store: ProtectedStore, table_store: TableStore, - block_store: BlockStore, + #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, crypto: Crypto, attachment_manager: AttachmentManager, storage_manager: StorageManager, @@ -52,6 +54,7 @@ impl ServicesContext { update_callback, protected_store: Some(protected_store), table_store: Some(table_store), + #[cfg(feature = "unstable-blockstore")] block_store: Some(block_store), crypto: Some(crypto), attachment_manager: Some(attachment_manager), @@ -103,14 +106,17 @@ impl ServicesContext { self.crypto = Some(crypto.clone()); // Set up block store - trace!("init block store"); - let block_store = BlockStore::new(self.config.clone()); - if let Err(e) = block_store.init().await { - error!("failed to init block store: {}", e); - self.shutdown().await; - return Err(e); + #[cfg(feature = "unstable-blockstore")] + { + trace!("init block store"); + let block_store = BlockStore::new(self.config.clone()); + if let Err(e) = block_store.init().await { + error!("failed to init block store: {}", e); + self.shutdown().await; + return Err(e); + } + self.block_store = Some(block_store.clone()); } - self.block_store = Some(block_store.clone()); // Set up storage manager trace!("init storage manager"); @@ -119,6 +125,7 @@ impl ServicesContext { self.crypto.clone().unwrap(), self.protected_store.clone().unwrap(), self.table_store.clone().unwrap(), + #[cfg(feature = "unstable-blockstore")] self.block_store.clone().unwrap(), ); if let Err(e) = storage_manager.init().await { @@ -136,6 +143,7 @@ impl ServicesContext { storage_manager, protected_store, table_store, + #[cfg(feature = "unstable-blockstore")] block_store, crypto, ); @@ -162,6 +170,7 @@ impl ServicesContext { trace!("terminate storage manager"); storage_manager.terminate().await; } + #[cfg(feature = "unstable-blockstore")] if let Some(block_store) = &mut self.block_store { trace!("terminate block store"); block_store.terminate().await; @@ -198,6 +207,7 @@ pub struct VeilidCoreContext { pub storage_manager: StorageManager, pub protected_store: ProtectedStore, pub table_store: TableStore, + #[cfg(feature = "unstable-blockstore")] pub block_store: BlockStore, pub crypto: Crypto, pub attachment_manager: AttachmentManager, @@ -251,6 +261,7 @@ impl VeilidCoreContext { storage_manager: sc.storage_manager.unwrap(), protected_store: sc.protected_store.unwrap(), table_store: sc.table_store.unwrap(), + #[cfg(feature = "unstable-blockstore")] block_store: sc.block_store.unwrap(), crypto: sc.crypto.unwrap(), attachment_manager: sc.attachment_manager.unwrap(), @@ -264,6 +275,7 @@ impl VeilidCoreContext { self.update_callback.clone(), self.protected_store, self.table_store, + #[cfg(feature = "unstable-blockstore")] self.block_store, self.crypto, self.attachment_manager, diff --git a/veilid-core/src/intf/native/mod.rs b/veilid-core/src/intf/native/mod.rs index 018cba41..d8cc6811 100644 --- a/veilid-core/src/intf/native/mod.rs +++ b/veilid-core/src/intf/native/mod.rs @@ -1,8 +1,12 @@ +#[cfg(feature = "unstable-blockstore")] mod block_store; + mod protected_store; mod system; +#[cfg(feature = "unstable-blockstore")] pub use block_store::*; + pub use protected_store::*; pub use system::*; diff --git a/veilid-core/src/intf/wasm/mod.rs b/veilid-core/src/intf/wasm/mod.rs index b69ada7b..113d35db 100644 --- a/veilid-core/src/intf/wasm/mod.rs +++ b/veilid-core/src/intf/wasm/mod.rs @@ -1,8 +1,12 @@ +#[cfg(feature = "unstable-blockstore")] mod block_store; + mod protected_store; mod system; +#[cfg(feature = "unstable-blockstore")] pub use block_store::*; + pub use protected_store::*; pub use system::*; diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 3a57e134..9456c625 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -150,6 +150,7 @@ struct NetworkManagerUnlockedInner { storage_manager: StorageManager, protected_store: ProtectedStore, table_store: TableStore, + #[cfg(feature="unstable-blockstore")] block_store: BlockStore, crypto: Crypto, // Accessors @@ -181,6 +182,7 @@ impl NetworkManager { storage_manager: StorageManager, protected_store: ProtectedStore, table_store: TableStore, + #[cfg(feature="unstable-blockstore")] block_store: BlockStore, crypto: Crypto, ) -> NetworkManagerUnlockedInner { @@ -189,6 +191,7 @@ impl NetworkManager { storage_manager, protected_store, table_store, + #[cfg(feature="unstable-blockstore")] block_store, crypto, routing_table: RwLock::new(None), @@ -204,6 +207,7 @@ impl NetworkManager { storage_manager: StorageManager, protected_store: ProtectedStore, table_store: TableStore, + #[cfg(feature="unstable-blockstore")] block_store: BlockStore, crypto: Crypto, ) -> Self { @@ -214,6 +218,7 @@ impl NetworkManager { storage_manager, protected_store, table_store, + #[cfg(feature="unstable-blockstore")] block_store, crypto, )), @@ -241,6 +246,7 @@ impl NetworkManager { pub fn table_store(&self) -> TableStore { self.unlocked_inner.table_store.clone() } + #[cfg(feature="unstable-blockstore")] pub fn block_store(&self) -> BlockStore { self.unlocked_inner.block_store.clone() } diff --git a/veilid-core/src/routing_table/tests/test_serialize.rs b/veilid-core/src/routing_table/tests/test_serialize.rs index 14e07930..b3609745 100644 --- a/veilid-core/src/routing_table/tests/test_serialize.rs +++ b/veilid-core/src/routing_table/tests/test_serialize.rs @@ -2,6 +2,7 @@ use crate::*; fn fake_routing_table() -> routing_table::RoutingTable { let veilid_config = VeilidConfig::new(); + #[cfg(feature = "unstable-blockstore")] let block_store = BlockStore::new(veilid_config.clone()); let protected_store = ProtectedStore::new(veilid_config.clone()); let table_store = TableStore::new(veilid_config.clone(), protected_store.clone()); @@ -11,6 +12,7 @@ fn fake_routing_table() -> routing_table::RoutingTable { crypto.clone(), protected_store.clone(), table_store.clone(), + #[cfg(feature = "unstable-blockstore")] block_store.clone(), ); let network_manager = network_manager::NetworkManager::new( @@ -18,6 +20,7 @@ fn fake_routing_table() -> routing_table::RoutingTable { storage_manager, protected_store.clone(), table_store.clone(), + #[cfg(feature = "unstable-blockstore")] block_store.clone(), crypto.clone(), ); diff --git a/veilid-core/src/rpc_processor/coders/operations/answer.rs b/veilid-core/src/rpc_processor/coders/operations/answer.rs index c1aeb4c4..356d1ba5 100644 --- a/veilid-core/src/rpc_processor/coders/operations/answer.rs +++ b/veilid-core/src/rpc_processor/coders/operations/answer.rs @@ -37,7 +37,9 @@ pub enum RPCAnswerDetail { GetValueA(RPCOperationGetValueA), SetValueA(RPCOperationSetValueA), WatchValueA(RPCOperationWatchValueA), + #[cfg(feature = "unstable-blockstore")] SupplyBlockA(RPCOperationSupplyBlockA), + #[cfg(feature = "unstable-blockstore")] FindBlockA(RPCOperationFindBlockA), #[cfg(feature = "unstable-tunnels")] StartTunnelA(RPCOperationStartTunnelA), @@ -56,7 +58,9 @@ impl RPCAnswerDetail { RPCAnswerDetail::GetValueA(_) => "GetValueA", RPCAnswerDetail::SetValueA(_) => "SetValueA", RPCAnswerDetail::WatchValueA(_) => "WatchValueA", + #[cfg(feature = "unstable-blockstore")] RPCAnswerDetail::SupplyBlockA(_) => "SupplyBlockA", + #[cfg(feature = "unstable-blockstore")] RPCAnswerDetail::FindBlockA(_) => "FindBlockA", #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::StartTunnelA(_) => "StartTunnelA", @@ -74,7 +78,9 @@ impl RPCAnswerDetail { RPCAnswerDetail::GetValueA(r) => r.validate(validate_context), RPCAnswerDetail::SetValueA(r) => r.validate(validate_context), RPCAnswerDetail::WatchValueA(r) => r.validate(validate_context), + #[cfg(feature = "unstable-blockstore")] RPCAnswerDetail::SupplyBlockA(r) => r.validate(validate_context), + #[cfg(feature = "unstable-blockstore")] RPCAnswerDetail::FindBlockA(r) => r.validate(validate_context), #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::StartTunnelA(r) => r.validate(validate_context), @@ -119,11 +125,13 @@ impl RPCAnswerDetail { let out = RPCOperationWatchValueA::decode(&op_reader)?; RPCAnswerDetail::WatchValueA(out) } + #[cfg(feature = "unstable-blockstore")] veilid_capnp::answer::detail::SupplyBlockA(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationSupplyBlockA::decode(&op_reader)?; RPCAnswerDetail::SupplyBlockA(out) } + #[cfg(feature = "unstable-blockstore")] veilid_capnp::answer::detail::FindBlockA(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationFindBlockA::decode(&op_reader)?; @@ -163,9 +171,11 @@ impl RPCAnswerDetail { RPCAnswerDetail::WatchValueA(d) => { d.encode(&mut builder.reborrow().init_watch_value_a()) } + #[cfg(feature = "unstable-blockstore")] RPCAnswerDetail::SupplyBlockA(d) => { d.encode(&mut builder.reborrow().init_supply_block_a()) } + #[cfg(feature = "unstable-blockstore")] RPCAnswerDetail::FindBlockA(d) => d.encode(&mut builder.reborrow().init_find_block_a()), #[cfg(feature = "unstable-tunnels")] RPCAnswerDetail::StartTunnelA(d) => { diff --git a/veilid-core/src/rpc_processor/coders/operations/mod.rs b/veilid-core/src/rpc_processor/coders/operations/mod.rs index db390b9f..72f4c520 100644 --- a/veilid-core/src/rpc_processor/coders/operations/mod.rs +++ b/veilid-core/src/rpc_processor/coders/operations/mod.rs @@ -2,7 +2,6 @@ mod answer; mod operation; mod operation_app_call; mod operation_app_message; -mod operation_find_block; mod operation_find_node; mod operation_get_value; mod operation_return_receipt; @@ -10,7 +9,7 @@ mod operation_route; mod operation_set_value; mod operation_signal; mod operation_status; -mod operation_supply_block; + mod operation_validate_dial_info; mod operation_value_changed; mod operation_watch_value; @@ -18,6 +17,11 @@ mod question; mod respond_to; mod statement; +#[cfg(feature = "unstable-blockstore")] +mod operation_find_block; +#[cfg(feature = "unstable-blockstore")] +mod operation_supply_block; + #[cfg(feature = "unstable-tunnels")] mod operation_cancel_tunnel; #[cfg(feature = "unstable-tunnels")] @@ -29,7 +33,6 @@ pub use answer::*; pub use operation::*; pub use operation_app_call::*; pub use operation_app_message::*; -pub use operation_find_block::*; pub use operation_find_node::*; pub use operation_get_value::*; pub use operation_return_receipt::*; @@ -37,7 +40,6 @@ pub use operation_route::*; pub use operation_set_value::*; pub use operation_signal::*; pub use operation_status::*; -pub use operation_supply_block::*; pub use operation_validate_dial_info::*; pub use operation_value_changed::*; pub use operation_watch_value::*; @@ -45,6 +47,11 @@ pub use question::*; pub use respond_to::*; pub use statement::*; +#[cfg(feature = "unstable-blockstore")] +pub use operation_find_block::*; +#[cfg(feature = "unstable-blockstore")] +pub use operation_supply_block::*; + #[cfg(feature = "unstable-tunnels")] pub use operation_cancel_tunnel::*; #[cfg(feature = "unstable-tunnels")] diff --git a/veilid-core/src/rpc_processor/coders/operations/question.rs b/veilid-core/src/rpc_processor/coders/operations/question.rs index 03debcb5..12591220 100644 --- a/veilid-core/src/rpc_processor/coders/operations/question.rs +++ b/veilid-core/src/rpc_processor/coders/operations/question.rs @@ -49,7 +49,9 @@ pub enum RPCQuestionDetail { GetValueQ(RPCOperationGetValueQ), SetValueQ(RPCOperationSetValueQ), WatchValueQ(RPCOperationWatchValueQ), + #[cfg(feature = "unstable-blockstore")] SupplyBlockQ(RPCOperationSupplyBlockQ), + #[cfg(feature = "unstable-blockstore")] FindBlockQ(RPCOperationFindBlockQ), #[cfg(feature = "unstable-tunnels")] StartTunnelQ(RPCOperationStartTunnelQ), @@ -68,7 +70,9 @@ impl RPCQuestionDetail { RPCQuestionDetail::GetValueQ(_) => "GetValueQ", RPCQuestionDetail::SetValueQ(_) => "SetValueQ", RPCQuestionDetail::WatchValueQ(_) => "WatchValueQ", + #[cfg(feature = "unstable-blockstore")] RPCQuestionDetail::SupplyBlockQ(_) => "SupplyBlockQ", + #[cfg(feature = "unstable-blockstore")] RPCQuestionDetail::FindBlockQ(_) => "FindBlockQ", #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::StartTunnelQ(_) => "StartTunnelQ", @@ -86,7 +90,9 @@ impl RPCQuestionDetail { RPCQuestionDetail::GetValueQ(r) => r.validate(validate_context), RPCQuestionDetail::SetValueQ(r) => r.validate(validate_context), RPCQuestionDetail::WatchValueQ(r) => r.validate(validate_context), + #[cfg(feature = "unstable-blockstore")] RPCQuestionDetail::SupplyBlockQ(r) => r.validate(validate_context), + #[cfg(feature = "unstable-blockstore")] RPCQuestionDetail::FindBlockQ(r) => r.validate(validate_context), #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::StartTunnelQ(r) => r.validate(validate_context), @@ -132,11 +138,13 @@ impl RPCQuestionDetail { let out = RPCOperationWatchValueQ::decode(&op_reader)?; RPCQuestionDetail::WatchValueQ(out) } + #[cfg(feature = "unstable-blockstore")] veilid_capnp::question::detail::SupplyBlockQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationSupplyBlockQ::decode(&op_reader)?; RPCQuestionDetail::SupplyBlockQ(out) } + #[cfg(feature = "unstable-blockstore")] veilid_capnp::question::detail::FindBlockQ(r) => { let op_reader = r.map_err(RPCError::protocol)?; let out = RPCOperationFindBlockQ::decode(&op_reader)?; @@ -176,9 +184,11 @@ impl RPCQuestionDetail { RPCQuestionDetail::WatchValueQ(d) => { d.encode(&mut builder.reborrow().init_watch_value_q()) } + #[cfg(feature = "unstable-blockstore")] RPCQuestionDetail::SupplyBlockQ(d) => { d.encode(&mut builder.reborrow().init_supply_block_q()) } + #[cfg(feature = "unstable-blockstore")] RPCQuestionDetail::FindBlockQ(d) => { d.encode(&mut builder.reborrow().init_find_block_q()) } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index d76d31fe..ba24517e 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -5,7 +5,6 @@ mod operation_waiter; mod rpc_app_call; mod rpc_app_message; mod rpc_error; -mod rpc_find_block; mod rpc_find_node; mod rpc_get_value; mod rpc_return_receipt; @@ -13,11 +12,15 @@ mod rpc_route; mod rpc_set_value; mod rpc_signal; mod rpc_status; -mod rpc_supply_block; mod rpc_validate_dial_info; mod rpc_value_changed; mod rpc_watch_value; +#[cfg(feature = "unstable-blockstore")] +mod rpc_find_block; +#[cfg(feature = "unstable-blockstore")] +mod rpc_supply_block; + #[cfg(feature = "unstable-tunnels")] mod rpc_cancel_tunnel; #[cfg(feature = "unstable-tunnels")] @@ -1412,7 +1415,9 @@ impl RPCProcessor { RPCQuestionDetail::GetValueQ(_) => self.process_get_value_q(msg).await, RPCQuestionDetail::SetValueQ(_) => self.process_set_value_q(msg).await, RPCQuestionDetail::WatchValueQ(_) => self.process_watch_value_q(msg).await, + #[cfg(feature = "unstable-blockstore")] RPCQuestionDetail::SupplyBlockQ(_) => self.process_supply_block_q(msg).await, + #[cfg(feature = "unstable-blockstore")] RPCQuestionDetail::FindBlockQ(_) => self.process_find_block_q(msg).await, #[cfg(feature = "unstable-tunnels")] RPCQuestionDetail::StartTunnelQ(_) => self.process_start_tunnel_q(msg).await, diff --git a/veilid-core/src/storage_manager/mod.rs b/veilid-core/src/storage_manager/mod.rs index 96f772d2..42f30d94 100644 --- a/veilid-core/src/storage_manager/mod.rs +++ b/veilid-core/src/storage_manager/mod.rs @@ -30,6 +30,7 @@ struct StorageManagerUnlockedInner { crypto: Crypto, protected_store: ProtectedStore, table_store: TableStore, + #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, // Background processes @@ -48,13 +49,14 @@ impl StorageManager { crypto: Crypto, protected_store: ProtectedStore, table_store: TableStore, - block_store: BlockStore, + #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, ) -> StorageManagerUnlockedInner { StorageManagerUnlockedInner { config, crypto, protected_store, table_store, + #[cfg(feature = "unstable-blockstore")] block_store, flush_record_stores_task: TickTask::new(FLUSH_RECORD_STORES_INTERVAL_SECS), } @@ -68,13 +70,14 @@ impl StorageManager { crypto: Crypto, protected_store: ProtectedStore, table_store: TableStore, - block_store: BlockStore, + #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, ) -> StorageManager { let unlocked_inner = Arc::new(Self::new_unlocked_inner( config, crypto, protected_store, table_store, + #[cfg(feature = "unstable-blockstore")] block_store, )); let this = StorageManager { diff --git a/veilid-core/src/veilid_api/api.rs b/veilid-core/src/veilid_api/api.rs index 59a9f2db..33d4ed91 100644 --- a/veilid-core/src/veilid_api/api.rs +++ b/veilid-core/src/veilid_api/api.rs @@ -70,6 +70,7 @@ impl VeilidAPI { } Err(VeilidAPIError::not_initialized()) } + #[cfg(feature = "unstable-blockstore")] pub fn block_store(&self) -> VeilidAPIResult { let inner = self.inner.lock(); if let Some(context) = &inner.context { diff --git a/veilid-core/src/veilid_api/json_api/crypto_system.rs b/veilid-core/src/veilid_api/json_api/crypto_system.rs index 73f08923..7f7e9aef 100644 --- a/veilid-core/src/veilid_api/json_api/crypto_system.rs +++ b/veilid-core/src/veilid_api/json_api/crypto_system.rs @@ -18,49 +18,210 @@ pub struct CryptoSystemResponse { #[serde(tag = "cs_op")] pub enum CryptoSystemRequestOp { Release, - CachedDh, - ComputeDh, - RandomBytes, + CachedDh { + #[schemars(with = "String")] + key: PublicKey, + #[schemars(with = "String")] + secret: SecretKey, + }, + ComputeDh { + #[schemars(with = "String")] + key: PublicKey, + #[schemars(with = "String")] + secret: SecretKey, + }, + RandomBytes { + len: u32, + }, DefaultSaltLength, - HashPassword, - VerifyPassword, - DeriveSharedSecret, + HashPassword { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + password: Vec, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + salt: Vec, + }, + VerifyPassword { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + password: Vec, + password_hash: String, + }, + DeriveSharedSecret { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + password: Vec, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + salt: Vec, + }, RandomNonce, RandomSharedSecret, GenerateKeyPair, - GenerateHash, - ValidateKeyPair, - ValidateHash, - Distance, - Sign, - Verify, + GenerateHash { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + data: Vec, + }, + ValidateKeyPair { + #[schemars(with = "String")] + key: PublicKey, + #[schemars(with = "String")] + secret: SecretKey, + }, + ValidateHash { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + data: Vec, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + hash_digest: Vec, + }, + Distance { + #[schemars(with = "String")] + key1: CryptoKey, + #[schemars(with = "String")] + key2: CryptoKey, + }, + Sign { + #[schemars(with = "String")] + key: PublicKey, + #[schemars(with = "String")] + secret: SecretKey, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + data: Vec, + }, + Verify { + #[schemars(with = "String")] + key: PublicKey, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + data: Vec, + #[schemars(with = "String")] + secret: Signature, + }, AeadOverhead, - DecryptAead, - EncryptAead, - CryptNoAuth, + DecryptAead { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + body: Vec, + #[schemars(with = "String")] + nonce: Nonce, + #[schemars(with = "String")] + shared_secret: SharedSecret, + #[serde(with = "opt_json_as_base64")] + #[schemars(with = "Option")] + associated_data: Option>, + }, + EncryptAead { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + body: Vec, + #[schemars(with = "String")] + nonce: Nonce, + #[schemars(with = "String")] + shared_secret: SharedSecret, + #[serde(with = "opt_json_as_base64")] + #[schemars(with = "Option")] + associated_data: Option>, + }, + CryptNoAuth { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + body: Vec, + #[schemars(with = "String")] + nonce: Nonce, + #[schemars(with = "String")] + shared_secret: SharedSecret, + }, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "cs_op")] pub enum CryptoSystemResponseOp { Release, - CachedDh, - ComputeDh, - RandomBytes, - DefaultSaltLength, - HashPassword, - VerifyPassword, - DeriveSharedSecret, - RandomNonce, - RandomSharedSecret, - GenerateKeyPair, - GenerateHash, - ValidateKeyPair, - ValidateHash, - Distance, - Sign, - Verify, - AeadOverhead, - DecryptAead, - EncryptAead, - CryptNoAuth, + CachedDh { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithString, + }, + ComputeDh { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithString, + }, + RandomBytes { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithVecU8, + }, + DefaultSaltLength { + value: u32, + }, + HashPassword { + #[serde(flatten)] + result: ApiResult, + }, + VerifyPassword { + #[serde(flatten)] + result: ApiResult, + }, + DeriveSharedSecret { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithString, + }, + RandomNonce { + #[schemars(with = "String")] + value: Nonce, + }, + RandomSharedSecret { + #[schemars(with = "String")] + value: SharedSecret, + }, + GenerateKeyPair { + #[schemars(with = "String")] + value: KeyPair, + }, + GenerateHash { + #[schemars(with = "String")] + value: HashDigest, + }, + ValidateKeyPair { + value: bool, + }, + ValidateHash { + value: bool, + }, + Distance { + #[schemars(with = "String")] + value: CryptoKeyDistance, + }, + Sign { + #[schemars(with = "String")] + value: Signature, + }, + Verify { + #[serde(flatten)] + result: ApiResult<()>, + }, + AeadOverhead { + value: u32, + }, + DecryptAead { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithVecU8, + }, + EncryptAead { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithVecU8, + }, + CryptNoAuth { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithVecU8, + }, } diff --git a/veilid-core/src/veilid_api/json_api/mod.rs b/veilid-core/src/veilid_api/json_api/mod.rs index 5fb4b07b..9fab5eae 100644 --- a/veilid-core/src/veilid_api/json_api/mod.rs +++ b/veilid-core/src/veilid_api/json_api/mod.rs @@ -11,7 +11,8 @@ pub use crypto_system::*; #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct Request { - /// Operation Id (pairs with Response) + /// Operation Id (pairs with Response, or empty if unidirectional) + #[serde(default)] id: String, /// The request operation variant #[serde(flatten)] @@ -20,7 +21,8 @@ pub struct Request { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct Response { - /// Operation Id (pairs with Request) + /// Operation Id (pairs with Request, or empty if unidirectional) + #[serde(default)] id: String, /// The response operation variant #[serde(flatten)] @@ -36,17 +38,14 @@ pub enum RequestOp { NewPrivateRoute, NewCustomPrivateRoute { #[schemars(with = "Vec")] - crypto_kinds: Vec, + kinds: Vec, #[serde(default)] stability: Stability, #[serde(default)] sequencing: Sequencing, }, ImportRemotePrivateRoute { - #[serde( - serialize_with = "json_as_base64::serialize", - deserialize_with = "json_as_base64::deserialize" - )] + #[serde(with = "json_as_base64")] #[schemars(with = "String")] blob: Vec, }, @@ -57,10 +56,7 @@ pub enum RequestOp { AppCallReply { #[schemars(with = "String")] call_id: OperationId, - #[serde( - serialize_with = "json_as_base64::serialize", - deserialize_with = "json_as_base64::deserialize" - )] + #[serde(with = "json_as_base64")] #[schemars(with = "String")] message: Vec, }, @@ -79,27 +75,21 @@ pub enum RequestOp { // Crypto GetCryptoSystem { #[schemars(with = "String")] - crypto_kind: CryptoKind, + kind: CryptoKind, }, BestCryptoSystem, CryptoSystem(CryptoSystemRequest), VerifySignatures { #[schemars(with = "Vec")] node_ids: Vec, - #[serde( - serialize_with = "json_as_base64::serialize", - deserialize_with = "json_as_base64::deserialize" - )] + #[serde(with = "json_as_base64")] #[schemars(with = "String")] data: Vec, #[schemars(with = "Vec")] signatures: Vec, }, GenerateSignatures { - #[serde( - serialize_with = "json_as_base64::serialize", - deserialize_with = "json_as_base64::deserialize" - )] + #[serde(with = "json_as_base64")] #[schemars(with = "String")] data: Vec, #[schemars(with = "Vec")] @@ -107,7 +97,7 @@ pub enum RequestOp { }, GenerateKeyPair { #[schemars(with = "String")] - crypto_kind: CryptoKind, + kind: CryptoKind, }, // Misc Now, @@ -122,10 +112,7 @@ pub enum RequestOp { pub struct NewPrivateRouteResult { #[schemars(with = "String")] route_id: RouteId, - #[serde( - serialize_with = "json_as_base64::serialize", - deserialize_with = "json_as_base64::deserialize" - )] + #[serde(with = "json_as_base64")] #[schemars(with = "String")] blob: Vec, } @@ -133,6 +120,9 @@ pub struct NewPrivateRouteResult { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "op")] pub enum ResponseOp { + Update { + value: VeilidUpdate, + }, GetState { #[serde(flatten)] result: ApiResult, @@ -153,28 +143,74 @@ pub enum ResponseOp { #[serde(flatten)] result: ApiResult, }, - ImportRemotePrivateRoute, - ReleasePrivateRoute, - AppCallReply, + ImportRemotePrivateRoute { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithString, + }, + ReleasePrivateRoute { + #[serde(flatten)] + result: ApiResult<()>, + }, + AppCallReply { + #[serde(flatten)] + result: ApiResult<()>, + }, // Routing Context - NewRoutingContext, + NewRoutingContext { + value: String, + }, RoutingContext(RoutingContextResponse), // TableDb - OpenTableDb, - DeleteTableDb, + OpenTableDb { + #[serde(flatten)] + result: ApiResult, + }, + DeleteTableDb { + #[serde(flatten)] + result: ApiResult, + }, TableDb(TableDbResponse), // Crypto - GetCryptoSystem, - BestCryptoSystem, + GetCryptoSystem { + #[serde(flatten)] + result: ApiResult, + }, + BestCryptoSystem { + value: String, + }, CryptoSystem(CryptoSystemResponse), - VerifySignatures, - GenerateSignatures, - GenerateKeyPair, + VerifySignatures { + #[serde(flatten)] + #[schemars(with = "ApiResult>")] + result: ApiResultWithVecString, + }, + GenerateSignatures { + #[serde(flatten)] + #[schemars(with = "ApiResult>")] + result: ApiResultWithVecString, + }, + GenerateKeyPair { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithString, + }, // Misc - Now, - Debug, - VeilidVersionString, - VeilidVersion, + Now { + #[schemars(with = "String")] + value: Timestamp, + }, + Debug { + value: String, + }, + VeilidVersionString { + value: String, + }, + VeilidVersion { + major: u32, + minor: u32, + patch: u32, + }, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -187,6 +223,49 @@ where Err { error: VeilidAPIError }, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum ApiResultWithString +where + T: Clone + fmt::Debug, +{ + Ok { + #[schemars(with = "String")] + value: T, + }, + Err { + error: VeilidAPIError, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum ApiResultWithVecU8 { + Ok { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + value: Vec, + }, + Err { + error: VeilidAPIError, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum ApiResultWithVecString +where + T: Clone + fmt::Debug, +{ + Ok { + #[schemars(with = "Vec")] + value: T, + }, + Err { + error: VeilidAPIError, + }, +} + pub fn emit_schemas(out: &mut HashMap) { let schema_request = schema_for!(Request); let schema_response = schema_for!(Response); diff --git a/veilid-core/src/veilid_api/json_api/routing_context.rs b/veilid-core/src/veilid_api/json_api/routing_context.rs index c4a9405c..9376d81f 100644 --- a/veilid-core/src/veilid_api/json_api/routing_context.rs +++ b/veilid-core/src/veilid_api/json_api/routing_context.rs @@ -19,34 +19,122 @@ pub struct RoutingContextResponse { pub enum RoutingContextRequestOp { Release, WithPrivacy, - WithCustomPrivacy, - WithSequencing, - AppCall, - AppMessage, - CreateDhtRecord, - OpenDhtRecord, - CloseDhtRecord, - DeleteDhtRecord, - GetDhtValue, - SetDhtValue, - WatchDhtValues, - CancelDhtWatch, + WithCustomPrivacy { + stability: Stability, + }, + WithSequencing { + sequencing: Sequencing, + }, + AppCall { + target: String, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + request: Vec, + }, + AppMessage { + target: String, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + message: Vec, + }, + CreateDhtRecord { + #[schemars(with = "String")] + kind: CryptoKind, + schema: DHTSchema, + }, + OpenDhtRecord { + #[schemars(with = "String")] + key: TypedKey, + #[schemars(with = "Option")] + writer: Option, + }, + CloseDhtRecord { + #[schemars(with = "String")] + key: TypedKey, + }, + DeleteDhtRecord { + #[schemars(with = "String")] + key: TypedKey, + }, + GetDhtValue { + #[schemars(with = "String")] + key: TypedKey, + subkey: ValueSubkey, + force_refresh: bool, + }, + SetDhtValue { + #[schemars(with = "String")] + key: TypedKey, + subkey: ValueSubkey, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + data: Vec, + }, + WatchDhtValues { + #[schemars(with = "String")] + key: TypedKey, + subkeys: ValueSubkeyRangeSet, + expiration: Timestamp, + count: u32, + }, + CancelDhtWatch { + #[schemars(with = "String")] + key: TypedKey, + subkeys: ValueSubkeyRangeSet, + }, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "rc_op")] pub enum RoutingContextResponseOp { Release, - WithPrivacy, - WithCustomPrivacy, - WithSequencing, - AppCall, - AppMessage, - CreateDhtRecord, - OpenDhtRecord, - CloseDhtRecord, - DeleteDhtRecord, - GetDhtValue, - SetDhtValue, - WatchDhtValues, - CancelDhtWatch, + WithPrivacy { + value: String, + }, + WithCustomPrivacy { + value: String, + }, + WithSequencing { + value: String, + }, + AppCall { + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithVecU8, + }, + AppMessage { + #[serde(flatten)] + result: ApiResult<()>, + }, + CreateDhtRecord { + #[serde(flatten)] + result: ApiResult, + }, + OpenDhtRecord { + #[serde(flatten)] + result: ApiResult, + }, + CloseDhtRecord { + #[serde(flatten)] + result: ApiResult<()>, + }, + DeleteDhtRecord { + #[serde(flatten)] + result: ApiResult<()>, + }, + GetDhtValue { + #[serde(flatten)] + result: ApiResult>, + }, + SetDhtValue { + #[serde(flatten)] + result: ApiResult>, + }, + WatchDhtValues { + #[serde(flatten)] + result: ApiResult, + }, + CancelDhtWatch { + #[serde(flatten)] + result: ApiResult, + }, } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 50afee6f..9f05e56d 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -22,6 +22,7 @@ pub use alloc::string::ToString; pub use attachment_manager::AttachmentManager; pub use core::str::FromStr; pub use crypto::*; +#[cfg(feature = "unstable-blockstore")] pub use intf::BlockStore; pub use intf::ProtectedStore; pub use network_manager::NetworkManager; diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index 952a42b7..e73e54ca 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -290,10 +290,12 @@ impl RoutingContext { /////////////////////////////////// /// Block Store + #[cfg(feature = "unstable-blockstore")] pub async fn find_block(&self, _block_id: PublicKey) -> VeilidAPIResult> { panic!("unimplemented"); } + #[cfg(feature = "unstable-blockstore")] pub async fn supply_block(&self, _block_id: PublicKey) -> VeilidAPIResult { panic!("unimplemented"); } diff --git a/veilid-core/src/veilid_api/serialize_helpers/serialize_json.rs b/veilid-core/src/veilid_api/serialize_helpers/serialize_json.rs index 5e98624c..afdab167 100644 --- a/veilid-core/src/veilid_api/serialize_helpers/serialize_json.rs +++ b/veilid-core/src/veilid_api/serialize_helpers/serialize_json.rs @@ -55,6 +55,27 @@ pub mod json_as_base64 { } } +pub mod opt_json_as_base64 { + use data_encoding::BASE64URL_NOPAD; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(v: &Option>, s: S) -> Result { + let base64 = v.as_ref().map(|x| BASE64URL_NOPAD.encode(&x)); + Option::::serialize(&base64, s) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result>, D::Error> { + let base64 = Option::::deserialize(d)?; + base64 + .map(|x| { + BASE64URL_NOPAD + .decode(x.as_bytes()) + .map_err(|e| serde::de::Error::custom(e)) + }) + .transpose() + } +} + pub mod json_as_string { use std::fmt::Display; use std::str::FromStr; diff --git a/veilid-core/src/veilid_api/types/aligned_u64.rs b/veilid-core/src/veilid_api/types/aligned_u64.rs index 11ee022d..566bac84 100644 --- a/veilid-core/src/veilid_api/types/aligned_u64.rs +++ b/veilid-core/src/veilid_api/types/aligned_u64.rs @@ -24,10 +24,7 @@ use super::*; #[archive_attr(repr(C, align(8)), derive(CheckBytes))] #[serde(transparent)] pub struct AlignedU64( - #[serde( - serialize_with = "json_as_string::serialize", - deserialize_with = "json_as_string::deserialize" - )] + #[serde(with = "json_as_string")] #[schemars(with = "String")] u64, ); diff --git a/veilid-core/src/veilid_api/types/app_message_call.rs b/veilid-core/src/veilid_api/types/app_message_call.rs index 2f3c77be..fb33ae50 100644 --- a/veilid-core/src/veilid_api/types/app_message_call.rs +++ b/veilid-core/src/veilid_api/types/app_message_call.rs @@ -16,18 +16,12 @@ use super::*; #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidAppMessage { /// Some(sender) if the message was sent directly, None if received via a private/safety route - #[serde( - serialize_with = "opt_json_as_string::serialize", - deserialize_with = "opt_json_as_string::deserialize" - )] + #[serde(with = "opt_json_as_string")] #[schemars(with = "Option")] sender: Option, /// The content of the message to deliver to the application - #[serde( - serialize_with = "json_as_base64::serialize", - deserialize_with = "json_as_base64::deserialize" - )] + #[serde(with = "json_as_base64")] #[schemars(with = "String")] message: Vec, } @@ -61,26 +55,17 @@ impl VeilidAppMessage { #[archive_attr(repr(C), derive(CheckBytes))] pub struct VeilidAppCall { /// Some(sender) if the request was sent directly, None if received via a private/safety route - #[serde( - serialize_with = "opt_json_as_string::serialize", - deserialize_with = "opt_json_as_string::deserialize" - )] + #[serde(with = "opt_json_as_string")] #[schemars(with = "Option")] sender: Option, /// The content of the request to deliver to the application - #[serde( - serialize_with = "json_as_base64::serialize", - deserialize_with = "json_as_base64::deserialize" - )] + #[serde(with = "json_as_base64")] #[schemars(with = "String")] message: Vec, /// The id to reply to - #[serde( - serialize_with = "json_as_string::serialize", - deserialize_with = "json_as_string::deserialize" - )] + #[serde(with = "json_as_string")] #[schemars(with = "String")] id: OperationId, } diff --git a/veilid-core/src/veilid_api/types/dht/value_data.rs b/veilid-core/src/veilid_api/types/dht/value_data.rs index eae2e2c4..c57fc2e5 100644 --- a/veilid-core/src/veilid_api/types/dht/value_data.rs +++ b/veilid-core/src/veilid_api/types/dht/value_data.rs @@ -21,10 +21,7 @@ pub struct ValueData { seq: ValueSeqNum, /// The contents of a DHT Record - #[serde( - serialize_with = "json_as_base64::serialize", - deserialize_with = "json_as_base64::deserialize" - )] + #[serde(with = "json_as_base64")] #[schemars(with = "String")] data: Vec, diff --git a/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs b/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs index 955c9d01..567081f6 100644 --- a/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs +++ b/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs @@ -18,12 +18,10 @@ use range_set_blaze::*; JsonSchema, )] #[archive_attr(repr(C), derive(CheckBytes))] +#[serde(transparent)] pub struct ValueSubkeyRangeSet { #[with(RkyvRangeSetBlaze)] - #[serde( - serialize_with = "serialize_range_set_blaze::serialize", - deserialize_with = "serialize_range_set_blaze::deserialize" - )] + #[serde(with = "serialize_range_set_blaze")] #[schemars(with = "Vec<(u32,u32)>")] data: RangeSetBlaze, } From 0e52c1fb0ad928c7babc0ca6311d1c3243fe20a4 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 4 Jun 2023 22:08:46 -0400 Subject: [PATCH 03/23] more json schema --- veilid-core/src/veilid_api/api.rs | 8 ++ veilid-core/src/veilid_api/json_api/mod.rs | 37 +++++- .../src/veilid_api/json_api/table_db.rs | 109 ++++++++++++++++++ veilid-flutter/lib/veilid_ffi.dart | 10 +- veilid-flutter/lib/veilid_js.dart | 2 +- veilid-flutter/lib/veilid_table_db.dart | 2 +- veilid-flutter/rust/src/dart_ffi.rs | 4 +- 7 files changed, 159 insertions(+), 13 deletions(-) diff --git a/veilid-core/src/veilid_api/api.rs b/veilid-core/src/veilid_api/api.rs index 33d4ed91..cd6d96dc 100644 --- a/veilid-core/src/veilid_api/api.rs +++ b/veilid-core/src/veilid_api/api.rs @@ -295,4 +295,12 @@ impl VeilidAPI { pub async fn cancel_tunnel(&self, _tunnel_id: TunnelId) -> VeilidAPIResult { panic!("unimplemented"); } + + //////////////////////////////////////////////////////////////// + // JSON API + + #[instrument(level = "debug", skip(self))] + pub async fn json_request(&self, request: json_api::Request) -> json_api::Response { + panic!("unimplemented"); + } } diff --git a/veilid-core/src/veilid_api/json_api/mod.rs b/veilid-core/src/veilid_api/json_api/mod.rs index 9fab5eae..f435b102 100644 --- a/veilid-core/src/veilid_api/json_api/mod.rs +++ b/veilid-core/src/veilid_api/json_api/mod.rs @@ -19,6 +19,13 @@ pub struct Request { op: RequestOp, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type")] +pub enum RecvMessage { + Response(Response), + Update(VeilidUpdate), +} + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct Response { /// Operation Id (pairs with Request, or empty if unidirectional) @@ -72,6 +79,7 @@ pub enum RequestOp { name: String, }, TableDb(TableDbRequest), + TableDbTransaction(TableDbTransactionRequest), // Crypto GetCryptoSystem { #[schemars(with = "String")] @@ -120,9 +128,6 @@ pub struct NewPrivateRouteResult { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "op")] pub enum ResponseOp { - Update { - value: VeilidUpdate, - }, GetState { #[serde(flatten)] result: ApiResult, @@ -171,6 +176,7 @@ pub enum ResponseOp { result: ApiResult, }, TableDb(TableDbResponse), + TableDbTransaction(TableDbTransactionResponse), // Crypto GetCryptoSystem { #[serde(flatten)] @@ -250,6 +256,25 @@ pub enum ApiResultWithVecU8 { error: VeilidAPIError, }, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(transparent)] +pub struct VecU8 { + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + value: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum ApiResultWithVecVecU8 { + Ok { + #[schemars(with = "Vec")] + value: Vec, + }, + Err { + error: VeilidAPIError, + }, +} #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] @@ -268,7 +293,7 @@ where pub fn emit_schemas(out: &mut HashMap) { let schema_request = schema_for!(Request); - let schema_response = schema_for!(Response); + let schema_recv_message = schema_for!(RecvMessage); out.insert( "Request".to_owned(), @@ -276,7 +301,7 @@ pub fn emit_schemas(out: &mut HashMap) { ); out.insert( - "Response".to_owned(), - serde_json::to_string_pretty(&schema_response).unwrap(), + "RecvMessage".to_owned(), + serde_json::to_string_pretty(&schema_recv_message).unwrap(), ); } diff --git a/veilid-core/src/veilid_api/json_api/table_db.rs b/veilid-core/src/veilid_api/json_api/table_db.rs index 62190e83..483c5d71 100644 --- a/veilid-core/src/veilid_api/json_api/table_db.rs +++ b/veilid-core/src/veilid_api/json_api/table_db.rs @@ -18,9 +18,118 @@ pub struct TableDbResponse { #[serde(tag = "db_op")] pub enum TableDbRequestOp { Release, + GetColumnCount, + GetKeys { + col: i32, + }, + Transact, + Store { + col: i32, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + key: Vec, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + value: Vec, + }, + Load { + col: i32, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + key: Vec, + }, + Delete { + col: i32, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + key: Vec, + }, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "db_op")] pub enum TableDbResponseOp { Release, + GetColumnCount { + value: i32, + }, + GetKeys { + #[serde(flatten)] + #[schemars(with = "ApiResult>")] + result: ApiResultWithVecVecU8, + }, + Transact { + value: String, + }, + Store { + #[serde(flatten)] + result: ApiResult<()>, + }, + Load { + #[serde(flatten)] + #[schemars(with = "ApiResult>")] + result: ApiResult>, + }, + Delete { + #[serde(flatten)] + #[schemars(with = "ApiResult>")] + result: ApiResult>, + }, +} + +////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct TableDbTransactionRequest { + tx_id: String, + #[serde(flatten)] + tx_op: TableDbTransactionRequestOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct TableDbTransactionResponse { + tx_id: String, + #[serde(flatten)] + tx_op: TableDbTransactionResponseOp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "tx_op")] +pub enum TableDbTransactionRequestOp { + Commit, + Rollback, + Store { + col: i32, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + key: Vec, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + value: Vec, + }, + Delete { + col: i32, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + key: Vec, + }, +} +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "tx_op")] +pub enum TableDbTransactionResponseOp { + Commit { + #[serde(flatten)] + result: ApiResult<()>, + }, + Rollback { + #[serde(flatten)] + result: ApiResult<()>, + }, + Store { + #[serde(flatten)] + result: ApiResult<()>, + }, + Delete { + #[serde(flatten)] + result: ApiResult<()>, + }, } diff --git a/veilid-flutter/lib/veilid_ffi.dart b/veilid-flutter/lib/veilid_ffi.dart index 5f17e7ec..a0a9bdcd 100644 --- a/veilid-flutter/lib/veilid_ffi.dart +++ b/veilid-flutter/lib/veilid_ffi.dart @@ -797,7 +797,7 @@ class VeilidTableDBTransactionFFI extends VeilidTableDBTransaction { } @override - Future delete(int col, Uint8List key) { + Future delete(int col, Uint8List key) { final nativeEncodedKey = base64UrlNoPadEncode(key).toNativeUtf8(); final recvPort = ReceivePort("veilid_table_db_transaction_delete"); @@ -888,7 +888,7 @@ class VeilidTableDBFFI extends VeilidTableDB { } @override - Future delete(int col, Uint8List key) { + Future delete(int col, Uint8List key) async { final nativeEncodedKey = base64UrlNoPadEncode(key).toNativeUtf8(); final recvPort = ReceivePort("veilid_table_db_delete"); @@ -899,7 +899,11 @@ class VeilidTableDBFFI extends VeilidTableDB { col, nativeEncodedKey, ); - return processFuturePlain(recvPort.first); + String? out = await processFuturePlain(recvPort.first); + if (out == null) { + return null; + } + return base64UrlNoPadDecode(out); } } diff --git a/veilid-flutter/lib/veilid_js.dart b/veilid-flutter/lib/veilid_js.dart index 8a323516..139c10a5 100644 --- a/veilid-flutter/lib/veilid_js.dart +++ b/veilid-flutter/lib/veilid_js.dart @@ -368,7 +368,7 @@ class VeilidTableDBTransactionJS extends VeilidTableDBTransaction { } @override - Future delete(int col, Uint8List key) { + Future delete(int col, Uint8List key) { final encodedKey = base64UrlNoPadEncode(key); return _wrapApiPromise(js_util.callMethod( diff --git a/veilid-flutter/lib/veilid_table_db.dart b/veilid-flutter/lib/veilid_table_db.dart index 84923fd0..19bee7f2 100644 --- a/veilid-flutter/lib/veilid_table_db.dart +++ b/veilid-flutter/lib/veilid_table_db.dart @@ -8,7 +8,7 @@ abstract class VeilidTableDBTransaction { Future commit(); Future rollback(); Future store(int col, Uint8List key, Uint8List value); - Future delete(int col, Uint8List key); + Future delete(int col, Uint8List key); Future storeJson(int col, Uint8List key, Object? object, {Object? Function(Object? nonEncodable)? toEncodable}) async { diff --git a/veilid-flutter/rust/src/dart_ffi.rs b/veilid-flutter/rust/src/dart_ffi.rs index d18d0e30..238f1805 100644 --- a/veilid-flutter/rust/src/dart_ffi.rs +++ b/veilid-flutter/rust/src/dart_ffi.rs @@ -908,8 +908,8 @@ pub extern "C" fn table_db_transaction_delete(port: i64, id: u32, col: u32, key: tdbt.clone() }; - let out = tdbt.delete(col, &key); - APIResult::Ok(out) + tdbt.delete(col, &key); + APIRESULT_VOID }); } From 89f5770fd1b036e30cd8009d5319e6652ca77e28 Mon Sep 17 00:00:00 2001 From: John Smith Date: Mon, 5 Jun 2023 22:02:37 -0400 Subject: [PATCH 04/23] more json api --- veilid-core/src/veilid_api/api.rs | 8 - .../src/veilid_api/json_api/crypto_system.rs | 9 +- veilid-core/src/veilid_api/json_api/mod.rs | 15 +- .../src/veilid_api/json_api/process.rs | 503 ++++++++++++++++++ .../veilid_api/json_api/routing_context.rs | 17 +- .../src/veilid_api/json_api/table_db.rs | 50 +- veilid-core/src/veilid_api/mod.rs | 3 +- 7 files changed, 550 insertions(+), 55 deletions(-) create mode 100644 veilid-core/src/veilid_api/json_api/process.rs diff --git a/veilid-core/src/veilid_api/api.rs b/veilid-core/src/veilid_api/api.rs index cd6d96dc..33d4ed91 100644 --- a/veilid-core/src/veilid_api/api.rs +++ b/veilid-core/src/veilid_api/api.rs @@ -295,12 +295,4 @@ impl VeilidAPI { pub async fn cancel_tunnel(&self, _tunnel_id: TunnelId) -> VeilidAPIResult { panic!("unimplemented"); } - - //////////////////////////////////////////////////////////////// - // JSON API - - #[instrument(level = "debug", skip(self))] - pub async fn json_request(&self, request: json_api::Request) -> json_api::Response { - panic!("unimplemented"); - } } diff --git a/veilid-core/src/veilid_api/json_api/crypto_system.rs b/veilid-core/src/veilid_api/json_api/crypto_system.rs index 7f7e9aef..58d5107f 100644 --- a/veilid-core/src/veilid_api/json_api/crypto_system.rs +++ b/veilid-core/src/veilid_api/json_api/crypto_system.rs @@ -2,16 +2,16 @@ use super::*; #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct CryptoSystemRequest { - cs_id: String, + pub cs_id: u32, #[serde(flatten)] - cs_op: CryptoSystemRequestOp, + pub cs_op: CryptoSystemRequestOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct CryptoSystemResponse { - cs_id: String, + pub cs_id: u32, #[serde(flatten)] - cs_op: CryptoSystemResponseOp, + pub cs_op: CryptoSystemResponseOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -140,6 +140,7 @@ pub enum CryptoSystemRequestOp { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "cs_op")] pub enum CryptoSystemResponseOp { + InvalidId, Release, CachedDh { #[serde(flatten)] diff --git a/veilid-core/src/veilid_api/json_api/mod.rs b/veilid-core/src/veilid_api/json_api/mod.rs index f435b102..c0ec39bc 100644 --- a/veilid-core/src/veilid_api/json_api/mod.rs +++ b/veilid-core/src/veilid_api/json_api/mod.rs @@ -9,11 +9,14 @@ pub use table_db::*; mod crypto_system; pub use crypto_system::*; +mod process; +pub use process::*; + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct Request { /// Operation Id (pairs with Response, or empty if unidirectional) #[serde(default)] - id: String, + id: u32, /// The request operation variant #[serde(flatten)] op: RequestOp, @@ -30,7 +33,7 @@ pub enum RecvMessage { pub struct Response { /// Operation Id (pairs with Request, or empty if unidirectional) #[serde(default)] - id: String, + id: u32, /// The response operation variant #[serde(flatten)] op: ResponseOp, @@ -163,13 +166,13 @@ pub enum ResponseOp { }, // Routing Context NewRoutingContext { - value: String, + value: u32, }, RoutingContext(RoutingContextResponse), // TableDb OpenTableDb { #[serde(flatten)] - result: ApiResult, + result: ApiResult, }, DeleteTableDb { #[serde(flatten)] @@ -180,10 +183,10 @@ pub enum ResponseOp { // Crypto GetCryptoSystem { #[serde(flatten)] - result: ApiResult, + result: ApiResult, }, BestCryptoSystem { - value: String, + value: u32, }, CryptoSystem(CryptoSystemResponse), VerifySignatures { diff --git a/veilid-core/src/veilid_api/json_api/process.rs b/veilid-core/src/veilid_api/json_api/process.rs new file mode 100644 index 00000000..cc635005 --- /dev/null +++ b/veilid-core/src/veilid_api/json_api/process.rs @@ -0,0 +1,503 @@ +use super::*; +use futures_util::FutureExt; + +fn to_json_api_result( + r: VeilidAPIResult, +) -> json_api::ApiResult { + match r { + Err(e) => json_api::ApiResult::Err { error: e }, + Ok(v) => json_api::ApiResult::Ok { value: v }, + } +} + +fn to_json_api_result_with_string( + r: VeilidAPIResult, +) -> json_api::ApiResultWithString { + match r { + Err(e) => json_api::ApiResultWithString::Err { error: e }, + Ok(v) => json_api::ApiResultWithString::Ok { value: v }, + } +} + +fn to_json_api_result_with_vec_u8(r: VeilidAPIResult>) -> json_api::ApiResultWithVecU8 { + match r { + Err(e) => json_api::ApiResultWithVecU8::Err { error: e }, + Ok(v) => json_api::ApiResultWithVecU8::Ok { value: v }, + } +} + +fn to_json_api_result_with_vec_vec_u8( + r: VeilidAPIResult>>, +) -> json_api::ApiResultWithVecVecU8 { + match r { + Err(e) => json_api::ApiResultWithVecVecU8::Err { error: e }, + Ok(v) => json_api::ApiResultWithVecVecU8::Ok { + value: v.into_iter().map(|v| VecU8 { value: v }).collect(), + }, + } +} + +pub struct JsonRequestProcessor { + api: VeilidAPI, + routing_contexts: Mutex>, + table_dbs: Mutex>, + table_db_transactions: Mutex>, +} + +impl JsonRequestProcessor { + pub fn new(api: VeilidAPI) -> Self { + Self { + api, + routing_contexts: Default::default(), + table_dbs: Default::default(), + table_db_transactions: Default::default(), + } + } + + // Routing Context + fn add_routing_context(&self, routing_context: RoutingContext) -> u32 { + let mut next_id: u32 = 1; + let mut rc = self.routing_contexts.lock(); + while rc.contains_key(&next_id) { + next_id += 1; + } + rc.insert(next_id, routing_context); + next_id + } + fn lookup_routing_context(&self, id: u32, rc_id: u32) -> Result { + let routing_contexts = self.routing_contexts.lock(); + let Some(routing_context) = routing_contexts.get(&rc_id).cloned() else { + return Err(Response { + id, + op: ResponseOp::RoutingContext(RoutingContextResponse { + rc_id, + rc_op: RoutingContextResponseOp::InvalidId + }) + }); + }; + Ok(routing_context) + } + fn release_routing_context(&self, id: u32) -> i32 { + let mut rc = self.routing_contexts.lock(); + if rc.remove(&id).is_none() { + return 0; + } + return 1; + } + + // TableDB + fn add_table_db(&self, table_db: TableDB) -> u32 { + let mut next_id: u32 = 1; + let mut rc = self.table_dbs.lock(); + while rc.contains_key(&next_id) { + next_id += 1; + } + rc.insert(next_id, table_db); + next_id + } + fn lookup_table_db(&self, id: u32, db_id: u32) -> Result { + let table_dbs = self.table_dbs.lock(); + let Some(table_db) = table_dbs.get(&db_id).cloned() else { + return Err(Response { + id, + op: ResponseOp::TableDb(TableDbResponse { + db_id, + db_op: TableDbResponseOp::InvalidId + }) + }); + }; + Ok(table_db) + } + fn release_table_db(&self, id: u32) -> i32 { + let mut rc = self.table_dbs.lock(); + if rc.remove(&id).is_none() { + return 0; + } + return 1; + } + + // TableDBTransaction + fn add_table_db_transaction(&self, tdbt: TableDBTransaction) -> u32 { + let mut next_id: u32 = 1; + let mut tdbts = self.table_db_transactions.lock(); + while tdbts.contains_key(&next_id) { + next_id += 1; + } + tdbts.insert(next_id, tdbt); + next_id + } + fn lookup_table_db_transaction( + &self, + id: u32, + tx_id: u32, + ) -> Result { + let table_db_transactions = self.table_db_transactions.lock(); + let Some(table_db_transaction) = table_db_transactions.get(&tx_id).cloned() else { + return Err(Response { + id, + op: ResponseOp::TableDbTransaction(TableDbTransactionResponse { + tx_id, + tx_op: TableDbTransactionResponseOp::InvalidId + }) + }); + }; + Ok(table_db_transaction) + } + fn release_table_db_transaction(&self, id: u32) -> i32 { + let mut tdbts = self.table_db_transactions.lock(); + if tdbts.remove(&id).is_none() { + return 0; + } + return 1; + } + + // Target + + // Parse target + async fn parse_target(&self, s: String) -> VeilidAPIResult { + // Is this a route id? + if let Ok(rrid) = RouteId::from_str(&s) { + let routing_table = self.api.routing_table()?; + let rss = routing_table.route_spec_store(); + + // Is this a valid remote route id? (can't target allocated routes) + if rss.is_route_id_remote(&rrid) { + return Ok(Target::PrivateRoute(rrid)); + } + } + + // Is this a node id? + if let Ok(nid) = TypedKey::from_str(&s) { + return Ok(Target::NodeId(nid)); + } + + Err(VeilidAPIError::invalid_target()) + } + + ////////////////////////////////////////////////////////////////////////////////////// + + pub async fn process_routing_context_request( + &self, + routing_context: RoutingContext, + rcr: RoutingContextRequest, + ) -> RoutingContextResponse { + let rc_op = match rcr.rc_op { + RoutingContextRequestOp::Release => { + self.release_routing_context(rcr.rc_id); + RoutingContextResponseOp::Release {} + } + RoutingContextRequestOp::WithPrivacy => RoutingContextResponseOp::WithPrivacy { + result: to_json_api_result( + routing_context + .clone() + .with_privacy() + .map(|new_rc| self.add_routing_context(new_rc)), + ), + }, + RoutingContextRequestOp::WithCustomPrivacy { stability } => { + RoutingContextResponseOp::WithCustomPrivacy { + result: to_json_api_result( + routing_context + .clone() + .with_custom_privacy(stability) + .map(|new_rc| self.add_routing_context(new_rc)), + ), + } + } + RoutingContextRequestOp::WithSequencing { sequencing } => { + RoutingContextResponseOp::WithSequencing { + value: self + .add_routing_context(routing_context.clone().with_sequencing(sequencing)), + } + } + RoutingContextRequestOp::AppCall { target, request } => { + RoutingContextResponseOp::AppCall { + result: to_json_api_result_with_vec_u8( + self.parse_target(target) + .then(|tr| async { routing_context.app_call(tr?, request).await }) + .await, + ), + } + } + RoutingContextRequestOp::AppMessage { target, message } => { + RoutingContextResponseOp::AppMessage { + result: to_json_api_result( + self.parse_target(target) + .then(|tr| async { routing_context.app_message(tr?, message).await }) + .await, + ), + } + } + RoutingContextRequestOp::CreateDhtRecord { kind, schema } => { + RoutingContextResponseOp::CreateDhtRecord { + result: to_json_api_result( + routing_context.create_dht_record(kind, schema).await, + ), + } + } + RoutingContextRequestOp::OpenDhtRecord { key, writer } => { + RoutingContextResponseOp::OpenDhtRecord { + result: to_json_api_result(routing_context.open_dht_record(key, writer).await), + } + } + RoutingContextRequestOp::CloseDhtRecord { key } => { + RoutingContextResponseOp::CloseDhtRecord { + result: to_json_api_result(routing_context.close_dht_record(key).await), + } + } + RoutingContextRequestOp::DeleteDhtRecord { key } => { + RoutingContextResponseOp::DeleteDhtRecord { + result: to_json_api_result(routing_context.delete_dht_record(key).await), + } + } + RoutingContextRequestOp::GetDhtValue { + key, + subkey, + force_refresh, + } => RoutingContextResponseOp::GetDhtValue { + result: to_json_api_result( + routing_context + .get_dht_value(key, subkey, force_refresh) + .await, + ), + }, + RoutingContextRequestOp::SetDhtValue { key, subkey, data } => { + RoutingContextResponseOp::SetDhtValue { + result: to_json_api_result( + routing_context.set_dht_value(key, subkey, data).await, + ), + } + } + RoutingContextRequestOp::WatchDhtValues { + key, + subkeys, + expiration, + count, + } => RoutingContextResponseOp::WatchDhtValues { + result: to_json_api_result( + routing_context + .watch_dht_values(key, subkeys, expiration, count) + .await, + ), + }, + RoutingContextRequestOp::CancelDhtWatch { key, subkeys } => { + RoutingContextResponseOp::CancelDhtWatch { + result: to_json_api_result( + routing_context.cancel_dht_watch(key, subkeys).await, + ), + } + } + }; + RoutingContextResponse { + rc_id: rcr.rc_id, + rc_op, + } + } + + pub async fn process_table_db_request( + &self, + table_db: TableDB, + tdr: TableDbRequest, + ) -> TableDbResponse { + let db_op = match tdr.db_op { + TableDbRequestOp::Release => { + self.release_table_db(tdr.db_id); + TableDbResponseOp::Release {} + } + TableDbRequestOp::GetColumnCount => TableDbResponseOp::GetColumnCount { + result: to_json_api_result(table_db.get_column_count()), + }, + TableDbRequestOp::GetKeys { col } => TableDbResponseOp::GetKeys { + result: to_json_api_result_with_vec_vec_u8(table_db.get_keys(col).await), + }, + TableDbRequestOp::Transact => TableDbResponseOp::Transact { + value: self.add_table_db_transaction(table_db.transact()), + }, + TableDbRequestOp::Store { col, key, value } => TableDbResponseOp::Store { + result: to_json_api_result(table_db.store(col, &key, &value).await), + }, + TableDbRequestOp::Load { col, key } => TableDbResponseOp::Load { + result: to_json_api_result( + table_db + .load(col, &key) + .await + .map(|vopt| vopt.map(|v| VecU8 { value: v })), + ), + }, + TableDbRequestOp::Delete { col, key } => TableDbResponseOp::Delete { + result: to_json_api_result( + table_db + .delete(col, &key) + .await + .map(|vopt| vopt.map(|v| VecU8 { value: v })), + ), + }, + }; + TableDbResponse { + db_id: tdr.db_id, + db_op, + } + } + + pub async fn process_table_db_transaction_request( + &self, + table_db_transaction: TableDBTransaction, + tdtr: TableDbTransactionRequest, + ) -> TableDbTransactionResponse { + let tx_op = match tdtr.tx_op { + TableDbTransactionRequestOp::Commit => TableDbTransactionResponseOp::Commit { + result: to_json_api_result(table_db_transaction.commit().await.map(|_| { + self.release_table_db_transaction(tdtr.tx_id); + })), + }, + TableDbTransactionRequestOp::Rollback => { + table_db_transaction.rollback(); + self.release_table_db_transaction(tdtr.tx_id); + TableDbTransactionResponseOp::Rollback {} + } + TableDbTransactionRequestOp::Store { col, key, value } => { + table_db_transaction.store(col, &key, &value); + TableDbTransactionResponseOp::Store {} + } + TableDbTransactionRequestOp::Delete { col, key } => { + table_db_transaction.delete(col, &key); + TableDbTransactionResponseOp::Delete {} + } + }; + TableDbTransactionResponse { + tx_id: tdtr.tx_id, + tx_op, + } + } + + pub async fn process_request(&self, request: Request) -> Response { + let id = request.id; + + let op = match request.op { + RequestOp::GetState => ResponseOp::GetState { + result: to_json_api_result(self.api.get_state().await), + }, + RequestOp::Attach => ResponseOp::Attach { + result: to_json_api_result(self.api.attach().await), + }, + RequestOp::Detach => ResponseOp::Detach { + result: to_json_api_result(self.api.detach().await), + }, + RequestOp::NewPrivateRoute => ResponseOp::NewPrivateRoute { + result: to_json_api_result(self.api.new_private_route().await.map(|r| { + NewPrivateRouteResult { + route_id: r.0, + blob: r.1, + } + })), + }, + RequestOp::NewCustomPrivateRoute { + kinds, + stability, + sequencing, + } => ResponseOp::NewCustomPrivateRoute { + result: to_json_api_result( + self.api + .new_custom_private_route(&kinds, stability, sequencing) + .await + .map(|r| NewPrivateRouteResult { + route_id: r.0, + blob: r.1, + }), + ), + }, + RequestOp::ImportRemotePrivateRoute { blob } => ResponseOp::ImportRemotePrivateRoute { + result: to_json_api_result_with_string(self.api.import_remote_private_route(blob)), + }, + RequestOp::ReleasePrivateRoute { route_id } => ResponseOp::ReleasePrivateRoute { + result: to_json_api_result(self.api.release_private_route(route_id)), + }, + RequestOp::AppCallReply { call_id, message } => ResponseOp::AppCallReply { + result: to_json_api_result(self.api.app_call_reply(call_id, message).await), + }, + RequestOp::NewRoutingContext => ResponseOp::NewRoutingContext { + value: self.add_routing_context(self.api.routing_context()), + }, + RequestOp::RoutingContext(rcr) => { + let routing_context = match self.lookup_routing_context(id, rcr.rc_id) { + Ok(v) => v, + Err(e) => return e, + }; + ResponseOp::RoutingContext( + self.process_routing_context_request(routing_context, rcr) + .await, + ) + } + RequestOp::OpenTableDb { name, column_count } => { + let table_store = match self.api.table_store() { + Ok(v) => v, + Err(e) => { + return Response { + id, + op: ResponseOp::OpenTableDb { + result: to_json_api_result(Err(e)), + }, + } + } + }; + ResponseOp::OpenTableDb { + result: to_json_api_result( + table_store + .open(&name, column_count) + .await + .map(|table_db| self.add_table_db(table_db)), + ), + } + } + RequestOp::DeleteTableDb { name } => { + let table_store = match self.api.table_store() { + Ok(v) => v, + Err(e) => { + return Response { + id, + op: ResponseOp::OpenTableDb { + result: to_json_api_result(Err(e)), + }, + } + } + }; + ResponseOp::DeleteTableDb { + result: to_json_api_result(table_store.delete(&name).await), + } + } + RequestOp::TableDb(tdr) => { + let table_db = match self.lookup_table_db(id, tdr.db_id) { + Ok(v) => v, + Err(e) => return e, + }; + ResponseOp::TableDb(self.process_table_db_request(table_db, tdr).await) + } + RequestOp::TableDbTransaction(tdtr) => { + let table_db_transaction = match self.lookup_table_db_transaction(id, tdtr.tx_id) { + Ok(v) => v, + Err(e) => return e, + }; + ResponseOp::TableDbTransaction( + self.process_table_db_transaction_request(table_db_transaction, tdtr) + .await, + ) + } + RequestOp::GetCryptoSystem { kind } => todo!(), + RequestOp::BestCryptoSystem => todo!(), + RequestOp::CryptoSystem(_) => todo!(), + RequestOp::VerifySignatures { + node_ids, + data, + signatures, + } => todo!(), + RequestOp::GenerateSignatures { data, key_pairs } => todo!(), + RequestOp::GenerateKeyPair { kind } => todo!(), + RequestOp::Now => todo!(), + RequestOp::Debug { command } => todo!(), + RequestOp::VeilidVersionString => todo!(), + RequestOp::VeilidVersion => todo!(), + }; + + Response { id, op } + } +} diff --git a/veilid-core/src/veilid_api/json_api/routing_context.rs b/veilid-core/src/veilid_api/json_api/routing_context.rs index 9376d81f..94c55a9e 100644 --- a/veilid-core/src/veilid_api/json_api/routing_context.rs +++ b/veilid-core/src/veilid_api/json_api/routing_context.rs @@ -2,16 +2,16 @@ use super::*; #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct RoutingContextRequest { - rc_id: String, + pub rc_id: u32, #[serde(flatten)] - rc_op: RoutingContextRequestOp, + pub rc_op: RoutingContextRequestOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct RoutingContextResponse { - rc_id: String, + pub rc_id: u32, #[serde(flatten)] - rc_op: RoutingContextResponseOp, + pub rc_op: RoutingContextResponseOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -86,15 +86,18 @@ pub enum RoutingContextRequestOp { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "rc_op")] pub enum RoutingContextResponseOp { + InvalidId, Release, WithPrivacy { - value: String, + #[serde(flatten)] + result: ApiResult, }, WithCustomPrivacy { - value: String, + #[serde(flatten)] + result: ApiResult, }, WithSequencing { - value: String, + value: u32, }, AppCall { #[serde(flatten)] diff --git a/veilid-core/src/veilid_api/json_api/table_db.rs b/veilid-core/src/veilid_api/json_api/table_db.rs index 483c5d71..e2741730 100644 --- a/veilid-core/src/veilid_api/json_api/table_db.rs +++ b/veilid-core/src/veilid_api/json_api/table_db.rs @@ -2,16 +2,16 @@ use super::*; #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct TableDbRequest { - db_id: String, + pub db_id: u32, #[serde(flatten)] - db_op: TableDbRequestOp, + pub db_op: TableDbRequestOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct TableDbResponse { - db_id: String, + pub db_id: u32, #[serde(flatten)] - db_op: TableDbResponseOp, + pub db_op: TableDbResponseOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -20,11 +20,11 @@ pub enum TableDbRequestOp { Release, GetColumnCount, GetKeys { - col: i32, + col: u32, }, Transact, Store { - col: i32, + col: u32, #[serde(with = "json_as_base64")] #[schemars(with = "String")] key: Vec, @@ -33,13 +33,13 @@ pub enum TableDbRequestOp { value: Vec, }, Load { - col: i32, + col: u32, #[serde(with = "json_as_base64")] #[schemars(with = "String")] key: Vec, }, Delete { - col: i32, + col: u32, #[serde(with = "json_as_base64")] #[schemars(with = "String")] key: Vec, @@ -48,9 +48,11 @@ pub enum TableDbRequestOp { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "db_op")] pub enum TableDbResponseOp { + InvalidId, Release, GetColumnCount { - value: i32, + #[serde(flatten)] + result: ApiResult, }, GetKeys { #[serde(flatten)] @@ -58,7 +60,7 @@ pub enum TableDbResponseOp { result: ApiResultWithVecVecU8, }, Transact { - value: String, + value: u32, }, Store { #[serde(flatten)] @@ -80,16 +82,16 @@ pub enum TableDbResponseOp { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct TableDbTransactionRequest { - tx_id: String, + pub tx_id: u32, #[serde(flatten)] - tx_op: TableDbTransactionRequestOp, + pub tx_op: TableDbTransactionRequestOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct TableDbTransactionResponse { - tx_id: String, + pub tx_id: u32, #[serde(flatten)] - tx_op: TableDbTransactionResponseOp, + pub tx_op: TableDbTransactionResponseOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -98,7 +100,7 @@ pub enum TableDbTransactionRequestOp { Commit, Rollback, Store { - col: i32, + col: u32, #[serde(with = "json_as_base64")] #[schemars(with = "String")] key: Vec, @@ -107,7 +109,7 @@ pub enum TableDbTransactionRequestOp { value: Vec, }, Delete { - col: i32, + col: u32, #[serde(with = "json_as_base64")] #[schemars(with = "String")] key: Vec, @@ -116,20 +118,12 @@ pub enum TableDbTransactionRequestOp { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "tx_op")] pub enum TableDbTransactionResponseOp { + InvalidId, Commit { #[serde(flatten)] result: ApiResult<()>, }, - Rollback { - #[serde(flatten)] - result: ApiResult<()>, - }, - Store { - #[serde(flatten)] - result: ApiResult<()>, - }, - Delete { - #[serde(flatten)] - result: ApiResult<()>, - }, + Rollback {}, + Store {}, + Delete {}, } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index 9f05e56d..9c8e4e2d 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -3,17 +3,16 @@ mod api; mod debug; mod error; -mod json_api; mod routing_context; mod serialize_helpers; mod types; +pub mod json_api; pub mod tests; pub use api::*; pub use debug::*; pub use error::*; -pub use json_api::*; pub use routing_context::*; pub use serialize_helpers::*; pub use types::*; From 771a7c9e7e538f8690598a9bf0b610e49d52de26 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 6 Jun 2023 18:48:37 -0400 Subject: [PATCH 05/23] json api --- .../src/veilid_api/json_api/crypto_system.rs | 20 +- veilid-core/src/veilid_api/json_api/mod.rs | 8 +- .../src/veilid_api/json_api/process.rs | 279 +++++++++++++++++- 3 files changed, 284 insertions(+), 23 deletions(-) diff --git a/veilid-core/src/veilid_api/json_api/crypto_system.rs b/veilid-core/src/veilid_api/json_api/crypto_system.rs index 58d5107f..5fa499c8 100644 --- a/veilid-core/src/veilid_api/json_api/crypto_system.rs +++ b/veilid-core/src/veilid_api/json_api/crypto_system.rs @@ -74,9 +74,8 @@ pub enum CryptoSystemRequestOp { #[serde(with = "json_as_base64")] #[schemars(with = "String")] data: Vec, - #[serde(with = "json_as_base64")] #[schemars(with = "String")] - hash_digest: Vec, + hash_digest: HashDigest, }, Distance { #[schemars(with = "String")] @@ -153,9 +152,9 @@ pub enum CryptoSystemResponseOp { result: ApiResultWithString, }, RandomBytes { - #[serde(flatten)] - #[schemars(with = "ApiResult")] - result: ApiResultWithVecU8, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + value: Vec, }, DefaultSaltLength { value: u32, @@ -200,8 +199,9 @@ pub enum CryptoSystemResponseOp { value: CryptoKeyDistance, }, Sign { - #[schemars(with = "String")] - value: Signature, + #[serde(flatten)] + #[schemars(with = "ApiResult")] + result: ApiResultWithString, }, Verify { #[serde(flatten)] @@ -221,8 +221,8 @@ pub enum CryptoSystemResponseOp { result: ApiResultWithVecU8, }, CryptNoAuth { - #[serde(flatten)] - #[schemars(with = "ApiResult")] - result: ApiResultWithVecU8, + #[serde(with = "json_as_base64")] + #[schemars(with = "String")] + value: Vec, }, } diff --git a/veilid-core/src/veilid_api/json_api/mod.rs b/veilid-core/src/veilid_api/json_api/mod.rs index c0ec39bc..01257559 100644 --- a/veilid-core/src/veilid_api/json_api/mod.rs +++ b/veilid-core/src/veilid_api/json_api/mod.rs @@ -186,7 +186,8 @@ pub enum ResponseOp { result: ApiResult, }, BestCryptoSystem { - value: u32, + #[serde(flatten)] + result: ApiResult, }, CryptoSystem(CryptoSystemResponse), VerifySignatures { @@ -197,7 +198,7 @@ pub enum ResponseOp { GenerateSignatures { #[serde(flatten)] #[schemars(with = "ApiResult>")] - result: ApiResultWithVecString, + result: ApiResultWithVecString>, }, GenerateKeyPair { #[serde(flatten)] @@ -210,7 +211,8 @@ pub enum ResponseOp { value: Timestamp, }, Debug { - value: String, + #[serde(flatten)] + result: ApiResult, }, VeilidVersionString { value: String, diff --git a/veilid-core/src/veilid_api/json_api/process.rs b/veilid-core/src/veilid_api/json_api/process.rs index cc635005..15304f84 100644 --- a/veilid-core/src/veilid_api/json_api/process.rs +++ b/veilid-core/src/veilid_api/json_api/process.rs @@ -19,6 +19,15 @@ fn to_json_api_result_with_string( } } +fn to_json_api_result_with_vec_string( + r: VeilidAPIResult, +) -> json_api::ApiResultWithVecString { + match r { + Err(e) => json_api::ApiResultWithVecString::Err { error: e }, + Ok(v) => json_api::ApiResultWithVecString::Ok { value: v }, + } +} + fn to_json_api_result_with_vec_u8(r: VeilidAPIResult>) -> json_api::ApiResultWithVecU8 { match r { Err(e) => json_api::ApiResultWithVecU8::Err { error: e }, @@ -42,6 +51,7 @@ pub struct JsonRequestProcessor { routing_contexts: Mutex>, table_dbs: Mutex>, table_db_transactions: Mutex>, + crypto_systems: Mutex>, } impl JsonRequestProcessor { @@ -51,6 +61,7 @@ impl JsonRequestProcessor { routing_contexts: Default::default(), table_dbs: Default::default(), table_db_transactions: Default::default(), + crypto_systems: Default::default(), } } @@ -151,6 +162,37 @@ impl JsonRequestProcessor { return 1; } + // CryptoSystem + fn add_crypto_system(&self, csv: CryptoSystemVersion) -> u32 { + let mut next_id: u32 = 1; + let mut crypto_systems = self.crypto_systems.lock(); + while crypto_systems.contains_key(&next_id) { + next_id += 1; + } + crypto_systems.insert(next_id, csv); + next_id + } + fn lookup_crypto_system(&self, id: u32, cs_id: u32) -> Result { + let crypto_systems = self.crypto_systems.lock(); + let Some(crypto_system) = crypto_systems.get(&cs_id).cloned() else { + return Err(Response { + id, + op: ResponseOp::CryptoSystem(CryptoSystemResponse { + cs_id, + cs_op: CryptoSystemResponseOp::InvalidId + }) + }); + }; + Ok(crypto_system) + } + fn release_crypto_system(&self, id: u32) -> i32 { + let mut crypto_systems = self.crypto_systems.lock(); + if crypto_systems.remove(&id).is_none() { + return 0; + } + return 1; + } + // Target // Parse target @@ -370,6 +412,122 @@ impl JsonRequestProcessor { } } + pub async fn process_crypto_system_request( + &self, + csv: CryptoSystemVersion, + csr: CryptoSystemRequest, + ) -> CryptoSystemResponse { + let cs_op = match csr.cs_op { + CryptoSystemRequestOp::Release => { + self.release_crypto_system(csr.cs_id); + CryptoSystemResponseOp::Release {} + } + CryptoSystemRequestOp::CachedDh { key, secret } => CryptoSystemResponseOp::CachedDh { + result: to_json_api_result_with_string(csv.cached_dh(&key, &secret)), + }, + CryptoSystemRequestOp::ComputeDh { key, secret } => CryptoSystemResponseOp::ComputeDh { + result: to_json_api_result_with_string(csv.compute_dh(&key, &secret)), + }, + CryptoSystemRequestOp::RandomBytes { len } => CryptoSystemResponseOp::RandomBytes { + value: csv.random_bytes(len), + }, + CryptoSystemRequestOp::DefaultSaltLength => CryptoSystemResponseOp::DefaultSaltLength { + value: csv.default_salt_length(), + }, + CryptoSystemRequestOp::HashPassword { password, salt } => { + CryptoSystemResponseOp::HashPassword { + result: to_json_api_result(csv.hash_password(&password, &salt)), + } + } + CryptoSystemRequestOp::VerifyPassword { + password, + password_hash, + } => CryptoSystemResponseOp::VerifyPassword { + result: to_json_api_result(csv.verify_password(&password, &password_hash)), + }, + CryptoSystemRequestOp::DeriveSharedSecret { password, salt } => { + CryptoSystemResponseOp::DeriveSharedSecret { + result: to_json_api_result_with_string( + csv.derive_shared_secret(&password, &salt), + ), + } + } + CryptoSystemRequestOp::RandomNonce => CryptoSystemResponseOp::RandomNonce { + value: csv.random_nonce(), + }, + CryptoSystemRequestOp::RandomSharedSecret => { + CryptoSystemResponseOp::RandomSharedSecret { + value: csv.random_shared_secret(), + } + } + CryptoSystemRequestOp::GenerateKeyPair => CryptoSystemResponseOp::GenerateKeyPair { + value: csv.generate_keypair(), + }, + CryptoSystemRequestOp::GenerateHash { data } => CryptoSystemResponseOp::GenerateHash { + value: csv.generate_hash(&data), + }, + CryptoSystemRequestOp::ValidateKeyPair { key, secret } => { + CryptoSystemResponseOp::ValidateKeyPair { + value: csv.validate_keypair(&key, &secret), + } + } + CryptoSystemRequestOp::ValidateHash { data, hash_digest } => { + CryptoSystemResponseOp::ValidateHash { + value: csv.validate_hash(&data, &hash_digest), + } + } + CryptoSystemRequestOp::Distance { key1, key2 } => CryptoSystemResponseOp::Distance { + value: csv.distance(&key1, &key2), + }, + CryptoSystemRequestOp::Sign { key, secret, data } => CryptoSystemResponseOp::Sign { + result: to_json_api_result_with_string(csv.sign(&key, &secret, &data)), + }, + CryptoSystemRequestOp::Verify { key, data, secret } => CryptoSystemResponseOp::Verify { + result: to_json_api_result(csv.verify(&key, &data, &secret)), + }, + CryptoSystemRequestOp::AeadOverhead => CryptoSystemResponseOp::AeadOverhead { + value: csv.aead_overhead() as u32, + }, + CryptoSystemRequestOp::DecryptAead { + body, + nonce, + shared_secret, + associated_data, + } => CryptoSystemResponseOp::DecryptAead { + result: to_json_api_result_with_vec_u8(csv.decrypt_aead( + &body, + &nonce, + &shared_secret, + associated_data.as_ref().map(|ad| ad.as_slice()), + )), + }, + CryptoSystemRequestOp::EncryptAead { + body, + nonce, + shared_secret, + associated_data, + } => CryptoSystemResponseOp::EncryptAead { + result: to_json_api_result_with_vec_u8(csv.encrypt_aead( + &body, + &nonce, + &shared_secret, + associated_data.as_ref().map(|ad| ad.as_slice()), + )), + }, + CryptoSystemRequestOp::CryptNoAuth { + body, + nonce, + shared_secret, + } => CryptoSystemResponseOp::CryptNoAuth { + value: csv.crypt_no_auth_unaligned(&body, &nonce, &shared_secret), + }, + }; + CryptoSystemResponse { + cs_id: csr.cs_id, + cs_op, + } + } + pub async fn process_request(&self, request: Request) -> Response { let id = request.id; @@ -482,20 +640,121 @@ impl JsonRequestProcessor { .await, ) } - RequestOp::GetCryptoSystem { kind } => todo!(), - RequestOp::BestCryptoSystem => todo!(), - RequestOp::CryptoSystem(_) => todo!(), + RequestOp::GetCryptoSystem { kind } => { + let crypto = match self.api.crypto() { + Ok(v) => v, + Err(e) => { + return Response { + id, + op: ResponseOp::GetCryptoSystem { + result: to_json_api_result(Err(e)), + }, + } + } + }; + ResponseOp::GetCryptoSystem { + result: to_json_api_result( + crypto + .get(kind) + .ok_or_else(|| { + VeilidAPIError::invalid_argument( + "unsupported cryptosystem", + "kind", + kind, + ) + }) + .map(|csv| self.add_crypto_system(csv)), + ), + } + } + RequestOp::BestCryptoSystem => { + let crypto = match self.api.crypto() { + Ok(v) => v, + Err(e) => { + return Response { + id, + op: ResponseOp::GetCryptoSystem { + result: to_json_api_result(Err(e)), + }, + } + } + }; + ResponseOp::BestCryptoSystem { + result: to_json_api_result(Ok(self.add_crypto_system(crypto.best()))), + } + } + RequestOp::CryptoSystem(csr) => { + let csv = match self.lookup_crypto_system(id, csr.cs_id) { + Ok(v) => v, + Err(e) => return e, + }; + ResponseOp::CryptoSystem(self.process_crypto_system_request(csv, csr).await) + } RequestOp::VerifySignatures { node_ids, data, signatures, - } => todo!(), - RequestOp::GenerateSignatures { data, key_pairs } => todo!(), - RequestOp::GenerateKeyPair { kind } => todo!(), - RequestOp::Now => todo!(), - RequestOp::Debug { command } => todo!(), - RequestOp::VeilidVersionString => todo!(), - RequestOp::VeilidVersion => todo!(), + } => { + let crypto = match self.api.crypto() { + Ok(v) => v, + Err(e) => { + return Response { + id, + op: ResponseOp::GetCryptoSystem { + result: to_json_api_result(Err(e)), + }, + } + } + }; + ResponseOp::VerifySignatures { + result: to_json_api_result_with_vec_string(crypto.verify_signatures( + &node_ids, + &data, + &signatures, + )), + } + } + RequestOp::GenerateSignatures { data, key_pairs } => { + let crypto = match self.api.crypto() { + Ok(v) => v, + Err(e) => { + return Response { + id, + op: ResponseOp::GetCryptoSystem { + result: to_json_api_result(Err(e)), + }, + } + } + }; + ResponseOp::GenerateSignatures { + result: to_json_api_result_with_vec_string(crypto.generate_signatures( + &data, + &key_pairs, + |k, s| TypedSignature::new(k.kind, s), + )), + } + } + RequestOp::GenerateKeyPair { kind } => ResponseOp::GenerateKeyPair { + result: to_json_api_result_with_string(Crypto::generate_keypair(kind)), + }, + RequestOp::Now => ResponseOp::Now { + value: get_aligned_timestamp(), + }, + RequestOp::Debug { command } => ResponseOp::Debug { + result: to_json_api_result(self.api.debug(command).await), + }, + RequestOp::VeilidVersionString => ResponseOp::VeilidVersionString { + value: veilid_version_string(), + }, + RequestOp::VeilidVersion => { + let (major, minor, patch) = veilid_version(); + + ResponseOp::VeilidVersion { + major, + minor, + patch, + } + } }; Response { id, op } From 88db69c28f2bad73bc5a584411017e17c0198f22 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 6 Jun 2023 19:09:29 -0400 Subject: [PATCH 06/23] checkpoint --- Cargo.lock | 3 - veilid-server/Cargo.toml | 8 +- veilid-server/build.rs | 6 - veilid-server/proto/veilid-client.capnp | 25 -- veilid-server/src/client_api.rs | 378 ++++-------------------- veilid-server/src/main.rs | 7 +- 6 files changed, 63 insertions(+), 364 deletions(-) delete mode 100644 veilid-server/build.rs delete mode 100644 veilid-server/proto/veilid-client.capnp diff --git a/Cargo.lock b/Cargo.lock index 9866cef7..556f52af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6329,9 +6329,6 @@ dependencies = [ "async-tungstenite 0.22.2", "backtrace", "bugsalot", - "capnp", - "capnp-rpc", - "capnpc", "cfg-if 1.0.0", "clap 3.2.25", "color-eyre", diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 977e55a1..57dd2ba1 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -3,7 +3,6 @@ name = "veilid-server" version = "0.1.0" authors = ["John Smith "] edition = "2021" -build = "build.rs" license = "LGPL-2.0-or-later OR MPL-2.0 OR (MIT AND BSD-3-Clause)" [[bin]] @@ -39,9 +38,7 @@ color-eyre = { version = "^0", default-features = false } backtrace = "^0" clap = "^3" directories = "^4" -capnp = "^0" parking_lot = "^0" -capnp-rpc = "^0" config = { version = "^0", features = ["yaml"] } cfg-if = "^1" serde = "^1" @@ -73,7 +70,4 @@ nix = "^0" tracing-journald = "^0" [dev-dependencies] -serial_test = "^0" - -[build-dependencies] -capnpc = "^0" +serial_test = "^0" \ No newline at end of file diff --git a/veilid-server/build.rs b/veilid-server/build.rs deleted file mode 100644 index a9e4db80..00000000 --- a/veilid-server/build.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn main() { - ::capnpc::CompilerCommand::new() - .file("proto/veilid-client.capnp") - .run() - .expect("compiling schema"); -} diff --git a/veilid-server/proto/veilid-client.capnp b/veilid-server/proto/veilid-client.capnp deleted file mode 100644 index 469a3b09..00000000 --- a/veilid-server/proto/veilid-client.capnp +++ /dev/null @@ -1,25 +0,0 @@ -@0xd29582d26b2fb073; - -struct ApiResult @0x8111724bdb812929 { - union { - ok @0 :Text; - err @1 :Text; - } -} - -interface Registration @0xdd45f30a7c22e391 {} - -interface VeilidServer @0xcb2c699f14537f94 { - register @0 (veilidClient :VeilidClient) -> (registration :Registration, state :Text, settings :Text); - debug @1 (command :Text) -> (result :ApiResult); - attach @2 () -> (result :ApiResult); - detach @3 () -> (result :ApiResult); - shutdown @4 (); - getState @5 () -> (result :ApiResult); - changeLogLevel @6 (layer :Text, logLevel :Text) -> (result :ApiResult); - appCallReply @7 (id :UInt64, message :Data) -> (result :ApiResult); -} - -interface VeilidClient @0xbfcea60fb2ba4736 { - update @0 (veilidUpdate :Text); -} \ No newline at end of file diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index 269dce3a..b3bc15cc 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -2,8 +2,6 @@ use crate::settings::*; use crate::tools::*; use crate::veilid_client_capnp::*; use crate::veilid_logs::VeilidLogs; -use capnp::capability::Promise; -use capnp_rpc::{pry, rpc_twoparty_capnp, twoparty, RpcSystem}; use cfg_if::*; use futures_util::{future::try_join_all, FutureExt as FuturesFutureExt, StreamExt}; use serde::*; @@ -11,267 +9,72 @@ use std::cell::RefCell; use std::collections::HashMap; use std::fmt; use std::net::SocketAddr; -use std::rc::Rc; use stop_token::future::FutureExt; use stop_token::*; use tracing::*; use veilid_core::*; -// Encoding for ApiResult -fn encode_api_result( - result: &VeilidAPIResult, - builder: &mut api_result::Builder, -) { - match result { - Ok(v) => { - builder.set_ok(&serialize_json(v)); - } - Err(e) => { - builder.set_err(&serialize_json(e)); - } - } -} +// struct VeilidServerImpl { +// veilid_api: veilid_core::VeilidAPI, +// veilid_logs: VeilidLogs, +// settings: Settings, +// next_id: u64, +// } -// --- interface Registration --------------------------------- +// impl VeilidServerImpl { +// #[instrument(level = "trace", skip_all)] +// pub fn new( +// veilid_api: veilid_core::VeilidAPI, +// veilid_logs: VeilidLogs, +// settings: Settings, +// ) -> Self { +// Self { +// next_id: 0, +// veilid_api, +// veilid_logs, +// settings, +// } +// } -struct RegistrationHandle { - client: veilid_client::Client, - requests_in_flight: i32, -} +// #[instrument(level = "trace", skip_all)] +// fn shutdown( +// &mut self, +// _params: veilid_server::ShutdownParams, +// mut _results: veilid_server::ShutdownResults, +// ) -> Promise<(), ::capnp::Error> { +// trace!("VeilidServerImpl::shutdown"); -struct RegistrationMap { - registrations: HashMap, -} +// cfg_if::cfg_if! { +// if #[cfg(windows)] { +// assert!(false, "write me!"); +// } +// else { +// crate::server::shutdown(); +// } +// } -impl RegistrationMap { - fn new() -> Self { - Self { - registrations: HashMap::new(), - } - } -} +// Promise::ok(()) +// } -struct RegistrationImpl { - id: u64, - registration_map: Rc>, -} +// #[instrument(level = "trace", skip_all)] +// fn change_log_level( +// &mut self, +// params: veilid_server::ChangeLogLevelParams, +// mut results: veilid_server::ChangeLogLevelResults, +// ) -> Promise<(), ::capnp::Error> { +// trace!("VeilidServerImpl::change_log_level"); -impl RegistrationImpl { - fn new(id: u64, registrations: Rc>) -> Self { - Self { - id, - registration_map: registrations, - } - } -} +// let layer = pry!(pry!(params.get()).get_layer()).to_owned(); +// let log_level_json = pry!(pry!(params.get()).get_log_level()).to_owned(); +// let log_level: veilid_core::VeilidConfigLogLevel = +// pry!(veilid_core::deserialize_json(&log_level_json) +// .map_err(|e| ::capnp::Error::failed(format!("{:?}", e)))); -impl Drop for RegistrationImpl { - fn drop(&mut self) { - debug!("Registration dropped"); - self.registration_map - .borrow_mut() - .registrations - .remove(&self.id); - } -} - -impl registration::Server for RegistrationImpl {} - -// --- interface VeilidServer --------------------------------- - -struct VeilidServerImpl { - veilid_api: veilid_core::VeilidAPI, - veilid_logs: VeilidLogs, - settings: Settings, - next_id: u64, - pub registration_map: Rc>, -} - -impl VeilidServerImpl { - #[instrument(level = "trace", skip_all)] - pub fn new( - veilid_api: veilid_core::VeilidAPI, - veilid_logs: VeilidLogs, - settings: Settings, - ) -> Self { - Self { - next_id: 0, - registration_map: Rc::new(RefCell::new(RegistrationMap::new())), - veilid_api, - veilid_logs, - settings, - } - } -} - -impl veilid_server::Server for VeilidServerImpl { - #[instrument(level = "trace", skip_all)] - fn register( - &mut self, - params: veilid_server::RegisterParams, - mut results: veilid_server::RegisterResults, - ) -> Promise<(), ::capnp::Error> { - trace!("VeilidServerImpl::register"); - - self.registration_map.borrow_mut().registrations.insert( - self.next_id, - RegistrationHandle { - client: pry!(pry!(params.get()).get_veilid_client()), - requests_in_flight: 0, - }, - ); - - let veilid_api = self.veilid_api.clone(); - let settings = self.settings.clone(); - let registration = capnp_rpc::new_client(RegistrationImpl::new( - self.next_id, - self.registration_map.clone(), - )); - self.next_id += 1; - - Promise::from_future(async move { - let state = veilid_api - .get_state() - .await - .map_err(|e| ::capnp::Error::failed(format!("{:?}", e)))?; - let state = serialize_json(state); - - let mut res = results.get(); - res.set_registration(registration); - res.set_state(&state); - - let settings = &*settings.read(); - let settings_json_string = serialize_json(settings); - let mut settings_json = json::parse(&settings_json_string) - .map_err(|e| ::capnp::Error::failed(format!("{:?}", e)))?; - settings_json["core"]["network"].remove("node_id_secret"); - let safe_settings_json = settings_json.to_string(); - res.set_settings(&safe_settings_json); - - Ok(()) - }) - } - - #[instrument(level = "trace", skip_all)] - fn debug( - &mut self, - params: veilid_server::DebugParams, - mut results: veilid_server::DebugResults, - ) -> Promise<(), ::capnp::Error> { - trace!("VeilidServerImpl::debug"); - let veilid_api = self.veilid_api.clone(); - let command = pry!(pry!(params.get()).get_command()).to_owned(); - - Promise::from_future(async move { - let result = veilid_api.debug(command).await; - encode_api_result(&result, &mut results.get().init_result()); - Ok(()) - }) - } - - #[instrument(level = "trace", skip_all)] - fn attach( - &mut self, - _params: veilid_server::AttachParams, - mut results: veilid_server::AttachResults, - ) -> Promise<(), ::capnp::Error> { - trace!("VeilidServerImpl::attach"); - let veilid_api = self.veilid_api.clone(); - Promise::from_future(async move { - let result = veilid_api.attach().await; - encode_api_result(&result, &mut results.get().init_result()); - Ok(()) - }) - } - - #[instrument(level = "trace", skip_all)] - fn detach( - &mut self, - _params: veilid_server::DetachParams, - mut results: veilid_server::DetachResults, - ) -> Promise<(), ::capnp::Error> { - trace!("VeilidServerImpl::detach"); - let veilid_api = self.veilid_api.clone(); - Promise::from_future(async move { - let result = veilid_api.detach().await; - encode_api_result(&result, &mut results.get().init_result()); - Ok(()) - }) - } - - #[instrument(level = "trace", skip_all)] - fn shutdown( - &mut self, - _params: veilid_server::ShutdownParams, - mut _results: veilid_server::ShutdownResults, - ) -> Promise<(), ::capnp::Error> { - trace!("VeilidServerImpl::shutdown"); - - cfg_if::cfg_if! { - if #[cfg(windows)] { - assert!(false, "write me!"); - } - else { - crate::server::shutdown(); - } - } - - Promise::ok(()) - } - - #[instrument(level = "trace", skip_all)] - fn get_state( - &mut self, - _params: veilid_server::GetStateParams, - mut results: veilid_server::GetStateResults, - ) -> Promise<(), ::capnp::Error> { - trace!("VeilidServerImpl::get_state"); - let veilid_api = self.veilid_api.clone(); - Promise::from_future(async move { - let result = veilid_api.get_state().await; - encode_api_result(&result, &mut results.get().init_result()); - Ok(()) - }) - } - - #[instrument(level = "trace", skip_all)] - fn change_log_level( - &mut self, - params: veilid_server::ChangeLogLevelParams, - mut results: veilid_server::ChangeLogLevelResults, - ) -> Promise<(), ::capnp::Error> { - trace!("VeilidServerImpl::change_log_level"); - - let layer = pry!(pry!(params.get()).get_layer()).to_owned(); - let log_level_json = pry!(pry!(params.get()).get_log_level()).to_owned(); - let log_level: veilid_core::VeilidConfigLogLevel = - pry!(veilid_core::deserialize_json(&log_level_json) - .map_err(|e| ::capnp::Error::failed(format!("{:?}", e)))); - - let result = self.veilid_logs.change_log_level(layer, log_level); - encode_api_result(&result, &mut results.get().init_result()); - Promise::ok(()) - } - - #[instrument(level = "trace", skip_all)] - fn app_call_reply( - &mut self, - params: veilid_server::AppCallReplyParams, - mut results: veilid_server::AppCallReplyResults, - ) -> Promise<(), ::capnp::Error> { - trace!("VeilidServerImpl::app_call_reply"); - - let id = OperationId::new(pry!(params.get()).get_id()); - let message = pry!(pry!(params.get()).get_message()).to_owned(); - - let veilid_api = self.veilid_api.clone(); - Promise::from_future(async move { - let result = veilid_api.app_call_reply(id, message).await; - encode_api_result(&result, &mut results.get().init_result()); - Ok(()) - }) - } -} +// let result = self.veilid_logs.change_log_level(layer, log_level); +// encode_api_result(&result, &mut results.get().init_result()); +// Promise::ok(()) +// } +// } // --- Client API Server-Side --------------------------------- @@ -282,13 +85,12 @@ struct ClientApiInner { veilid_api: veilid_core::VeilidAPI, veilid_logs: VeilidLogs, settings: Settings, - registration_map: Rc>, stop: Option, join_handle: Option, } pub struct ClientApi { - inner: RefCell, + inner: Arc>, } impl ClientApi { @@ -303,7 +105,6 @@ impl ClientApi { veilid_api, veilid_logs, settings, - registration_map: Rc::new(RefCell::new(RegistrationMap::new())), stop: Some(StopSource::new()), join_handle: None, }), @@ -329,11 +130,10 @@ impl ClientApi { trace!("ClientApi::stop: stopped"); } - #[instrument(level = "trace", skip(self, client), err)] + #[instrument(level = "trace", skip(self), err)] async fn handle_incoming( - self: Rc, + self, bind_addr: SocketAddr, - client: veilid_server::Client, ) -> Result<(), Box> { let listener = TcpListener::bind(bind_addr).await?; debug!("Client API listening on: {:?}", bind_addr); @@ -347,7 +147,7 @@ impl ClientApi { } } - let stop_token = self.inner.borrow().stop.as_ref().unwrap().token(); + let stop_token = self.inner.lock().stop.as_ref().unwrap().token(); let incoming_loop = async move { while let Ok(Some(stream_result)) = incoming_stream.next().timeout_at(stop_token.clone()).await @@ -365,15 +165,8 @@ impl ClientApi { let writer = writer.compat_write(); } } - let network = twoparty::VatNetwork::new( - reader, - writer, - rpc_twoparty_capnp::Side::Server, - Default::default(), - ); - - let rpc_system = RpcSystem::new(Box::new(network), Some(client.clone().client)); + xxx spawn json_api handler spawn_local(rpc_system.map(drop)); } Ok::<(), Box>(()) @@ -382,44 +175,6 @@ impl ClientApi { incoming_loop.await } - #[instrument(level = "trace", skip_all)] - fn send_request_to_all_clients(self: Rc, request: F) - where - F: Fn(u64, &mut RegistrationHandle) -> Option<::capnp::capability::RemotePromise>, - T: capnp::traits::Pipelined + capnp::traits::Owned + 'static + Unpin, - { - // Send status update to each registered client - let registration_map = self.inner.borrow().registration_map.clone(); - let registration_map1 = registration_map.clone(); - let regs = &mut registration_map.borrow_mut().registrations; - for (&id, mut registration) in regs.iter_mut() { - if registration.requests_in_flight >= 256 { - println!( - "too many requests in flight: {}", - registration.requests_in_flight - ); - } - registration.requests_in_flight += 1; - - if let Some(request_promise) = request(id, registration) { - let registration_map2 = registration_map1.clone(); - spawn_local(request_promise.promise.map(move |r| match r { - Ok(_) => { - if let Some(ref mut s) = - registration_map2.borrow_mut().registrations.get_mut(&id) - { - s.requests_in_flight -= 1; - } - } - Err(e) => { - println!("Got error: {:?}. Dropping registation.", e); - registration_map2.borrow_mut().registrations.remove(&id); - } - })); - } - } - } - #[instrument(level = "trace", skip(self))] pub fn handle_update(self: Rc, veilid_update: veilid_core::VeilidUpdate) { // serialize update @@ -444,21 +199,10 @@ impl ClientApi { } #[instrument(level = "trace", skip(self))] - pub fn run(self: Rc, bind_addrs: Vec) { - // Create client api VeilidServer - let veilid_server_impl = VeilidServerImpl::new( - self.inner.borrow().veilid_api.clone(), - self.inner.borrow().veilid_logs.clone(), - self.inner.borrow().settings.clone(), - ); - self.inner.borrow_mut().registration_map = veilid_server_impl.registration_map.clone(); - - // Make a client object for the server to send to each rpc client - let client: veilid_server::Client = capnp_rpc::new_client(veilid_server_impl); - + pub fn run(&self, bind_addrs: Vec) { let bind_futures = bind_addrs .iter() - .map(|addr| self.clone().handle_incoming(*addr, client.clone())); + .map(|addr| self.clone().handle_incoming(*addr)); let bind_futures_join = try_join_all(bind_futures); self.inner.borrow_mut().join_handle = Some(spawn_local(bind_futures_join)); } diff --git a/veilid-server/src/main.rs b/veilid-server/src/main.rs index 31a60cc3..b228a68e 100644 --- a/veilid-server/src/main.rs +++ b/veilid-server/src/main.rs @@ -24,11 +24,6 @@ use tools::*; use tracing::*; use veilid_logs::*; -#[allow(clippy::all)] -pub mod veilid_client_capnp { - include!(concat!(env!("OUT_DIR"), "/proto/veilid_client_capnp.rs")); -} - #[instrument(err)] fn main() -> EyreResult<()> { #[cfg(windows)] @@ -76,7 +71,7 @@ fn main() -> EyreResult<()> { if matches.occurrences_of("emit-schema") != 0 { if let Some(esstr) = matches.value_of("emit-schema") { let mut schemas = HashMap::::new(); - veilid_core::emit_schemas(&mut schemas); + veilid_core::json_api::emit_schemas(&mut schemas); if let Some(schema) = schemas.get(esstr) { println!("{}", schema); From 317f036598388f1114b2f531fd64aa5d8e450ec3 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 7 Jun 2023 17:39:10 -0400 Subject: [PATCH 07/23] server api --- Cargo.lock | 46 +-- veilid-cli/Cargo.toml | 6 - veilid-cli/build.rs | 7 - veilid-cli/src/client_api_connection.rs | 22 - veilid-cli/src/main.rs | 5 - veilid-core/src/veilid_api/json_api/mod.rs | 15 +- .../src/veilid_api/json_api/process.rs | 96 +++-- veilid-core/src/veilid_config.rs | 7 +- veilid-server/Cargo.toml | 15 +- veilid-server/src/client_api.rs | 376 +++++++++++++----- veilid-server/src/server.rs | 6 +- veilid-server/src/tools.rs | 42 +- veilid-server/src/unix.rs | 23 +- veilid-tools/src/must_join_handle.rs | 16 + 14 files changed, 423 insertions(+), 259 deletions(-) delete mode 100644 veilid-cli/build.rs diff --git a/Cargo.lock b/Cargo.lock index 556f52af..22646141 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -845,27 +845,6 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13e2d432d1601d61d1e11140d04e9d239b5cf7316fa1106523c3d86eea19c29d" -[[package]] -name = "capnp-futures" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d520e0af228b92de357f230f4987ee4f9786f2b8aa24b9cfe53f5b11c17198" -dependencies = [ - "capnp", - "futures", -] - -[[package]] -name = "capnp-rpc" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab8e869783e491cbcc350427a5e775aa4d8a1deaa5198d74332957cfa430779" -dependencies = [ - "capnp", - "capnp-futures", - "futures", -] - [[package]] name = "capnpc" version = "0.17.1" @@ -5927,6 +5906,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "triomphe" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ee9bd9239c339d714d657fac840c6d2a4f9c45f4f9ec7b0975113458be78db" +dependencies = [ + "serde", + "stable_deref_trait", +] + [[package]] name = "trust-dns-proto" version = "0.22.0" @@ -6161,9 +6150,6 @@ dependencies = [ "async-std", "async-tungstenite 0.8.0", "bugsalot", - "capnp", - "capnp-rpc", - "capnpc", "cfg-if 1.0.0", "clap 3.2.25", "config", @@ -6366,6 +6352,7 @@ dependencies = [ "tracing-subscriber", "url", "veilid-core", + "wg", "windows-service", ] @@ -6676,6 +6663,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "wg" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f390449c16e0679435fc97a6b49d24e67f09dd05fea1de54db1b60902896d273" +dependencies = [ + "atomic-waker", + "parking_lot 0.12.1", + "triomphe", +] + [[package]] name = "which" version = "4.4.0" diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index f98dd173..cc58e0d3 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -3,7 +3,6 @@ name = "veilid-cli" version = "0.1.0" authors = ["John Smith "] edition = "2021" -build = "build.rs" license = "LGPL-2.0-or-later OR MPL-2.0 OR (MIT AND BSD-3-Clause)" [[bin]] @@ -36,8 +35,6 @@ serde = "^1" serde_derive = "^1" parking_lot = "^0" cfg-if = "^1" -capnp = "^0" -capnp-rpc = "^0" config = { version = "^0", features = ["yaml"] } bugsalot = { git = "https://github.com/crioux/bugsalot.git" } flexi_logger = { version = "^0", features = ["use_chrono_for_offset"] } @@ -49,6 +46,3 @@ json = "^0" [dev-dependencies] serial_test = "^0" - -[build-dependencies] -capnpc = "^0" diff --git a/veilid-cli/build.rs b/veilid-cli/build.rs deleted file mode 100644 index fa57d860..00000000 --- a/veilid-cli/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn main() { - ::capnpc::CompilerCommand::new() - .file("../veilid-server/proto/veilid-client.capnp") - .src_prefix("../veilid-server/") - .run() - .expect("compiling schema"); -} diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index b2681257..bdabcb96 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -1,8 +1,5 @@ use crate::command_processor::*; use crate::tools::*; -use crate::veilid_client_capnp::*; -use capnp::capability::Promise; -use capnp_rpc::{pry, rpc_twoparty_capnp, twoparty, Disconnector, RpcSystem}; use futures::future::FutureExt; use serde::de::DeserializeOwned; use std::cell::RefCell; @@ -11,25 +8,6 @@ use std::rc::Rc; use veilid_core::tools::*; use veilid_core::*; -macro_rules! capnp_failed { - ($ex:expr) => {{ - let msg = format!("Capnp Error: {}", $ex); - error!("{}", msg); - Promise::err(capnp::Error::failed(msg)) - }}; -} - -macro_rules! pry_result { - ($ex:expr) => { - match $ex { - Ok(v) => v, - Err(e) => { - return capnp_failed!(e); - } - } - }; -} - fn map_to_internal_error(e: T) -> VeilidAPIError { VeilidAPIError::Internal { message: e.to_string(), diff --git a/veilid-cli/src/main.rs b/veilid-cli/src/main.rs index 9239478f..73ac351d 100644 --- a/veilid-cli/src/main.rs +++ b/veilid-cli/src/main.rs @@ -18,11 +18,6 @@ mod settings; mod tools; mod ui; -#[allow(clippy::all)] -pub mod veilid_client_capnp { - include!(concat!(env!("OUT_DIR"), "/proto/veilid_client_capnp.rs")); -} - fn parse_command_line(default_config_path: &OsStr) -> Result { let matches = Command::new("veilid-cli") .version("0.1") diff --git a/veilid-core/src/veilid_api/json_api/mod.rs b/veilid-core/src/veilid_api/json_api/mod.rs index 01257559..4e0c4bde 100644 --- a/veilid-core/src/veilid_api/json_api/mod.rs +++ b/veilid-core/src/veilid_api/json_api/mod.rs @@ -16,10 +16,10 @@ pub use process::*; pub struct Request { /// Operation Id (pairs with Response, or empty if unidirectional) #[serde(default)] - id: u32, + pub id: u32, /// The request operation variant #[serde(flatten)] - op: RequestOp, + pub op: RequestOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -33,15 +33,18 @@ pub enum RecvMessage { pub struct Response { /// Operation Id (pairs with Request, or empty if unidirectional) #[serde(default)] - id: u32, + pub id: u32, /// The response operation variant #[serde(flatten)] - op: ResponseOp, + pub op: ResponseOp, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "op")] pub enum RequestOp { + Control { + args: Vec, + }, GetState, Attach, Detach, @@ -131,6 +134,10 @@ pub struct NewPrivateRouteResult { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "op")] pub enum ResponseOp { + Control { + #[serde(flatten)] + result: ApiResult, + }, GetState { #[serde(flatten)] result: ApiResult, diff --git a/veilid-core/src/veilid_api/json_api/process.rs b/veilid-core/src/veilid_api/json_api/process.rs index 15304f84..84eb97eb 100644 --- a/veilid-core/src/veilid_api/json_api/process.rs +++ b/veilid-core/src/veilid_api/json_api/process.rs @@ -1,7 +1,7 @@ use super::*; use futures_util::FutureExt; -fn to_json_api_result( +pub fn to_json_api_result( r: VeilidAPIResult, ) -> json_api::ApiResult { match r { @@ -10,7 +10,7 @@ fn to_json_api_result( } } -fn to_json_api_result_with_string( +pub fn to_json_api_result_with_string( r: VeilidAPIResult, ) -> json_api::ApiResultWithString { match r { @@ -19,7 +19,7 @@ fn to_json_api_result_with_string( } } -fn to_json_api_result_with_vec_string( +pub fn to_json_api_result_with_vec_string( r: VeilidAPIResult, ) -> json_api::ApiResultWithVecString { match r { @@ -28,14 +28,14 @@ fn to_json_api_result_with_vec_string( } } -fn to_json_api_result_with_vec_u8(r: VeilidAPIResult>) -> json_api::ApiResultWithVecU8 { +pub fn to_json_api_result_with_vec_u8(r: VeilidAPIResult>) -> json_api::ApiResultWithVecU8 { match r { Err(e) => json_api::ApiResultWithVecU8::Err { error: e }, Ok(v) => json_api::ApiResultWithVecU8::Ok { value: v }, } } -fn to_json_api_result_with_vec_vec_u8( +pub fn to_json_api_result_with_vec_vec_u8( r: VeilidAPIResult>>, ) -> json_api::ApiResultWithVecVecU8 { match r { @@ -46,38 +46,45 @@ fn to_json_api_result_with_vec_vec_u8( } } +struct JsonRequestProcessorInner { + routing_contexts: BTreeMap, + table_dbs: BTreeMap, + table_db_transactions: BTreeMap, + crypto_systems: BTreeMap, +} + +#[derive(Clone)] pub struct JsonRequestProcessor { api: VeilidAPI, - routing_contexts: Mutex>, - table_dbs: Mutex>, - table_db_transactions: Mutex>, - crypto_systems: Mutex>, + inner: Arc>, } impl JsonRequestProcessor { pub fn new(api: VeilidAPI) -> Self { Self { api, - routing_contexts: Default::default(), - table_dbs: Default::default(), - table_db_transactions: Default::default(), - crypto_systems: Default::default(), + inner: Arc::new(Mutex::new(JsonRequestProcessorInner { + routing_contexts: Default::default(), + table_dbs: Default::default(), + table_db_transactions: Default::default(), + crypto_systems: Default::default(), + })), } } // Routing Context fn add_routing_context(&self, routing_context: RoutingContext) -> u32 { + let mut inner = self.inner.lock(); let mut next_id: u32 = 1; - let mut rc = self.routing_contexts.lock(); - while rc.contains_key(&next_id) { + while inner.routing_contexts.contains_key(&next_id) { next_id += 1; } - rc.insert(next_id, routing_context); + inner.routing_contexts.insert(next_id, routing_context); next_id } fn lookup_routing_context(&self, id: u32, rc_id: u32) -> Result { - let routing_contexts = self.routing_contexts.lock(); - let Some(routing_context) = routing_contexts.get(&rc_id).cloned() else { + let inner = self.inner.lock(); + let Some(routing_context) = inner.routing_contexts.get(&rc_id).cloned() else { return Err(Response { id, op: ResponseOp::RoutingContext(RoutingContextResponse { @@ -89,8 +96,8 @@ impl JsonRequestProcessor { Ok(routing_context) } fn release_routing_context(&self, id: u32) -> i32 { - let mut rc = self.routing_contexts.lock(); - if rc.remove(&id).is_none() { + let mut inner = self.inner.lock(); + if inner.routing_contexts.remove(&id).is_none() { return 0; } return 1; @@ -98,17 +105,17 @@ impl JsonRequestProcessor { // TableDB fn add_table_db(&self, table_db: TableDB) -> u32 { + let mut inner = self.inner.lock(); let mut next_id: u32 = 1; - let mut rc = self.table_dbs.lock(); - while rc.contains_key(&next_id) { + while inner.table_dbs.contains_key(&next_id) { next_id += 1; } - rc.insert(next_id, table_db); + inner.table_dbs.insert(next_id, table_db); next_id } fn lookup_table_db(&self, id: u32, db_id: u32) -> Result { - let table_dbs = self.table_dbs.lock(); - let Some(table_db) = table_dbs.get(&db_id).cloned() else { + let inner = self.inner.lock(); + let Some(table_db) = inner.table_dbs.get(&db_id).cloned() else { return Err(Response { id, op: ResponseOp::TableDb(TableDbResponse { @@ -120,8 +127,8 @@ impl JsonRequestProcessor { Ok(table_db) } fn release_table_db(&self, id: u32) -> i32 { - let mut rc = self.table_dbs.lock(); - if rc.remove(&id).is_none() { + let mut inner = self.inner.lock(); + if inner.table_dbs.remove(&id).is_none() { return 0; } return 1; @@ -129,12 +136,12 @@ impl JsonRequestProcessor { // TableDBTransaction fn add_table_db_transaction(&self, tdbt: TableDBTransaction) -> u32 { + let mut inner = self.inner.lock(); let mut next_id: u32 = 1; - let mut tdbts = self.table_db_transactions.lock(); - while tdbts.contains_key(&next_id) { + while inner.table_db_transactions.contains_key(&next_id) { next_id += 1; } - tdbts.insert(next_id, tdbt); + inner.table_db_transactions.insert(next_id, tdbt); next_id } fn lookup_table_db_transaction( @@ -142,8 +149,8 @@ impl JsonRequestProcessor { id: u32, tx_id: u32, ) -> Result { - let table_db_transactions = self.table_db_transactions.lock(); - let Some(table_db_transaction) = table_db_transactions.get(&tx_id).cloned() else { + let inner = self.inner.lock(); + let Some(table_db_transaction) = inner.table_db_transactions.get(&tx_id).cloned() else { return Err(Response { id, op: ResponseOp::TableDbTransaction(TableDbTransactionResponse { @@ -155,8 +162,8 @@ impl JsonRequestProcessor { Ok(table_db_transaction) } fn release_table_db_transaction(&self, id: u32) -> i32 { - let mut tdbts = self.table_db_transactions.lock(); - if tdbts.remove(&id).is_none() { + let mut inner = self.inner.lock(); + if inner.table_db_transactions.remove(&id).is_none() { return 0; } return 1; @@ -164,17 +171,17 @@ impl JsonRequestProcessor { // CryptoSystem fn add_crypto_system(&self, csv: CryptoSystemVersion) -> u32 { + let mut inner = self.inner.lock(); let mut next_id: u32 = 1; - let mut crypto_systems = self.crypto_systems.lock(); - while crypto_systems.contains_key(&next_id) { + while inner.crypto_systems.contains_key(&next_id) { next_id += 1; } - crypto_systems.insert(next_id, csv); + inner.crypto_systems.insert(next_id, csv); next_id } fn lookup_crypto_system(&self, id: u32, cs_id: u32) -> Result { - let crypto_systems = self.crypto_systems.lock(); - let Some(crypto_system) = crypto_systems.get(&cs_id).cloned() else { + let inner = self.inner.lock(); + let Some(crypto_system) = inner.crypto_systems.get(&cs_id).cloned() else { return Err(Response { id, op: ResponseOp::CryptoSystem(CryptoSystemResponse { @@ -186,8 +193,8 @@ impl JsonRequestProcessor { Ok(crypto_system) } fn release_crypto_system(&self, id: u32) -> i32 { - let mut crypto_systems = self.crypto_systems.lock(); - if crypto_systems.remove(&id).is_none() { + let mut inner = self.inner.lock(); + if inner.crypto_systems.remove(&id).is_none() { return 0; } return 1; @@ -528,10 +535,15 @@ impl JsonRequestProcessor { } } - pub async fn process_request(&self, request: Request) -> Response { + pub async fn process_request(self, request: Request) -> Response { let id = request.id; let op = match request.op { + RequestOp::Control { args: _args } => ResponseOp::Control { + result: to_json_api_result(VeilidAPIResult::Err(VeilidAPIError::unimplemented( + "control should be handled by veilid-core host application", + ))), + }, RequestOp::GetState => ResponseOp::GetState { result: to_json_api_result(self.api.get_state().await), }, diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index 8559ba47..8b3c3e7e 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -750,7 +750,7 @@ impl VeilidConfig { self.inner.read() } - fn safe_config(&self) -> VeilidConfigInner { + pub fn safe_config(&self) -> VeilidConfigInner { let mut safe_cfg = self.inner.read().clone(); // Remove secrets @@ -773,6 +773,11 @@ impl VeilidConfig { let out = f(&mut editedinner)?; // Validate Self::validate(&mut editedinner)?; + // See if things have changed + if *inner == editedinner { + // No changes, return early + return Ok(out); + } // Commit changes *inner = editedinner.clone(); out diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 57dd2ba1..11317371 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -10,13 +10,13 @@ name = "veilid-server" path = "src/main.rs" [features] -default = [ "rt-tokio", "veilid-core/default" ] -crypto-test = [ "rt-tokio", "veilid-core/crypto-test"] -crypto-test-none = [ "rt-tokio", "veilid-core/crypto-test-none"] +default = ["rt-tokio", "veilid-core/default"] +crypto-test = ["rt-tokio", "veilid-core/crypto-test"] +crypto-test-none = ["rt-tokio", "veilid-core/crypto-test-none"] -rt-async-std = [ "veilid-core/rt-async-std", "async-std", "opentelemetry/rt-async-std", "opentelemetry-otlp/grpc-sys" ] -rt-tokio = [ "veilid-core/rt-tokio", "tokio", "tokio-stream", "tokio-util", "opentelemetry/rt-tokio", "console-subscriber" ] -tracking = [ "veilid-core/tracking" ] +rt-async-std = ["veilid-core/rt-async-std", "async-std", "opentelemetry/rt-async-std", "opentelemetry-otlp/grpc-sys"] +rt-tokio = ["veilid-core/rt-tokio", "tokio", "tokio-stream", "tokio-util", "opentelemetry/rt-tokio", "console-subscriber"] +tracking = ["veilid-core/tracking"] [dependencies] veilid-core = { path = "../veilid-core", default-features = false } @@ -55,6 +55,7 @@ rpassword = "^6" hostname = "^0" stop-token = { version = "^0", default-features = false } sysinfo = { version = "^0.28.4", default-features = false } +wg = "0.3.2" [target.'cfg(windows)'.dependencies] windows-service = "^0" @@ -70,4 +71,4 @@ nix = "^0" tracing-journald = "^0" [dev-dependencies] -serial_test = "^0" \ No newline at end of file +serial_test = "^0" diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index b3bc15cc..ededea30 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -1,19 +1,29 @@ use crate::settings::*; use crate::tools::*; -use crate::veilid_client_capnp::*; use crate::veilid_logs::VeilidLogs; use cfg_if::*; -use futures_util::{future::try_join_all, FutureExt as FuturesFutureExt, StreamExt}; -use serde::*; -use std::cell::RefCell; +use futures_util::{future::try_join_all, stream::FuturesUnordered, StreamExt}; + +use parking_lot::Mutex; use std::collections::HashMap; -use std::fmt; use std::net::SocketAddr; +use std::sync::Arc; use stop_token::future::FutureExt; use stop_token::*; use tracing::*; +use veilid_core::tools::*; use veilid_core::*; +use wg::AsyncWaitGroup; +cfg_if! { + if #[cfg(feature="rt-async-std")] { + use async_std::io::prelude::BufReadExt; + use async_std::io::WriteExt; + } else if #[cfg(feature="rt-tokio")] { + use tokio::io::AsyncBufReadExt; + use tokio::io::AsyncWriteExt; + } +} // struct VeilidServerImpl { // veilid_api: veilid_core::VeilidAPI, // veilid_logs: VeilidLogs, @@ -36,50 +46,11 @@ use veilid_core::*; // } // } -// #[instrument(level = "trace", skip_all)] -// fn shutdown( -// &mut self, -// _params: veilid_server::ShutdownParams, -// mut _results: veilid_server::ShutdownResults, -// ) -> Promise<(), ::capnp::Error> { -// trace!("VeilidServerImpl::shutdown"); - -// cfg_if::cfg_if! { -// if #[cfg(windows)] { -// assert!(false, "write me!"); -// } -// else { -// crate::server::shutdown(); -// } -// } - -// Promise::ok(()) -// } - -// #[instrument(level = "trace", skip_all)] -// fn change_log_level( -// &mut self, -// params: veilid_server::ChangeLogLevelParams, -// mut results: veilid_server::ChangeLogLevelResults, -// ) -> Promise<(), ::capnp::Error> { -// trace!("VeilidServerImpl::change_log_level"); - -// let layer = pry!(pry!(params.get()).get_layer()).to_owned(); -// let log_level_json = pry!(pry!(params.get()).get_log_level()).to_owned(); -// let log_level: veilid_core::VeilidConfigLogLevel = -// pry!(veilid_core::deserialize_json(&log_level_json) -// .map_err(|e| ::capnp::Error::failed(format!("{:?}", e)))); - -// let result = self.veilid_logs.change_log_level(layer, log_level); -// encode_api_result(&result, &mut results.get().init_result()); -// Promise::ok(()) -// } // } // --- Client API Server-Side --------------------------------- -type ClientApiAllFuturesJoinHandle = - JoinHandle, Box<(dyn std::error::Error + 'static)>>>; +type ClientApiAllFuturesJoinHandle = MustJoinHandle>>; struct ClientApiInner { veilid_api: veilid_core::VeilidAPI, @@ -87,8 +58,10 @@ struct ClientApiInner { settings: Settings, stop: Option, join_handle: Option, + update_channels: HashMap<(SocketAddr, SocketAddr), flume::Sender>, } +#[derive(Clone)] pub struct ClientApi { inner: Arc>, } @@ -99,23 +72,43 @@ impl ClientApi { veilid_api: veilid_core::VeilidAPI, veilid_logs: VeilidLogs, settings: Settings, - ) -> Rc { - Rc::new(Self { - inner: RefCell::new(ClientApiInner { + ) -> Self { + Self { + inner: Arc::new(Mutex::new(ClientApiInner { veilid_api, veilid_logs, settings, stop: Some(StopSource::new()), join_handle: None, - }), - }) + update_channels: HashMap::new(), + })), + } + } + + #[instrument(level = "trace", skip_all)] + fn shutdown(&self) { + trace!("ClientApi::shutdown"); + + crate::server::shutdown(); + } + + #[instrument(level = "trace", skip_all)] + fn change_log_level( + &self, + layer: String, + log_level: VeilidConfigLogLevel, + ) -> VeilidAPIResult<()> { + trace!("ClientApi::change_log_level"); + + let veilid_logs = self.inner.lock().veilid_logs.clone(); + veilid_logs.change_log_level(layer, log_level) } #[instrument(level = "trace", skip(self))] - pub async fn stop(self: Rc) { + pub async fn stop(&self) { trace!("ClientApi::stop requested"); let jh = { - let mut inner = self.inner.borrow_mut(); + let mut inner = self.inner.lock(); if inner.join_handle.is_none() { trace!("ClientApi stop ignored"); return; @@ -131,10 +124,7 @@ impl ClientApi { } #[instrument(level = "trace", skip(self), err)] - async fn handle_incoming( - self, - bind_addr: SocketAddr, - ) -> Result<(), Box> { + async fn handle_incoming(self, bind_addr: SocketAddr) -> std::io::Result<()> { let listener = TcpListener::bind(bind_addr).await?; debug!("Client API listening on: {:?}", bind_addr); @@ -147,55 +137,249 @@ impl ClientApi { } } + // Make wait group for all incoming connections + let awg = AsyncWaitGroup::new(); + let stop_token = self.inner.lock().stop.as_ref().unwrap().token(); - let incoming_loop = async move { - while let Ok(Some(stream_result)) = - incoming_stream.next().timeout_at(stop_token.clone()).await - { - let stream = stream_result?; - stream.set_nodelay(true)?; - cfg_if! { - if #[cfg(feature="rt-async-std")] { - use futures_util::AsyncReadExt; - let (reader, writer) = stream.split(); - } else if #[cfg(feature="rt-tokio")] { - use tokio_util::compat::*; - let (reader, writer) = stream.into_split(); - let reader = reader.compat(); - let writer = writer.compat_write(); - } + while let Ok(Some(stream_result)) = + incoming_stream.next().timeout_at(stop_token.clone()).await + { + // Get the stream to process + let stream = stream_result?; + stream.set_nodelay(true)?; + + // Increment wait group + awg.add(1); + let t_awg = awg.clone(); + + // Process the connection + spawn(self.clone().handle_connection(stream, t_awg)).detach(); + } + + // Wait for all connections to terminate + awg.wait().await; + + Ok(()) + } + + // Process control messages for the server + async fn process_control(self, args: Vec) -> VeilidAPIResult { + if args.len() == 0 { + apibail_generic!("no control request specified"); + } + if args[0] == "shutdown" { + if args.len() != 1 { + apibail_generic!("wrong number of arguments"); + } + self.shutdown(); + Ok("".to_owned()) + } else if args[0] == "change_log_level" { + if args.len() != 3 { + apibail_generic!("wrong number of arguments"); + } + let log_level: VeilidConfigLogLevel = deserialize_json(&args[2])?; + self.change_log_level(args[1].clone(), log_level)?; + Ok("".to_owned()) + } else if args[0] == "get_server_settings" { + if args.len() != 1 { + apibail_generic!("wrong number of arguments"); + } + let settings = self.inner.lock().settings.clone(); + let settings = &*settings.read(); + let settings_json_string = serialize_json(settings); + let mut settings_json = + json::parse(&settings_json_string).map_err(VeilidAPIError::internal)?; + settings_json["core"]["network"].remove("node_id_secret"); + settings_json["core"]["protected_store"].remove("device_encryption_key_password"); + settings_json["core"]["protected_store"].remove("new_device_encryption_key_password"); + let safe_settings_json = settings_json.to_string(); + Ok(safe_settings_json) + } else if args[0] == "emit_schema" { + if args.len() != 2 { + apibail_generic!("wrong number of arguments"); + } + + let mut schemas = HashMap::::new(); + veilid_core::json_api::emit_schemas(&mut schemas); + + let Some(schema) = schemas.get(&args[1]) else { + apibail_invalid_argument!("invalid schema", "schema", args[1].clone()); + }; + + Ok(schema.clone()) + } else { + apibail_generic!("unknown control message"); + } + } + + pub async fn handle_connection(self, stream: TcpStream, awg: AsyncWaitGroup) { + // Get address of peer + let peer_addr = match stream.peer_addr() { + Ok(v) => v, + Err(e) => { + error!("can't get peer address: {}", e); + return; + } + }; + // Get local address + let local_addr = match stream.local_addr() { + Ok(v) => v, + Err(e) => { + error!("can't get local address: {}", e); + return; + } + }; + // Get connection tuple + let conn_tuple = (local_addr, peer_addr); + + debug!( + "Accepted Client API Connection: {:?} -> {:?}", + peer_addr, local_addr + ); + + // Make stop token to quit when stop() is requested externally + let stop_token = self.inner.lock().stop.as_ref().unwrap().token(); + + // Split into reader and writer halves + // with line buffering on the reader + cfg_if! { + if #[cfg(feature="rt-async-std")] { + use futures_util::AsyncReadExt; + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + } else if #[cfg(feature="rt-tokio")] { + let (reader, mut writer) = stream.into_split(); + let mut reader = BufReader::new(reader); + } + } + + // Make request processor for this connection + let api = self.inner.lock().veilid_api.clone(); + let jrp = json_api::JsonRequestProcessor::new(api); + + // Futures to process unordered + let mut unord = FuturesUnordered::new(); + let (more_futures_tx, more_futures_rx) = flume::unbounded(); + + // Output to serialize + let (responses_tx, responses_rx) = flume::unbounded(); + + // Request receive processor + let this = self.clone(); + let recv_requests_future = async move { + // Start sending updates + this.inner + .lock() + .update_channels + .insert(conn_tuple, responses_tx.clone()); + + let mut line = String::new(); + while let Ok(size) = reader.read_line(&mut line).await { + // Eof? + if size == 0 { + break; } - xxx spawn json_api handler - spawn_local(rpc_system.map(drop)); - } - Ok::<(), Box>(()) - }; + // Put the processing in the async queue + let jrp = jrp.clone(); + let line = line.trim().to_owned(); + // Ignore newlines + if line.len() == 0 { + continue; + } + let responses_tx = responses_tx.clone(); + let this2 = this.clone(); + let process_request = async move { + // Unmarshal NDJSON - newline => json + // (trim all whitespace around input lines just to make things more permissive for API users) + let request: json_api::Request = deserialize_json(&line)?; - incoming_loop.await + // See if this is a control message or a veilid-core message + let response = if let json_api::RequestOp::Control { args } = request.op { + // Process control messages + json_api::Response { + id: request.id, + op: json_api::ResponseOp::Control { + result: json_api::to_json_api_result( + this2.process_control(args).await, + ), + }, + } + } else { + // Process with ndjson api + jrp.clone().process_request(request).await + }; + + // Marshal json + newline => NDJSON + let response_string = + serialize_json(json_api::RecvMessage::Response(response)) + "\n"; + if let Err(e) = responses_tx.send_async(response_string).await { + warn!("response not sent: {}", e) + } + VeilidAPIResult::Ok(()) + }; + if let Err(e) = more_futures_tx + .send_async(system_boxed(process_request)) + .await + { + warn!("request dropped: {}", e) + } + } + + // Stop sending updates + // Will cause send_responses_future to stop because we drop the responses_tx + this.inner.lock().update_channels.remove(&conn_tuple); + + VeilidAPIResult::Ok(()) + }; + unord.push(system_boxed(recv_requests_future)); + + // Response send processor + let send_responses_future = async move { + while let Ok(resp) = responses_rx.recv_async().await { + if let Err(e) = writer.write_all(resp.as_bytes()).await { + error!("failed to write response: {}", e) + } + } + VeilidAPIResult::Ok(()) + }; + unord.push(system_boxed(send_responses_future)); + + // Send and receive until we're done or a stop is requested + while let Ok(Some(r)) = unord.next().timeout_at(stop_token.clone()).await { + match r { + Ok(()) => {} + Err(e) => { + warn!("JSON API Failure: {}", e); + } + } + // Add more futures if we had one that completed + // Allows processing requests in an async fashion + for fut in more_futures_rx.drain() { + unord.push(fut); + } + } + + debug!( + "Closed Client API Connection: {:?} -> {:?}", + peer_addr, local_addr + ); + + awg.done(); } #[instrument(level = "trace", skip(self))] - pub fn handle_update(self: Rc, veilid_update: veilid_core::VeilidUpdate) { - // serialize update - let veilid_update = serialize_json(veilid_update); + pub fn handle_update(&self, veilid_update: veilid_core::VeilidUpdate) { + // serialize update to NDJSON + let veilid_update = serialize_json(json_api::RecvMessage::Update(veilid_update)) + "\n"; // Pass other updates to clients - self.send_request_to_all_clients(|_id, registration| { - match veilid_update - .len() - .try_into() - .map_err(|e| ::capnp::Error::failed(format!("{:?}", e))) - { - Ok(len) => { - let mut request = registration.client.update_request(); - let mut rpc_veilid_update = request.get().init_veilid_update(len); - rpc_veilid_update.push_str(&veilid_update); - Some(request.send()) - } - Err(_) => None, + let inner = self.inner.lock(); + for ch in inner.update_channels.values() { + if let Err(e) = ch.send(veilid_update.clone()) { + eprintln!("failed to send update: {}", e); } - }); + } } #[instrument(level = "trace", skip(self))] @@ -204,6 +388,6 @@ impl ClientApi { .iter() .map(|addr| self.clone().handle_incoming(*addr)); let bind_futures_join = try_join_all(bind_futures); - self.inner.borrow_mut().join_handle = Some(spawn_local(bind_futures_join)); + self.inner.lock().join_handle = Some(spawn(bind_futures_join)); } } diff --git a/veilid-server/src/server.rs b/veilid-server/src/server.rs index 1ffa64f1..2a16f91e 100644 --- a/veilid-server/src/server.rs +++ b/veilid-server/src/server.rs @@ -1,8 +1,6 @@ use crate::client_api; use crate::settings::*; -use crate::tools::*; use crate::veilid_logs::*; -use crate::*; use flume::{unbounded, Receiver, Sender}; use futures_util::select; use futures_util::FutureExt; @@ -11,7 +9,7 @@ use parking_lot::Mutex; use std::sync::Arc; use std::time::{Duration, Instant}; use tracing::*; -use veilid_core::tools::SingleShotEventual; +use veilid_core::tools::*; #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum ServerMode { @@ -140,7 +138,7 @@ pub async fn run_veilid_server_internal( break; } } - sleep(Duration::from_millis(100)).await; + sleep(100).await; } match veilid_api.debug("txtrecord".to_string()).await { Ok(v) => { diff --git a/veilid-server/src/tools.rs b/veilid-server/src/tools.rs index 0e7b51e3..06d87e1d 100644 --- a/veilid-server/src/tools.rs +++ b/veilid-server/src/tools.rs @@ -3,41 +3,43 @@ use core::future::Future; cfg_if! { if #[cfg(feature="rt-async-std")] { - pub use async_std::task::JoinHandle; +// pub use async_std::task::JoinHandle; pub use async_std::net::TcpListener; - //pub use async_std::net::TcpStream; + pub use async_std::net::TcpStream; + pub use async_std::io::BufReader; //pub use async_std::future::TimeoutError; - pub fn spawn + Send + 'static, T: Send + 'static>(f: F) -> JoinHandle { - async_std::task::spawn_local(f) - } - pub fn spawn_local + 'static, T: 'static>(f: F) -> JoinHandle { - async_std::task::spawn_local(f) - } + //pub fn spawn_detached + Send + 'static, T: Send + 'static>(f: F) -> JoinHandle { + //async_std::task::spawn(f) + //} + // pub fn spawn_local + 'static, T: 'static>(f: F) -> JoinHandle { + // async_std::task::spawn_local(f) + // } // pub fn spawn_detached_local + 'static, T: 'static>(f: F) { // let _ = async_std::task::spawn_local(f); // } - pub use async_std::task::sleep; - pub use async_std::future::timeout; + //pub use async_std::task::sleep; + //pub use async_std::future::timeout; pub fn block_on, T>(f: F) -> T { async_std::task::block_on(f) } } else if #[cfg(feature="rt-tokio")] { - pub use tokio::task::JoinHandle; + //pub use tokio::task::JoinHandle; pub use tokio::net::TcpListener; - //pub use tokio::net::TcpStream; + pub use tokio::net::TcpStream; + pub use tokio::io::BufReader; //pub use tokio_util::compat::*; //pub use tokio::time::error::Elapsed as TimeoutError; - pub fn spawn + Send + 'static, T: Send + 'static>(f: F) -> JoinHandle { - tokio::task::spawn(f) - } - pub fn spawn_local + 'static, T: 'static>(f: F) -> JoinHandle { - tokio::task::spawn_local(f) - } + //pub fn spawn_detached + Send + 'static, T: Send + 'static>(f: F) -> JoinHandle { + //tokio::task::spawn(f) + //} + // pub fn spawn_local + 'static, T: 'static>(f: F) -> JoinHandle { + // tokio::task::spawn_local(f) + // } // pub fn spawn_detached_local + 'static, T: 'static>(f: F) { // let _ = tokio::task::spawn_local(f); // } - pub use tokio::time::sleep; - pub use tokio::time::timeout; + //pub use tokio::time::sleep; + //pub use tokio::time::timeout; pub fn block_on, T>(f: F) -> T { let rt = tokio::runtime::Runtime::new().unwrap(); let local = tokio::task::LocalSet::new(); diff --git a/veilid-server/src/unix.rs b/veilid-server/src/unix.rs index 1d2aa389..68f10760 100644 --- a/veilid-server/src/unix.rs +++ b/veilid-server/src/unix.rs @@ -2,13 +2,11 @@ use crate::server::*; use crate::settings::Settings; use crate::tools::*; use crate::veilid_logs::*; -use crate::*; use clap::ArgMatches; use futures_util::StreamExt; use signal_hook::consts::signal::*; use signal_hook_async_std::Signals; -//use std::io::Read; -use tracing::*; +use veilid_core::tools::*; #[instrument(skip(signals))] async fn handle_signals(mut signals: Signals) { @@ -33,24 +31,7 @@ pub fn run_daemon(settings: Settings, _matches: ArgMatches) -> EyreResult<()> { let mut daemon = daemonize::Daemonize::new(); let s = settings.read(); if let Some(pid_file) = s.daemon.pid_file.clone() { - daemon = daemon.pid_file(pid_file.clone()); //.chown_pid_file(true); - // daemon = daemon.exit_action(move || { - // // wait for pid file to exist before exiting parent - // let pid_path = std::path::Path::new(&pid_file); - // loop { - // if let Ok(mut f) = std::fs::File::open(pid_path) { - // let mut s = String::new(); - // if f.read_to_string(&mut s).is_ok() - // && !s.is_empty() - // && s.parse::().is_ok() - // { - // println!("pidfile found"); - // break; - // } - // } - // std::thread::sleep(std::time::Duration::from_millis(100)); - // } - // }) + daemon = daemon.pid_file(pid_file.clone()); } if let Some(chroot) = &s.daemon.chroot { daemon = daemon.chroot(chroot); diff --git a/veilid-tools/src/must_join_handle.rs b/veilid-tools/src/must_join_handle.rs index ff7231f1..a4afd1dc 100644 --- a/veilid-tools/src/must_join_handle.rs +++ b/veilid-tools/src/must_join_handle.rs @@ -16,6 +16,22 @@ impl MustJoinHandle { } } + pub fn detach(mut self) { + cfg_if! { + if #[cfg(feature="rt-async-std")] { + self.join_handle = None; + } else if #[cfg(feature="rt-tokio")] { + self.join_handle = None; + } else if #[cfg(target_arch = "wasm32")] { + self.join_handle.take().detach(); + self.completed = true; + } else { + compile_error!("needs executor implementation") + } + } + self.completed = true; + } + #[allow(unused_mut)] pub async fn abort(mut self) { if !self.completed { From 59c14f3b227bdc9c0aabeedf305733d07f9c7ac4 Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 7 Jun 2023 21:55:23 -0400 Subject: [PATCH 08/23] checkpoint --- Cargo.lock | 4 +- veilid-cli/Cargo.toml | 8 +- veilid-cli/src/client_api_connection.rs | 516 +++++++++++++----------- veilid-cli/src/command_processor.rs | 23 +- veilid-cli/src/main.rs | 2 +- veilid-cli/src/peers_table_view.rs | 1 - veilid-cli/src/ui.rs | 67 +-- veilid-tools/src/lib.rs | 5 +- 8 files changed, 340 insertions(+), 286 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22646141..f841b887 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6160,6 +6160,7 @@ dependencies = [ "cursive_table_view", "directories", "flexi_logger", + "flume", "futures", "hex", "json", @@ -6168,10 +6169,11 @@ dependencies = [ "serde", "serde_derive", "serial_test", + "stop-token", "thiserror", "tokio 1.28.2", "tokio-util", - "veilid-core", + "veilid-tools", ] [[package]] diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index cc58e0d3..021275d2 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -12,8 +12,8 @@ path = "src/main.rs" [features] default = [ "rt-tokio" ] macos = [ "cursive/ncurses-backend" ] -rt-async-std = [ "async-std", "veilid-core/rt-async-std", "cursive/rt-async-std" ] -rt-tokio = [ "tokio", "tokio-util", "veilid-core/rt-tokio", "cursive/rt-tokio" ] +rt-async-std = [ "async-std", "veilid-tools/rt-async-std", "cursive/rt-async-std" ] +rt-tokio = [ "tokio", "tokio-util", "veilid-tools/rt-tokio", "cursive/rt-tokio" ] [dependencies] async-std = { version = "^1.9", features = ["unstable", "attributes"], optional = true } @@ -41,8 +41,10 @@ flexi_logger = { version = "^0", features = ["use_chrono_for_offset"] } thiserror = "^1" crossbeam-channel = "^0" hex = "^0" -veilid-core = { path = "../veilid-core" } +veilid-tools = { path = "../veilid-tools", default-features = false } json = "^0" +stop-token = { version = "^0", default-features = false } +flume = { version = "^0", features = ["async"] } [dev-dependencies] serial_test = "^0" diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index bdabcb96..21d4ba5c 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -1,94 +1,66 @@ use crate::command_processor::*; use crate::tools::*; -use futures::future::FutureExt; use serde::de::DeserializeOwned; use std::cell::RefCell; use std::net::SocketAddr; use std::rc::Rc; -use veilid_core::tools::*; -use veilid_core::*; +use stop_token::{future::FutureExt as _, StopSource, StopToken}; -fn map_to_internal_error(e: T) -> VeilidAPIError { - VeilidAPIError::Internal { - message: e.to_string(), +use veilid_tools::*; +cfg_if! { + if #[cfg(feature="rt-async-std")] { + use async_std::io::prelude::BufReadExt; + use async_std::io::WriteExt; + use async_std::io::BufReader; + } else if #[cfg(feature="rt-tokio")] { + use tokio::io::AsyncBufReadExt; + use tokio::io::AsyncWriteExt; + use tokio::io::BufReader; } } -fn decode_api_result( - reader: &api_result::Reader, -) -> VeilidAPIResult { - match reader.which().map_err(map_to_internal_error)? { - api_result::Which::Ok(v) => { - let ok_val = v.map_err(map_to_internal_error)?; - let res: T = veilid_core::deserialize_json(ok_val).map_err(map_to_internal_error)?; - Ok(res) - } - api_result::Which::Err(e) => { - let err_val = e.map_err(map_to_internal_error)?; - let res: VeilidAPIError = - veilid_core::deserialize_json(err_val).map_err(map_to_internal_error)?; - Err(res) - } - } -} +// fn map_to_internal_error(e: T) -> VeilidAPIError { +// VeilidAPIError::Internal { +// message: e.to_string(), +// } +// } -struct VeilidClientImpl { - comproc: CommandProcessor, -} +// fn decode_api_result( +// reader: &api_result::Reader, +// ) -> VeilidAPIResult { +// match reader.which().map_err(map_to_internal_error)? { +// api_result::Which::Ok(v) => { +// let ok_val = v.map_err(map_to_internal_error)?; +// let res: T = veilid_core::deserialize_json(ok_val).map_err(map_to_internal_error)?; +// Ok(res) +// } +// api_result::Which::Err(e) => { +// let err_val = e.map_err(map_to_internal_error)?; +// let res: VeilidAPIError = +// veilid_core::deserialize_json(err_val).map_err(map_to_internal_error)?; +// Err(res) +// } +// } +// } -impl VeilidClientImpl { - pub fn new(comproc: CommandProcessor) -> Self { - Self { comproc } - } -} +// struct VeilidClientImpl { +// comproc: CommandProcessor, +// } -impl veilid_client::Server for VeilidClientImpl { - fn update( - &mut self, - params: veilid_client::UpdateParams, - _results: veilid_client::UpdateResults, - ) -> Promise<(), ::capnp::Error> { - let veilid_update = pry!(pry!(params.get()).get_veilid_update()); - let veilid_update: VeilidUpdate = pry_result!(deserialize_json(veilid_update)); +// impl VeilidClientImpl { +// pub fn new(comproc: CommandProcessor) -> Self { +// Self { comproc } +// } +// } - match veilid_update { - VeilidUpdate::Log(log) => { - self.comproc.update_log(log); - } - VeilidUpdate::AppMessage(msg) => { - self.comproc.update_app_message(msg); - } - VeilidUpdate::AppCall(call) => { - self.comproc.update_app_call(call); - } - VeilidUpdate::Attachment(attachment) => { - self.comproc.update_attachment(attachment); - } - VeilidUpdate::Network(network) => { - self.comproc.update_network_status(network); - } - VeilidUpdate::Config(config) => { - self.comproc.update_config(config); - } - VeilidUpdate::RouteChange(route) => { - self.comproc.update_route(route); - } - VeilidUpdate::Shutdown => self.comproc.update_shutdown(), - VeilidUpdate::ValueChange(value_change) => { - self.comproc.update_value_change(value_change); - } - } - - Promise::ok(()) - } -} +// } struct ClientApiConnectionInner { comproc: CommandProcessor, connect_addr: Option, - disconnector: Option>, - server: Option>>, + server: Option>, server_settings: Option, + disconnector: Option, disconnect_requested: bool, cancel_eventual: Eventual, } @@ -106,9 +78,9 @@ impl ClientApiConnection { inner: Rc::new(RefCell::new(ClientApiConnectionInner { comproc, connect_addr: None, - disconnector: None, server: None, server_settings: None, + disconnector: None, disconnect_requested: false, cancel_eventual: Eventual::new(), })), @@ -123,196 +95,272 @@ impl ClientApiConnection { eventual.resolve(); // don't need to await this } - async fn process_veilid_state<'a>( - &'a mut self, - veilid_state: VeilidState, - ) -> Result<(), String> { - let mut inner = self.inner.borrow_mut(); - inner.comproc.update_attachment(veilid_state.attachment); - inner.comproc.update_network_status(veilid_state.network); - inner.comproc.update_config(veilid_state.config); - Ok(()) - } + // async fn process_veilid_state<'a>( + // &'a mut self, + // veilid_state: VeilidState, + // ) -> Result<(), String> { + // let mut inner = self.inner.borrow_mut(); + // inner.comproc.update_attachment(veilid_state.attachment); + // inner.comproc.update_network_status(veilid_state.network); + // inner.comproc.update_config(veilid_state.config); + // Ok(()) + // } - async fn spawn_rpc_system( - &mut self, - connect_addr: SocketAddr, - mut rpc_system: RpcSystem, - ) -> Result<(), String> { - let mut request; - { - let mut inner = self.inner.borrow_mut(); - - // Get the bootstrap server connection object - inner.server = Some(Rc::new(RefCell::new( - rpc_system.bootstrap(rpc_twoparty_capnp::Side::Server), - ))); - - // Store our disconnector future for later (must happen after bootstrap, contrary to documentation) - inner.disconnector = Some(rpc_system.get_disconnector()); - - // Get a client object to pass to the server for status update callbacks - let client = capnp_rpc::new_client(VeilidClientImpl::new(inner.comproc.clone())); - - // Register our client and get a registration object back - request = inner - .server - .as_ref() - .unwrap() - .borrow_mut() - .register_request(); - request.get().set_veilid_client(client); - - inner - .comproc - .set_connection_state(ConnectionState::Connected( - connect_addr, - std::time::SystemTime::now(), - )); - } - - let rpc_jh = spawn_local(rpc_system); - - let reg_res: Result = (async { - // Send the request and get the state object and the registration object - let response = request - .send() - .promise - .await - .map_err(|e| format!("failed to send register request: {}", e))?; - let response = response - .get() - .map_err(|e| format!("failed to get register response: {}", e))?; - - // Get the registration object, which drops our connection when it is dropped - let registration = response - .get_registration() - .map_err(|e| format!("failed to get registration object: {}", e))?; - - // Get the initial veilid state - let veilid_state = response - .get_state() - .map_err(|e| format!("failed to get initial veilid state: {}", e))?; - - // Set up our state for the first time - let veilid_state: VeilidState = deserialize_json(veilid_state) - .map_err(|e| format!("failed to get deserialize veilid state: {}", e))?; - self.process_veilid_state(veilid_state).await?; - - // Save server settings - let server_settings = response - .get_settings() - .map_err(|e| format!("failed to get initial veilid server settings: {}", e))? - .to_owned(); - self.inner.borrow_mut().server_settings = Some(server_settings.clone()); - - // Don't drop the registration, doing so will remove the client - // object mapping from the server which we need for the update backchannel - Ok(registration) - }) - .await; - - let _registration = match reg_res { - Ok(v) => v, - Err(e) => { - rpc_jh.abort().await; - return Err(e); - } + async fn process_update(&self, update: json::JsonValue) { + let comproc = self.inner.borrow().comproc.clone(); + let Some(kind) = update["kind"].as_str() else { + comproc.log_message(format!("missing update kind: {}", update)); + return; }; - - // Wait until rpc system completion or disconnect was requested - let res = rpc_jh.await; - res.map_err(|e| format!("client RPC system error: {}", e)) + match kind { + "Log" => { + comproc.update_log(update); + } + "AppMessage" => { + comproc.update_app_message(update); + } + "AppCall" => { + comproc.update_app_call(update); + } + "Attachment" => { + comproc.update_attachment(update); + } + "Network" => { + comproc.update_network_status(update); + } + "Config" => { + comproc.update_config(update); + } + "RouteChange" => { + comproc.update_route(update); + } + "Shutdown" => comproc.update_shutdown(), + "ValueChange" => { + comproc.update_value_change(update); + } + _ => { + comproc.log_message(format!("unknown update kind: {}", update)); + } + } } + // async fn spawn_rpc_system( + // &mut self, + // connect_addr: SocketAddr, + // mut rpc_system: RpcSystem, + // ) -> Result<(), String> { + // let mut request; + // { + // let mut inner = self.inner.borrow_mut(); + + // // Get the bootstrap server connection object + // inner.server = Some(Rc::new(RefCell::new( + // rpc_system.bootstrap(rpc_twoparty_capnp::Side::Server), + // ))); + + // // Store our disconnector future for later (must happen after bootstrap, contrary to documentation) + // inner.disconnector = Some(rpc_system.get_disconnector()); + + // // Get a client object to pass to the server for status update callbacks + // let client = capnp_rpc::new_client(VeilidClientImpl::new(inner.comproc.clone())); + + // // Register our client and get a registration object back + // request = inner + // .server + // .as_ref() + // .unwrap() + // .borrow_mut() + // .register_request(); + // request.get().set_veilid_client(client); + + // inner + // .comproc + // .set_connection_state(ConnectionState::Connected( + // connect_addr, + // std::time::SystemTime::now(), + // )); + // } + + // let rpc_jh = spawn_local(rpc_system); + + // let reg_res: Result = (async { + // // Send the request and get the state object and the registration object + // let response = request + // .send() + // .promise + // .await + // .map_err(|e| format!("failed to send register request: {}", e))?; + // let response = response + // .get() + // .map_err(|e| format!("failed to get register response: {}", e))?; + + // // Get the registration object, which drops our connection when it is dropped + // let registration = response + // .get_registration() + // .map_err(|e| format!("failed to get registration object: {}", e))?; + + // // Get the initial veilid state + // let veilid_state = response + // .get_state() + // .map_err(|e| format!("failed to get initial veilid state: {}", e))?; + + // // Set up our state for the first time + // let veilid_state: VeilidState = deserialize_json(veilid_state) + // .map_err(|e| format!("failed to get deserialize veilid state: {}", e))?; + // self.process_veilid_state(veilid_state).await?; + + // // Save server settings + // let server_settings = response + // .get_settings() + // .map_err(|e| format!("failed to get initial veilid server settings: {}", e))? + // .to_owned(); + // self.inner.borrow_mut().server_settings = Some(server_settings.clone()); + + // // Don't drop the registration, doing so will remove the client + // // object mapping from the server which we need for the update backchannel + // Ok(registration) + // }) + // .await; + + // let _registration = match reg_res { + // Ok(v) => v, + // Err(e) => { + // rpc_jh.abort().await; + // return Err(e); + // } + // }; + + // // Wait until rpc system completion or disconnect was requested + // let res = rpc_jh.await; + // res.map_err(|e| format!("client RPC system error: {}", e)) + // } + async fn handle_connection(&mut self, connect_addr: SocketAddr) -> Result<(), String> { trace!("ClientApiConnection::handle_connection"); - self.inner.borrow_mut().connect_addr = Some(connect_addr); + let stop_token = { + let stop_source = StopSource::new(); + let token = stop_source.token(); + let mut inner = self.inner.borrow_mut(); + inner.connect_addr = Some(connect_addr); + inner.disconnector = Some(stop_source); + token + }; + // Connect the TCP socket let stream = TcpStream::connect(connect_addr) .await .map_err(map_to_string)?; + // If it succeed, disable nagle algorithm stream.set_nodelay(true).map_err(map_to_string)?; - // Create the VAT network + // Split the stream cfg_if! { if #[cfg(feature="rt-async-std")] { use futures::AsyncReadExt; let (reader, writer) = stream.split(); + let mut reader = BufReader::new(reader); } else if #[cfg(feature="rt-tokio")] { - pub use tokio_util::compat::*; let (reader, writer) = stream.into_split(); - let reader = reader.compat(); - let writer = writer.compat_write(); + let mut reader = BufReader::new(reader); } } - let rpc_network = Box::new(twoparty::VatNetwork::new( - reader, - writer, - rpc_twoparty_capnp::Side::Client, - Default::default(), - )); - - // Create the rpc system - let rpc_system = RpcSystem::new(rpc_network, None); - - // Process the rpc system until we decide we're done - match self.spawn_rpc_system(connect_addr, rpc_system).await { - Ok(()) => {} - Err(e) => { - error!("Failed to spawn client RPC system: {}", e); + // Process lines + let mut line = String::new(); + while let Ok(r) = reader + .read_line(&mut line) + .timeout_at(stop_token.clone()) + .await + { + match r { + Ok(size) => { + // Exit on EOF + if size == 0 { + // Disconnected + return Err("Connection closed".to_owned()); + } + } + Err(e) => { + // Disconnected + return Err("Connection lost".to_owned()); + } } - } - // Drop the server and disconnector too (if we still have it) - let mut inner = self.inner.borrow_mut(); - let disconnect_requested = inner.disconnect_requested; - inner.server_settings = None; - inner.server = None; - inner.disconnector = None; - inner.disconnect_requested = false; - inner.connect_addr = None; - - if !disconnect_requested { - // Connection lost - Err("Connection lost".to_owned()) - } else { - // Connection finished - Ok(()) - } - } - - pub fn cancellable(&mut self, p: Promise) -> Promise - where - T: 'static, - { - let (mut cancel_instance, cancel_eventual) = { - let inner = self.inner.borrow(); - ( - inner.cancel_eventual.instance_empty().fuse(), - inner.cancel_eventual.clone(), - ) - }; - let mut p = p.fuse(); - - Promise::from_future(async move { - let out = select! { - a = p => { - a - }, - _ = cancel_instance => { - Err(capnp::Error::failed("cancelled".into())) + // Unmarshal json + let j = match json::parse(line.trim()) { + Ok(v) => v, + Err(e) => { + error!("failed to parse server response: {}", e); + continue; } }; - drop(cancel_instance); - cancel_eventual.reset(); - out - }) + + if j["type"] == "Update" { + self.process_update(j).await; + } + } + + // Connection finished + Ok(()) + + // let rpc_network = Box::new(twoparty::VatNetwork::new( + // reader, + // writer, + // rpc_twoparty_capnp::Side::Client, + // Default::default(), + // )); + + // // Create the rpc system + // let rpc_system = RpcSystem::new(rpc_network, None); + + // // Process the rpc system until we decide we're done + // match self.spawn_rpc_system(connect_addr, rpc_system).await { + // Ok(()) => {} + // Err(e) => { + // error!("Failed to spawn client RPC system: {}", e); + // } + // } + + // // Drop the server and disconnector too (if we still have it) + // let mut inner = self.inner.borrow_mut(); + // let disconnect_requested = inner.disconnect_requested; + // inner.server_settings = None; + // inner.server = None; + // inner.disconnector = None; + // inner.disconnect_requested = false; + // inner.connect_addr = None; } + // pub fn cancellable(&mut self, p: Promise) -> Promise + // where + // T: 'static, + // { + // let (mut cancel_instance, cancel_eventual) = { + // let inner = self.inner.borrow(); + // ( + // inner.cancel_eventual.instance_empty().fuse(), + // inner.cancel_eventual.clone(), + // ) + // }; + // let mut p = p.fuse(); + + // Promise::from_future(async move { + // let out = select! { + // a = p => { + // a + // }, + // _ = cancel_instance => { + // Err(capnp::Error::failed("cancelled".into())) + // } + // }; + // drop(cancel_instance); + // cancel_eventual.reset(); + // out + // }) + // } + pub async fn server_attach(&mut self) -> Result<(), String> { trace!("ClientApiConnection::server_attach"); let server = { diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index 60fe90d7..f47ca014 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -5,8 +5,7 @@ use std::cell::*; use std::net::SocketAddr; use std::rc::Rc; use std::time::SystemTime; -use veilid_core::tools::*; -use veilid_core::*; +use veilid_tools::*; pub fn convert_loglevel(s: &str) -> Result { match s.to_ascii_lowercase().as_str() { @@ -387,7 +386,11 @@ reply - reply to an AppCall not handled directly by the server // calls into ui //////////////////////////////////////////// - pub fn update_attachment(&mut self, attachment: veilid_core::VeilidStateAttachment) { + pub fn log_message(&mut self, message: String) { + self.inner().ui.add_node_event(message); + } + + pub fn update_attachment(&mut self, attachment: json::JsonValue) { self.inner_mut().ui.set_attachment_state( attachment.state, attachment.public_internet_ready, @@ -424,17 +427,17 @@ reply - reply to an AppCall not handled directly by the server self.inner().ui.add_node_event(out); } } - pub fn update_value_change(&mut self, value_change: veilid_core::VeilidValueChange) { - let out = format!("Value change: {:?}", value_change); + pub fn update_value_change(&mut self, value_change: json::JsonValue) { + let out = format!("Value change: {:?}", value_change.as_str().unwrap_or("???")); self.inner().ui.add_node_event(out); } - pub fn update_log(&mut self, log: veilid_core::VeilidLog) { + pub fn update_log(&mut self, log: json::JsonValue) { self.inner().ui.add_node_event(format!( "{}: {}{}", - log.log_level, - log.message, - if let Some(bt) = log.backtrace { + log["log_level"].as_str().unwrap_or("???"), + log["message"].as_str().unwrap_or("???"), + if let Some(bt) = log["backtrace"].as_str() { format!("\nBacktrace:\n{}", bt) } else { "".to_owned() @@ -442,7 +445,7 @@ reply - reply to an AppCall not handled directly by the server )); } - pub fn update_app_message(&mut self, msg: veilid_core::VeilidAppMessage) { + pub fn update_app_message(&mut self, msg: json::JsonValue) { // check is message body is ascii printable let mut printable = true; for c in msg.message() { diff --git a/veilid-cli/src/main.rs b/veilid-cli/src/main.rs index 73ac351d..517a02e2 100644 --- a/veilid-cli/src/main.rs +++ b/veilid-cli/src/main.rs @@ -3,7 +3,7 @@ #![recursion_limit = "256"] use crate::tools::*; -use veilid_core::tools::*; +use veilid_tools::*; use clap::{Arg, ColorChoice, Command}; use flexi_logger::*; diff --git a/veilid-cli/src/peers_table_view.rs b/veilid-cli/src/peers_table_view.rs index ff328476..e51aaf62 100644 --- a/veilid-cli/src/peers_table_view.rs +++ b/veilid-cli/src/peers_table_view.rs @@ -1,7 +1,6 @@ use super::*; use cursive_table_view::*; use std::cmp::Ordering; -use veilid_core::*; #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub enum PeerTableColumn { diff --git a/veilid-cli/src/ui.rs b/veilid-cli/src/ui.rs index 48b38db9..9a7c8ed3 100644 --- a/veilid-cli/src/ui.rs +++ b/veilid-cli/src/ui.rs @@ -12,12 +12,11 @@ use cursive::Cursive; use cursive::CursiveRunnable; use cursive_flexi_logger_view::{CursiveLogWriter, FlexiLoggerView}; //use cursive_multiplex::*; -use log::*; use std::cell::RefCell; use std::collections::{HashMap, VecDeque}; use std::rc::Rc; use thiserror::Error; -use veilid_core::*; +use veilid_tools::*; ////////////////////////////////////////////////////////////// /// @@ -50,20 +49,20 @@ impl Dirty { pub type UICallback = Box; struct UIState { - attachment_state: Dirty, + attachment_state: Dirty, public_internet_ready: Dirty, local_network_ready: Dirty, network_started: Dirty, network_down_up: Dirty<(f32, f32)>, connection_state: Dirty, - peers_state: Dirty>, + peers_state: Dirty>, node_id: Dirty, } impl UIState { pub fn new() -> Self { Self { - attachment_state: Dirty::new(AttachmentState::Detached), + attachment_state: Dirty::new("Detached".to_owned()), public_internet_ready: Dirty::new(false), local_network_ready: Dirty::new(false), network_started: Dirty::new(false), @@ -239,15 +238,16 @@ impl UI { s.find_name("peers").unwrap() } fn render_attachment_state(inner: &mut UIInner) -> String { - let att = match inner.ui_state.attachment_state.get() { - AttachmentState::Detached => "[----]", - AttachmentState::Attaching => "[/ ]", - AttachmentState::AttachedWeak => "[| ]", - AttachmentState::AttachedGood => "[|| ]", - AttachmentState::AttachedStrong => "[||| ]", - AttachmentState::FullyAttached => "[||||]", - AttachmentState::OverAttached => "[++++]", - AttachmentState::Detaching => "[////]", + let att = match inner.ui_state.attachment_state.get().as_str() { + "Detached" => "[----]", + "Attaching" => "[/ ]", + "AttachedWeak" => "[| ]", + "AttachedGood" => "[|| ]", + "AttachedStrong" => "[||| ]", + "FullyAttached" => "[||||]", + "OverAttached" => "[++++]", + "Detaching" => "[////]", + _ => "[????]", }; let pi = if *inner.ui_state.public_internet_ready.get() { "+P" @@ -272,15 +272,16 @@ impl UI { } fn render_button_attach<'a>(inner: &mut UIInner) -> (&'a str, bool) { if let ConnectionState::Connected(_, _) = inner.ui_state.connection_state.get() { - match inner.ui_state.attachment_state.get() { - AttachmentState::Detached => ("Attach", true), - AttachmentState::Attaching => ("Detach", true), - AttachmentState::AttachedWeak => ("Detach", true), - AttachmentState::AttachedGood => ("Detach", true), - AttachmentState::AttachedStrong => ("Detach", true), - AttachmentState::FullyAttached => ("Detach", true), - AttachmentState::OverAttached => ("Detach", true), - AttachmentState::Detaching => ("Detach", false), + match inner.ui_state.attachment_state.get().as_str() { + "Detached" => ("Attach", true), + "Attaching" => ("Detach", true), + "AttachedWeak" => ("Detach", true), + "AttachedGood" => ("Detach", true), + "AttachedStrong" => ("Detach", true), + "FullyAttached" => ("Detach", true), + "OverAttached" => ("Detach", true), + "Detaching" => ("Detach", false), + _ => ("???", false), } } else { (" ---- ", false) @@ -412,15 +413,17 @@ impl UI { } fn on_button_attach_pressed(s: &mut Cursive) { - let action: Option = match Self::inner_mut(s).ui_state.attachment_state.get() { - AttachmentState::Detached => Some(true), - AttachmentState::Attaching => Some(false), - AttachmentState::AttachedWeak => Some(false), - AttachmentState::AttachedGood => Some(false), - AttachmentState::AttachedStrong => Some(false), - AttachmentState::FullyAttached => Some(false), - AttachmentState::OverAttached => Some(false), - AttachmentState::Detaching => None, + let action: Option = match Self::inner_mut(s).ui_state.attachment_state.get().as_str() + { + "Detached" => Some(true), + "Attaching" => Some(false), + "AttachedWeak" => Some(false), + "AttachedGood" => Some(false), + "AttachedStrong" => Some(false), + "FullyAttached" => Some(false), + "OverAttached" => Some(false), + "Detaching" => None, + _ => None, }; let mut cmdproc = Self::command_processor(s); if let Some(a) = action { diff --git a/veilid-tools/src/lib.rs b/veilid-tools/src/lib.rs index ed8e90c8..788e143d 100644 --- a/veilid-tools/src/lib.rs +++ b/veilid-tools/src/lib.rs @@ -140,8 +140,5 @@ pub use wasm::*; pub mod tests; // For iOS tests - #[no_mangle] -pub extern "C" fn main_rs() { - // start game code here -} +pub extern "C" fn main_rs() {} From 419bfcd8ce85943969d3730cd50699d7e1fafa17 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 8 Jun 2023 14:07:09 -0400 Subject: [PATCH 09/23] checkpoint --- Cargo.lock | 1 + veilid-cli/Cargo.toml | 3 +- veilid-cli/src/client_api_connection.rs | 509 ++++++++---------- veilid-cli/src/command_processor.rs | 208 +++---- veilid-cli/src/main.rs | 9 +- veilid-cli/src/peers_table_view.rs | 75 ++- veilid-cli/src/tools.rs | 17 +- veilid-cli/src/ui.rs | 208 +++---- .../src/intf/native/protected_store.rs | 2 +- veilid-core/src/lib.rs | 25 +- .../network_manager/native/start_protocols.rs | 6 +- veilid-server/src/client_api.rs | 8 +- veilid-server/src/main.rs | 4 - veilid-server/src/server.rs | 1 + veilid-server/src/settings.rs | 1 + veilid-server/src/tools.rs | 7 +- veilid-tools/src/lib.rs | 37 +- veilid-tools/src/network_result.rs | 16 +- veilid-tools/src/tools.rs | 28 +- 19 files changed, 563 insertions(+), 602 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f841b887..eddd7050 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6158,6 +6158,7 @@ dependencies = [ "cursive-flexi-logger-view", "cursive_buffered_backend", "cursive_table_view", + "data-encoding", "directories", "flexi_logger", "flume", diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index 021275d2..1701aa1e 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -41,10 +41,11 @@ flexi_logger = { version = "^0", features = ["use_chrono_for_offset"] } thiserror = "^1" crossbeam-channel = "^0" hex = "^0" -veilid-tools = { path = "../veilid-tools", default-features = false } +veilid-tools = { path = "../veilid-tools" } json = "^0" stop-token = { version = "^0", default-features = false } flume = { version = "^0", features = ["async"] } +data-encoding = { version = "^2" } [dev-dependencies] serial_test = "^0" diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index 21d4ba5c..b7e12670 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -1,12 +1,12 @@ use crate::command_processor::*; use crate::tools::*; -use serde::de::DeserializeOwned; -use std::cell::RefCell; +use core::str::FromStr; +use futures::stream::FuturesUnordered; +use futures::StreamExt; use std::net::SocketAddr; -use std::rc::Rc; -use stop_token::{future::FutureExt as _, StopSource, StopToken}; +use std::time::SystemTime; +use stop_token::{future::FutureExt as _, StopSource}; -use veilid_tools::*; cfg_if! { if #[cfg(feature="rt-async-std")] { use async_std::io::prelude::BufReadExt; @@ -19,80 +19,41 @@ cfg_if! { } } -// fn map_to_internal_error(e: T) -> VeilidAPIError { -// VeilidAPIError::Internal { -// message: e.to_string(), -// } -// } - -// fn decode_api_result( -// reader: &api_result::Reader, -// ) -> VeilidAPIResult { -// match reader.which().map_err(map_to_internal_error)? { -// api_result::Which::Ok(v) => { -// let ok_val = v.map_err(map_to_internal_error)?; -// let res: T = veilid_core::deserialize_json(ok_val).map_err(map_to_internal_error)?; -// Ok(res) -// } -// api_result::Which::Err(e) => { -// let err_val = e.map_err(map_to_internal_error)?; -// let res: VeilidAPIError = -// veilid_core::deserialize_json(err_val).map_err(map_to_internal_error)?; -// Err(res) -// } -// } -// } - -// struct VeilidClientImpl { -// comproc: CommandProcessor, -// } - -// impl VeilidClientImpl { -// pub fn new(comproc: CommandProcessor) -> Self { -// Self { comproc } -// } -// } - -// } - struct ClientApiConnectionInner { comproc: CommandProcessor, connect_addr: Option, - server: Option>, + request_sender: Option>, server_settings: Option, disconnector: Option, disconnect_requested: bool, - cancel_eventual: Eventual, + reply_channels: HashMap>, + next_req_id: u32, } -type Handle = Rc>; - #[derive(Clone)] pub struct ClientApiConnection { - inner: Handle, + inner: Arc>, } impl ClientApiConnection { pub fn new(comproc: CommandProcessor) -> Self { Self { - inner: Rc::new(RefCell::new(ClientApiConnectionInner { + inner: Arc::new(Mutex::new(ClientApiConnectionInner { comproc, connect_addr: None, - server: None, + request_sender: None, server_settings: None, disconnector: None, disconnect_requested: false, - cancel_eventual: Eventual::new(), + reply_channels: HashMap::new(), + next_req_id: 0, })), } } - pub fn cancel(&self) { - let eventual = { - let inner = self.inner.borrow(); - inner.cancel_eventual.clone() - }; - eventual.resolve(); // don't need to await this + pub fn cancel_all(&self) { + let mut inner = self.inner.lock(); + inner.reply_channels.clear(); } // async fn process_veilid_state<'a>( @@ -106,8 +67,33 @@ impl ClientApiConnection { // Ok(()) // } + async fn process_response(&self, response: json::JsonValue) { + // find the operation id and send the response to the channel for it + let Some(id_str) = response["id"].as_str() else { + error!("missing id: {}", response); + return; + }; + let Ok(id) = u32::from_str(id_str) else { + error!("invalid id: {}", response); + return; + }; + + let reply_channel = { + let mut inner = self.inner.lock(); + inner.reply_channels.remove(&id) + }; + let Some(reply_channel) = reply_channel else { + warn!("received cancelled reply: {}", response); + return; + }; + if let Err(e) = reply_channel.send_async(response).await { + error!("failed to process reply: {}", e); + return; + } + } + async fn process_update(&self, update: json::JsonValue) { - let comproc = self.inner.borrow().comproc.clone(); + let comproc = self.inner.lock().comproc.clone(); let Some(kind) = update["kind"].as_str() else { comproc.log_message(format!("missing update kind: {}", update)); return; @@ -145,7 +131,7 @@ impl ClientApiConnection { } // async fn spawn_rpc_system( - // &mut self, + // &self, // connect_addr: SocketAddr, // mut rpc_system: RpcSystem, // ) -> Result<(), String> { @@ -235,18 +221,9 @@ impl ClientApiConnection { // res.map_err(|e| format!("client RPC system error: {}", e)) // } - async fn handle_connection(&mut self, connect_addr: SocketAddr) -> Result<(), String> { + async fn handle_connection(&self, connect_addr: SocketAddr) -> Result<(), String> { trace!("ClientApiConnection::handle_connection"); - let stop_token = { - let stop_source = StopSource::new(); - let token = stop_source.token(); - let mut inner = self.inner.borrow_mut(); - inner.connect_addr = Some(connect_addr); - inner.disconnector = Some(stop_source); - token - }; - // Connect the TCP socket let stream = TcpStream::connect(connect_addr) .await @@ -255,283 +232,245 @@ impl ClientApiConnection { // If it succeed, disable nagle algorithm stream.set_nodelay(true).map_err(map_to_string)?; + // State we connected + let comproc = self.inner.lock().comproc.clone(); + comproc.set_connection_state(ConnectionState::Connected(connect_addr, SystemTime::now())); + // Split the stream cfg_if! { if #[cfg(feature="rt-async-std")] { use futures::AsyncReadExt; - let (reader, writer) = stream.split(); + let (reader, mut writer) = stream.split(); let mut reader = BufReader::new(reader); } else if #[cfg(feature="rt-tokio")] { - let (reader, writer) = stream.into_split(); + let (reader, mut writer) = stream.into_split(); let mut reader = BufReader::new(reader); } } + // Requests to send + let (requests_tx, requests_rx) = flume::unbounded(); + + let stop_token = { + let stop_source = StopSource::new(); + let token = stop_source.token(); + let mut inner = self.inner.lock(); + inner.connect_addr = Some(connect_addr); + inner.disconnector = Some(stop_source); + inner.request_sender = Some(requests_tx); + token + }; + + // Futures to process unordered + let mut unord = FuturesUnordered::new(); + // Process lines - let mut line = String::new(); - while let Ok(r) = reader - .read_line(&mut line) - .timeout_at(stop_token.clone()) - .await - { - match r { - Ok(size) => { - // Exit on EOF - if size == 0 { - // Disconnected - return Err("Connection closed".to_owned()); - } - } - Err(e) => { + let this = self.clone(); + let recv_messages_future = async move { + let mut line = String::new(); + while let Ok(size) = reader.read_line(&mut line).await { + // Exit on EOF + if size == 0 { // Disconnected - return Err("Connection lost".to_owned()); + break; + } + + // Unmarshal json + let j = match json::parse(line.trim()) { + Ok(v) => v, + Err(e) => { + error!("failed to parse server response: {}", e); + continue; + } + }; + + if j["type"] == "Update" { + this.process_update(j).await; + } else if j["type"] == "Response" { + this.process_response(j).await; } } + // + let mut inner = this.inner.lock(); + inner.request_sender = None; + }; + unord.push(system_boxed(recv_messages_future)); - // Unmarshal json - let j = match json::parse(line.trim()) { - Ok(v) => v, - Err(e) => { - error!("failed to parse server response: {}", e); - continue; + // Requests send processor + let send_requests_future = async move { + while let Ok(req) = requests_rx.recv_async().await { + if let Err(e) = writer.write_all(req.as_bytes()).await { + error!("failed to write request: {}", e) } - }; - - if j["type"] == "Update" { - self.process_update(j).await; } - } + }; + unord.push(system_boxed(send_requests_future)); - // Connection finished - Ok(()) - - // let rpc_network = Box::new(twoparty::VatNetwork::new( - // reader, - // writer, - // rpc_twoparty_capnp::Side::Client, - // Default::default(), - // )); - - // // Create the rpc system - // let rpc_system = RpcSystem::new(rpc_network, None); - - // // Process the rpc system until we decide we're done - // match self.spawn_rpc_system(connect_addr, rpc_system).await { - // Ok(()) => {} - // Err(e) => { - // error!("Failed to spawn client RPC system: {}", e); - // } - // } + // Send and receive until we're done or a stop is requested + while let Ok(Some(())) = unord.next().timeout_at(stop_token.clone()).await {} // // Drop the server and disconnector too (if we still have it) - // let mut inner = self.inner.borrow_mut(); - // let disconnect_requested = inner.disconnect_requested; - // inner.server_settings = None; - // inner.server = None; - // inner.disconnector = None; - // inner.disconnect_requested = false; - // inner.connect_addr = None; + let mut inner = self.inner.lock(); + let disconnect_requested = inner.disconnect_requested; + inner.server_settings = None; + inner.request_sender = None; + inner.disconnector = None; + inner.disconnect_requested = false; + inner.connect_addr = None; + + // Connection finished + if disconnect_requested { + Ok(()) + } else { + Err("Connection lost".to_owned()) + } } - // pub fn cancellable(&mut self, p: Promise) -> Promise - // where - // T: 'static, - // { - // let (mut cancel_instance, cancel_eventual) = { - // let inner = self.inner.borrow(); - // ( - // inner.cancel_eventual.instance_empty().fuse(), - // inner.cancel_eventual.clone(), - // ) - // }; - // let mut p = p.fuse(); + async fn perform_request(&self, mut req: json::JsonValue) -> Option { + let (sender, reply_rx) = { + let mut inner = self.inner.lock(); - // Promise::from_future(async move { - // let out = select! { - // a = p => { - // a - // }, - // _ = cancel_instance => { - // Err(capnp::Error::failed("cancelled".into())) - // } - // }; - // drop(cancel_instance); - // cancel_eventual.reset(); - // out - // }) - // } + // Get the request sender + let Some(sender) = inner.request_sender.clone() else { + error!("dropping request, not connected"); + return None; + }; - pub async fn server_attach(&mut self) -> Result<(), String> { + // Get next id + let id = inner.next_req_id; + inner.next_req_id += 1; + + // Add the id + req["id"] = id.into(); + + // Make a reply receiver + let (reply_tx, reply_rx) = flume::bounded(1); + inner.reply_channels.insert(id, reply_tx); + (sender, reply_rx) + }; + + // Send the request + let req_ndjson = req.dump() + "\n"; + if let Err(e) = sender.send_async(req_ndjson).await { + error!("failed to send request: {}", e); + return None; + } + + // Wait for the reply + let Ok(r) = reply_rx.recv_async().await else { + // Cancelled + return None; + }; + + Some(r) + } + + pub async fn server_attach(&self) -> Result<(), String> { trace!("ClientApiConnection::server_attach"); - let server = { - let inner = self.inner.borrow(); - inner - .server - .as_ref() - .ok_or_else(|| "Not connected, ignoring attach request".to_owned())? - .clone() + + let mut req = json::JsonValue::new_object(); + req["op"] = "Attach".into(); + let Some(resp) = self.perform_request(req).await else { + return Err("Cancelled".to_owned()); }; - let request = server.borrow().attach_request(); - let response = self - .cancellable(request.send().promise) - .await - .map_err(map_to_string)?; - let reader = response - .get() - .map_err(map_to_string)? - .get_result() - .map_err(map_to_string)?; - let res: VeilidAPIResult<()> = decode_api_result(&reader); - res.map_err(map_to_string) + if resp.has_key("error") { + return Err(resp["error"].to_string()); + } + Ok(()) } - pub async fn server_detach(&mut self) -> Result<(), String> { + pub async fn server_detach(&self) -> Result<(), String> { trace!("ClientApiConnection::server_detach"); - let server = { - let inner = self.inner.borrow(); - inner - .server - .as_ref() - .ok_or_else(|| "Not connected, ignoring detach request".to_owned())? - .clone() + let mut req = json::JsonValue::new_object(); + req["op"] = "Detach".into(); + let Some(resp) = self.perform_request(req).await else { + return Err("Cancelled".to_owned()); }; - let request = server.borrow().detach_request(); - let response = self - .cancellable(request.send().promise) - .await - .map_err(map_to_string)?; - let reader = response - .get() - .map_err(map_to_string)? - .get_result() - .map_err(map_to_string)?; - let res: VeilidAPIResult<()> = decode_api_result(&reader); - res.map_err(map_to_string) + if resp.has_key("error") { + return Err(resp["error"].to_string()); + } + Ok(()) } - pub async fn server_shutdown(&mut self) -> Result<(), String> { + pub async fn server_shutdown(&self) -> Result<(), String> { trace!("ClientApiConnection::server_shutdown"); - let server = { - let inner = self.inner.borrow(); - inner - .server - .as_ref() - .ok_or_else(|| "Not connected, ignoring attach request".to_owned())? - .clone() + let mut req = json::JsonValue::new_object(); + req["op"] = "Control".into(); + req["args"] = json::JsonValue::new_array(); + req["args"].push("Shutdown").unwrap(); + let Some(resp) = self.perform_request(req).await else { + return Err("Cancelled".to_owned()); }; - let request = server.borrow().shutdown_request(); - let response = self - .cancellable(request.send().promise) - .await - .map_err(map_to_string)?; - response.get().map(drop).map_err(map_to_string) + if resp.has_key("error") { + return Err(resp["error"].to_string()); + } + Ok(()) } - pub async fn server_debug(&mut self, what: String) -> Result { + pub async fn server_debug(&self, what: String) -> Result { trace!("ClientApiConnection::server_debug"); - let server = { - let inner = self.inner.borrow(); - inner - .server - .as_ref() - .ok_or_else(|| "Not connected, ignoring debug request".to_owned())? - .clone() + let mut req = json::JsonValue::new_object(); + req["op"] = "Debug".into(); + req["command"] = what.into(); + let Some(resp) = self.perform_request(req).await else { + return Err("Cancelled".to_owned()); }; - let mut request = server.borrow().debug_request(); - request.get().set_command(&what); - let response = self - .cancellable(request.send().promise) - .await - .map_err(map_to_string)?; - let reader = response - .get() - .map_err(map_to_string)? - .get_result() - .map_err(map_to_string)?; - let res: VeilidAPIResult = decode_api_result(&reader); - res.map_err(map_to_string) + if resp.has_key("error") { + return Err(resp["error"].to_string()); + } + Ok(resp["value"].to_string()) } pub async fn server_change_log_level( - &mut self, + &self, layer: String, - log_level: VeilidConfigLogLevel, + log_level: String, ) -> Result<(), String> { trace!("ClientApiConnection::change_log_level"); - let server = { - let inner = self.inner.borrow(); - inner - .server - .as_ref() - .ok_or_else(|| "Not connected, ignoring change_log_level request".to_owned())? - .clone() + let mut req = json::JsonValue::new_object(); + req["op"] = "Control".into(); + req["args"] = json::JsonValue::new_array(); + req["args"].push("ChangeLogLevel").unwrap(); + req["args"].push(layer).unwrap(); + req["args"].push(log_level).unwrap(); + let Some(resp) = self.perform_request(req).await else { + return Err("Cancelled".to_owned()); }; - let mut request = server.borrow().change_log_level_request(); - request.get().set_layer(&layer); - let log_level_json = veilid_core::serialize_json(&log_level); - request.get().set_log_level(&log_level_json); - let response = self - .cancellable(request.send().promise) - .await - .map_err(map_to_string)?; - let reader = response - .get() - .map_err(map_to_string)? - .get_result() - .map_err(map_to_string)?; - let res: VeilidAPIResult<()> = decode_api_result(&reader); - res.map_err(map_to_string) + if resp.has_key("error") { + return Err(resp["error"].to_string()); + } + Ok(()) } - pub async fn server_appcall_reply( - &mut self, - id: OperationId, - msg: Vec, - ) -> Result<(), String> { + pub async fn server_appcall_reply(&self, id: u64, msg: Vec) -> Result<(), String> { trace!("ClientApiConnection::appcall_reply"); - let server = { - let inner = self.inner.borrow(); - inner - .server - .as_ref() - .ok_or_else(|| "Not connected, ignoring change_log_level request".to_owned())? - .clone() + let mut req = json::JsonValue::new_object(); + req["op"] = "AppCallReply".into(); + req["call_id"] = id.to_string().into(); + req["message"] = data_encoding::BASE64URL_NOPAD.encode(&msg).into(); + let Some(resp) = self.perform_request(req).await else { + return Err("Cancelled".to_owned()); }; - let mut request = server.borrow().app_call_reply_request(); - request.get().set_id(id.as_u64()); - request.get().set_message(&msg); - let response = self - .cancellable(request.send().promise) - .await - .map_err(map_to_string)?; - let reader = response - .get() - .map_err(map_to_string)? - .get_result() - .map_err(map_to_string)?; - let res: VeilidAPIResult<()> = decode_api_result(&reader); - res.map_err(map_to_string) + if resp.has_key("error") { + return Err(resp["error"].to_string()); + } + Ok(()) } // Start Client API connection - pub async fn connect(&mut self, connect_addr: SocketAddr) -> Result<(), String> { + pub async fn connect(&self, connect_addr: SocketAddr) -> Result<(), String> { trace!("ClientApiConnection::connect"); // Save the address to connect to self.handle_connection(connect_addr).await } // End Client API connection - pub async fn disconnect(&mut self) { + pub async fn disconnect(&self) { trace!("ClientApiConnection::disconnect"); - let disconnector = self.inner.borrow_mut().disconnector.take(); - match disconnector { - Some(d) => { - self.inner.borrow_mut().disconnect_requested = true; - d.await.unwrap(); - } - None => { - debug!("disconnector doesn't exist"); - } + let mut inner = self.inner.lock(); + if inner.disconnector.is_some() { + inner.disconnector = None; + inner.disconnect_requested = true; } } } diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index f47ca014..6cbc6331 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -1,20 +1,19 @@ use crate::client_api_connection::*; use crate::settings::Settings; +use crate::tools::*; use crate::ui::*; -use std::cell::*; use std::net::SocketAddr; -use std::rc::Rc; use std::time::SystemTime; use veilid_tools::*; -pub fn convert_loglevel(s: &str) -> Result { +pub fn convert_loglevel(s: &str) -> Result { match s.to_ascii_lowercase().as_str() { - "off" => Ok(VeilidConfigLogLevel::Off), - "error" => Ok(VeilidConfigLogLevel::Error), - "warn" => Ok(VeilidConfigLogLevel::Warn), - "info" => Ok(VeilidConfigLogLevel::Info), - "debug" => Ok(VeilidConfigLogLevel::Debug), - "trace" => Ok(VeilidConfigLogLevel::Trace), + "off" => Ok("Off".to_owned()), + "error" => Ok("Error".to_owned()), + "warn" => Ok("Warn".to_owned()), + "info" => Ok("Info".to_owned()), + "debug" => Ok("Debug".to_owned()), + "trace" => Ok("Trace".to_owned()), _ => Err(format!("Invalid log level: {}", s)), } } @@ -38,7 +37,7 @@ impl ConnectionState { } struct CommandProcessorInner { - ui: UI, + ui_sender: UISender, capi: Option, reconnect: bool, finished: bool, @@ -46,21 +45,19 @@ struct CommandProcessorInner { autoreconnect: bool, server_addr: Option, connection_waker: Eventual, - last_call_id: Option, + last_call_id: Option, } -type Handle = Rc>; - #[derive(Clone)] pub struct CommandProcessor { - inner: Handle, + inner: Arc>, } impl CommandProcessor { - pub fn new(ui: UI, settings: &Settings) -> Self { + pub fn new(ui_sender: UISender, settings: &Settings) -> Self { Self { - inner: Rc::new(RefCell::new(CommandProcessorInner { - ui, + inner: Arc::new(Mutex::new(CommandProcessorInner { + ui_sender, capi: None, reconnect: settings.autoreconnect, finished: false, @@ -72,20 +69,20 @@ impl CommandProcessor { })), } } - pub fn set_client_api_connection(&mut self, capi: ClientApiConnection) { - self.inner.borrow_mut().capi = Some(capi); + pub fn set_client_api_connection(&self, capi: ClientApiConnection) { + self.inner.lock().capi = Some(capi); } - fn inner(&self) -> Ref { - self.inner.borrow() + fn inner(&self) -> MutexGuard { + self.inner.lock() } - fn inner_mut(&self) -> RefMut { - self.inner.borrow_mut() + fn inner_mut(&self) -> MutexGuard { + self.inner.lock() } - fn ui(&self) -> UI { - self.inner.borrow().ui.clone() + fn ui_sender(&self) -> UISender { + self.inner.lock().ui_sender.clone() } fn capi(&self) -> ClientApiConnection { - self.inner.borrow().capi.as_ref().unwrap().clone() + self.inner.lock().capi.as_ref().unwrap().clone() } fn word_split(line: &str) -> (String, Option) { @@ -102,12 +99,12 @@ impl CommandProcessor { pub fn cancel_command(&self) { trace!("CommandProcessor::cancel_command"); let capi = self.capi(); - capi.cancel(); + capi.cancel_all(); } pub fn cmd_help(&self, _rest: Option, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_help"); - self.ui().add_node_event( + self.ui_sender().add_node_event( r#"Commands: exit/quit - exit the client disconnect - disconnect the client from the Veilid node @@ -120,14 +117,14 @@ reply - reply to an AppCall not handled directly by the server "# .to_owned(), ); - let ui = self.ui(); + let ui = self.ui_sender(); ui.send_callback(callback); Ok(()) } pub fn cmd_exit(&self, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_exit"); - let ui = self.ui(); + let ui = self.ui_sender(); ui.send_callback(callback); ui.quit(); Ok(()) @@ -135,8 +132,8 @@ reply - reply to an AppCall not handled directly by the server pub fn cmd_shutdown(&self, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_shutdown"); - let mut capi = self.capi(); - let ui = self.ui(); + let capi = self.capi(); + let ui = self.ui_sender(); spawn_detached_local(async move { if let Err(e) = capi.server_shutdown().await { error!("Server command 'shutdown' failed to execute: {}", e); @@ -148,8 +145,8 @@ reply - reply to an AppCall not handled directly by the server pub fn cmd_attach(&self, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_attach"); - let mut capi = self.capi(); - let ui = self.ui(); + let capi = self.capi(); + let ui = self.ui_sender(); spawn_detached_local(async move { if let Err(e) = capi.server_attach().await { error!("Server command 'attach' failed: {}", e); @@ -161,8 +158,8 @@ reply - reply to an AppCall not handled directly by the server pub fn cmd_detach(&self, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_detach"); - let mut capi = self.capi(); - let ui = self.ui(); + let capi = self.capi(); + let ui = self.ui_sender(); spawn_detached_local(async move { if let Err(e) = capi.server_detach().await { error!("Server command 'detach' failed: {}", e); @@ -174,8 +171,8 @@ reply - reply to an AppCall not handled directly by the server pub fn cmd_disconnect(&self, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_disconnect"); - let mut capi = self.capi(); - let ui = self.ui(); + let capi = self.capi(); + let ui = self.ui_sender(); spawn_detached_local(async move { capi.disconnect().await; ui.send_callback(callback); @@ -185,8 +182,8 @@ reply - reply to an AppCall not handled directly by the server pub fn cmd_debug(&self, rest: Option, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_debug"); - let mut capi = self.capi(); - let ui = self.ui(); + let capi = self.capi(); + let ui = self.ui_sender(); spawn_detached_local(async move { match capi.server_debug(rest.unwrap_or_default()).await { Ok(output) => ui.display_string_dialog("Debug Output", output, callback), @@ -202,8 +199,8 @@ reply - reply to an AppCall not handled directly by the server callback: UICallback, ) -> Result<(), String> { trace!("CommandProcessor::cmd_change_log_level"); - let mut capi = self.capi(); - let ui = self.ui(); + let capi = self.capi(); + let ui = self.ui_sender(); spawn_detached_local(async move { let (layer, rest) = Self::word_split(&rest.unwrap_or_default()); let log_level = match convert_loglevel(&rest.unwrap_or_default()) { @@ -234,8 +231,8 @@ reply - reply to an AppCall not handled directly by the server pub fn cmd_reply(&self, rest: Option, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_reply"); - let mut capi = self.capi(); - let ui = self.ui(); + let capi = self.capi(); + let ui = self.ui_sender(); let some_last_id = self.inner_mut().last_call_id.take(); spawn_detached_local(async move { let (first, second) = Self::word_split(&rest.clone().unwrap_or_default()); @@ -248,7 +245,7 @@ reply - reply to an AppCall not handled directly by the server } Ok(v) => v, }; - (OperationId::new(id), second) + (id, second) } else { let id = match some_last_id { None => { @@ -306,14 +303,14 @@ reply - reply to an AppCall not handled directly by the server "change_log_level" => self.cmd_change_log_level(rest, callback), "reply" => self.cmd_reply(rest, callback), _ => { - let ui = self.ui(); + let ui = self.ui_sender(); ui.send_callback(callback); Err(format!("Invalid command: {}", cmd)) } } } - pub async fn connection_manager(&mut self) { + pub async fn connection_manager(&self) { // Connect until we're done while !self.inner_mut().finished { // Wait for connection request @@ -341,7 +338,7 @@ reply - reply to an AppCall not handled directly by the server } else { debug!("Retrying connection to {}", server_addr); } - let mut capi = self.capi(); + let capi = self.capi(); let res = capi.connect(server_addr).await; if res.is_ok() { info!( @@ -376,7 +373,7 @@ reply - reply to an AppCall not handled directly by the server // called by ui //////////////////////////////////////////// - pub fn set_server_address(&mut self, server_addr: Option) { + pub fn set_server_address(&self, server_addr: Option) { self.inner_mut().server_addr = server_addr; } pub fn get_server_address(&self) -> Option { @@ -386,54 +383,61 @@ reply - reply to an AppCall not handled directly by the server // calls into ui //////////////////////////////////////////// - pub fn log_message(&mut self, message: String) { - self.inner().ui.add_node_event(message); + pub fn log_message(&self, message: String) { + self.inner().ui_sender.add_node_event(message); } - pub fn update_attachment(&mut self, attachment: json::JsonValue) { - self.inner_mut().ui.set_attachment_state( - attachment.state, - attachment.public_internet_ready, - attachment.local_network_ready, + pub fn update_attachment(&self, attachment: json::JsonValue) { + self.inner_mut().ui_sender.set_attachment_state( + attachment["state"].as_str().unwrap_or_default().to_owned(), + attachment["public_internet_ready"] + .as_bool() + .unwrap_or_default(), + attachment["local_network_ready"] + .as_bool() + .unwrap_or_default(), ); } - pub fn update_network_status(&mut self, network: veilid_core::VeilidStateNetwork) { - self.inner_mut().ui.set_network_status( - network.started, - network.bps_down.as_u64(), - network.bps_up.as_u64(), - network.peers, + pub fn update_network_status(&self, network: json::JsonValue) { + self.inner_mut().ui_sender.set_network_status( + network["started"].as_bool().unwrap_or_default(), + json_str_u64(&network["bps_down"]), + json_str_u64(&network["bps_up"]), + network["peers"] + .members() + .cloned() + .collect::>(), ); } - pub fn update_config(&mut self, config: veilid_core::VeilidStateConfig) { - self.inner_mut().ui.set_config(config.config) + pub fn update_config(&self, config: json::JsonValue) { + self.inner_mut().ui_sender.set_config(&config["config"]) } - pub fn update_route(&mut self, route: veilid_core::VeilidRouteChange) { + pub fn update_route(&self, route: json::JsonValue) { let mut out = String::new(); - if !route.dead_routes.is_empty() { - out.push_str(&format!("Dead routes: {:?}", route.dead_routes)); + if route["dead_routes"].len() != 0 { + out.push_str(&format!("Dead routes: {:?}", route["dead_routes"])); } - if !route.dead_remote_routes.is_empty() { + if route["dead_routes"].len() != 0 { if !out.is_empty() { out.push_str("\n"); } out.push_str(&format!( "Dead remote routes: {:?}", - route.dead_remote_routes + route["dead_remote_routes"] )); } if !out.is_empty() { - self.inner().ui.add_node_event(out); + self.inner().ui_sender.add_node_event(out); } } - pub fn update_value_change(&mut self, value_change: json::JsonValue) { + pub fn update_value_change(&self, value_change: json::JsonValue) { let out = format!("Value change: {:?}", value_change.as_str().unwrap_or("???")); - self.inner().ui.add_node_event(out); + self.inner().ui_sender.add_node_event(out); } - pub fn update_log(&mut self, log: json::JsonValue) { - self.inner().ui.add_node_event(format!( + pub fn update_log(&self, log: json::JsonValue) { + self.inner().ui_sender.add_node_event(format!( "{}: {}{}", log["log_level"].as_str().unwrap_or("???"), log["message"].as_str().unwrap_or("???"), @@ -445,79 +449,83 @@ reply - reply to an AppCall not handled directly by the server )); } - pub fn update_app_message(&mut self, msg: json::JsonValue) { + pub fn update_app_message(&self, msg: json::JsonValue) { + let message = json_str_vec_u8(&msg["message"]); + // check is message body is ascii printable let mut printable = true; - for c in msg.message() { + for c in &message { if *c < 32 || *c > 126 { printable = false; } } let strmsg = if printable { - String::from_utf8_lossy(msg.message()).to_string() + String::from_utf8_lossy(&message).to_string() } else { - hex::encode(msg.message()) + hex::encode(message) }; self.inner() - .ui - .add_node_event(format!("AppMessage ({:?}): {}", msg.sender(), strmsg)); + .ui_sender + .add_node_event(format!("AppMessage ({:?}): {}", msg["sender"], strmsg)); } - pub fn update_app_call(&mut self, call: veilid_core::VeilidAppCall) { + pub fn update_app_call(&self, call: json::JsonValue) { + let message = json_str_vec_u8(&call["message"]); + // check is message body is ascii printable let mut printable = true; - for c in call.message() { + for c in &message { if *c < 32 || *c > 126 { printable = false; } } let strmsg = if printable { - String::from_utf8_lossy(call.message()).to_string() + String::from_utf8_lossy(&message).to_string() } else { - format!("#{}", hex::encode(call.message())) + format!("#{}", hex::encode(&message)) }; - self.inner().ui.add_node_event(format!( + let id = json_str_u64(&call["id"]); + + self.inner().ui_sender.add_node_event(format!( "AppCall ({:?}) id = {:016x} : {}", - call.sender(), - call.id().as_u64(), - strmsg + call["sender"], id, strmsg )); - self.inner_mut().last_call_id = Some(call.id()); + self.inner_mut().last_call_id = Some(id); } - pub fn update_shutdown(&mut self) { + pub fn update_shutdown(&self) { // Do nothing with this, we'll process shutdown when rpc connection closes } // called by client_api_connection // calls into ui //////////////////////////////////////////// - pub fn set_connection_state(&mut self, state: ConnectionState) { - self.inner_mut().ui.set_connection_state(state); + pub fn set_connection_state(&self, state: ConnectionState) { + self.inner_mut().ui_sender.set_connection_state(state); } // called by ui //////////////////////////////////////////// - pub fn start_connection(&mut self) { + pub fn start_connection(&self) { self.inner_mut().reconnect = true; self.inner_mut().connection_waker.resolve(); } - // pub fn stop_connection(&mut self) { + // pub fn stop_connection(&self) { // self.inner_mut().reconnect = false; // let mut capi = self.capi().clone(); // spawn_detached(async move { // capi.disconnect().await; // }); // } - pub fn cancel_reconnect(&mut self) { + pub fn cancel_reconnect(&self) { self.inner_mut().reconnect = false; self.inner_mut().connection_waker.resolve(); } - pub fn quit(&mut self) { + pub fn quit(&self) { self.inner_mut().finished = true; self.inner_mut().reconnect = false; self.inner_mut().connection_waker.resolve(); @@ -526,8 +534,8 @@ reply - reply to an AppCall not handled directly by the server // called by ui // calls into client_api_connection //////////////////////////////////////////// - pub fn attach(&mut self) { - let mut capi = self.capi(); + pub fn attach(&self) { + let capi = self.capi(); spawn_detached_local(async move { if let Err(e) = capi.server_attach().await { @@ -536,8 +544,8 @@ reply - reply to an AppCall not handled directly by the server }); } - pub fn detach(&mut self) { - let mut capi = self.capi(); + pub fn detach(&self) { + let capi = self.capi(); spawn_detached_local(async move { if let Err(e) = capi.server_detach().await { diff --git a/veilid-cli/src/main.rs b/veilid-cli/src/main.rs index 517a02e2..f51c58e3 100644 --- a/veilid-cli/src/main.rs +++ b/veilid-cli/src/main.rs @@ -3,7 +3,6 @@ #![recursion_limit = "256"] use crate::tools::*; -use veilid_tools::*; use clap::{Arg, ColorChoice, Command}; use flexi_logger::*; @@ -92,7 +91,7 @@ fn main() -> Result<(), String> { } // Create UI object - let mut sivui = ui::UI::new(settings.interface.node_log.scrollback, &settings); + let (mut sivui, uisender) = ui::UI::new(settings.interface.node_log.scrollback, &settings); // Set up loggers { @@ -155,19 +154,19 @@ fn main() -> Result<(), String> { // Create command processor debug!("Creating Command Processor "); - let mut comproc = command_processor::CommandProcessor::new(sivui.clone(), &settings); + let comproc = command_processor::CommandProcessor::new(uisender, &settings); sivui.set_command_processor(comproc.clone()); // Create client api client side info!("Starting API connection"); - let mut capi = client_api_connection::ClientApiConnection::new(comproc.clone()); + let capi = client_api_connection::ClientApiConnection::new(comproc.clone()); // Save client api in command processor comproc.set_client_api_connection(capi.clone()); // Keep a connection to the server comproc.set_server_address(server_addr); - let mut comproc2 = comproc.clone(); + let comproc2 = comproc.clone(); let connection_future = comproc.connection_manager(); // Start async diff --git a/veilid-cli/src/peers_table_view.rs b/veilid-cli/src/peers_table_view.rs index e51aaf62..2d158de6 100644 --- a/veilid-cli/src/peers_table_view.rs +++ b/veilid-cli/src/peers_table_view.rs @@ -23,8 +23,11 @@ pub enum PeerTableColumn { // } // } -fn format_ts(ts: Timestamp) -> String { - let ts = ts.as_u64(); +fn format_ts(ts: &json::JsonValue) -> String { + if ts.is_null() { + return "---".to_owned(); + } + let ts = json_str_u64(ts); let secs = timestamp_to_secs(ts); if secs >= 1.0 { format!("{:.2}s", timestamp_to_secs(ts)) @@ -33,8 +36,11 @@ fn format_ts(ts: Timestamp) -> String { } } -fn format_bps(bps: ByteCount) -> String { - let bps = bps.as_u64(); +fn format_bps(bps: &json::JsonValue) -> String { + if bps.is_null() { + return "---".to_owned(); + } + let bps = json_str_u64(bps); if bps >= 1024u64 * 1024u64 * 1024u64 { format!("{:.2}GB/s", (bps / (1024u64 * 1024u64)) as f64 / 1024.0) } else if bps >= 1024u64 * 1024u64 { @@ -46,25 +52,20 @@ fn format_bps(bps: ByteCount) -> String { } } -impl TableViewItem for PeerTableData { +impl TableViewItem for json::JsonValue { fn to_column(&self, column: PeerTableColumn) -> String { match column { - PeerTableColumn::NodeId => self - .node_ids - .first() - .map(|n| n.to_string()) - .unwrap_or_else(|| "???".to_owned()), - PeerTableColumn::Address => self.peer_address.clone(), - PeerTableColumn::LatencyAvg => format!( - "{}", - self.peer_stats - .latency - .as_ref() - .map(|l| format_ts(l.average)) - .unwrap_or("---".to_owned()) - ), - PeerTableColumn::TransferDownAvg => format_bps(self.peer_stats.transfer.down.average), - PeerTableColumn::TransferUpAvg => format_bps(self.peer_stats.transfer.up.average), + PeerTableColumn::NodeId => self["node_ids"][0].to_string(), + PeerTableColumn::Address => self["peer_address"].to_string(), + PeerTableColumn::LatencyAvg => { + format!("{}", format_ts(&self["peer_stats"]["latency"]["average"])) + } + PeerTableColumn::TransferDownAvg => { + format_bps(&self["peer_stats"]["transfer"]["down"]["average"]) + } + PeerTableColumn::TransferUpAvg => { + format_bps(&self["peer_stats"]["transfer"]["up"]["average"]) + } } } @@ -75,26 +76,20 @@ impl TableViewItem for PeerTableData { match column { PeerTableColumn::NodeId => self.to_column(column).cmp(&other.to_column(column)), PeerTableColumn::Address => self.to_column(column).cmp(&other.to_column(column)), - PeerTableColumn::LatencyAvg => self - .peer_stats - .latency - .as_ref() - .map(|l| l.average) - .cmp(&other.peer_stats.latency.as_ref().map(|l| l.average)), - PeerTableColumn::TransferDownAvg => self - .peer_stats - .transfer - .down - .average - .cmp(&other.peer_stats.transfer.down.average), - PeerTableColumn::TransferUpAvg => self - .peer_stats - .transfer - .up - .average - .cmp(&other.peer_stats.transfer.up.average), + PeerTableColumn::LatencyAvg => json_str_u64(&self["peer_stats"]["latency"]["average"]) + .cmp(&json_str_u64(&other["peer_stats"]["latency"]["average"])), + PeerTableColumn::TransferDownAvg => { + json_str_u64(&self["peer_stats"]["transfer"]["down"]["average"]).cmp(&json_str_u64( + &other["peer_stats"]["transfer"]["down"]["average"], + )) + } + PeerTableColumn::TransferUpAvg => { + json_str_u64(&self["peer_stats"]["transfer"]["up"]["average"]).cmp(&json_str_u64( + &other["peer_stats"]["transfer"]["up"]["average"], + )) + } } } } -pub type PeersTableView = TableView; +pub type PeersTableView = TableView; diff --git a/veilid-cli/src/tools.rs b/veilid-cli/src/tools.rs index bf58d24c..7e6fe04c 100644 --- a/veilid-cli/src/tools.rs +++ b/veilid-cli/src/tools.rs @@ -1,5 +1,10 @@ -use cfg_if::*; +pub use cfg_if::*; +pub use log::*; +pub use parking_lot::*; +pub use veilid_tools::*; + use core::future::Future; +use core::str::FromStr; cfg_if! { if #[cfg(feature="rt-async-std")] { @@ -17,3 +22,13 @@ cfg_if! { } } + +pub fn json_str_u64(value: &json::JsonValue) -> u64 { + u64::from_str(value.as_str().unwrap_or_default()).unwrap_or_default() +} + +pub fn json_str_vec_u8(value: &json::JsonValue) -> Vec { + data_encoding::BASE64URL_NOPAD + .decode(value.as_str().unwrap_or_default().as_bytes()) + .unwrap_or_default() +} diff --git a/veilid-cli/src/ui.rs b/veilid-cli/src/ui.rs index 9a7c8ed3..a38ff6dc 100644 --- a/veilid-cli/src/ui.rs +++ b/veilid-cli/src/ui.rs @@ -1,6 +1,7 @@ use crate::command_processor::*; use crate::peers_table_view::*; use crate::settings::Settings; +use crate::tools::*; use crossbeam_channel::Sender; use cursive::align::*; use cursive::event::*; @@ -12,11 +13,8 @@ use cursive::Cursive; use cursive::CursiveRunnable; use cursive_flexi_logger_view::{CursiveLogWriter, FlexiLoggerView}; //use cursive_multiplex::*; -use std::cell::RefCell; use std::collections::{HashMap, VecDeque}; -use std::rc::Rc; use thiserror::Error; -use veilid_tools::*; ////////////////////////////////////////////////////////////// /// @@ -82,19 +80,15 @@ pub struct UIInner { ui_state: UIState, log_colors: HashMap, cmdproc: Option, - cb_sink: Sender>, cmd_history: VecDeque, cmd_history_position: usize, cmd_history_max_size: usize, connection_dialog_state: Option, } -type Handle = Rc>; - -#[derive(Clone)] pub struct UI { - siv: Handle, - inner: Handle, + siv: CursiveRunnable, + inner: Arc>, } #[derive(Error, Debug)] @@ -112,11 +106,11 @@ impl UI { inner.cmdproc.as_ref().unwrap().clone() } - fn inner(s: &mut Cursive) -> std::cell::Ref<'_, UIInner> { - s.user_data::>().unwrap().borrow() + fn inner(s: &mut Cursive) -> MutexGuard<'_, UIInner> { + s.user_data::>>().unwrap().lock() } - fn inner_mut(s: &mut Cursive) -> std::cell::RefMut<'_, UIInner> { - s.user_data::>().unwrap().borrow_mut() + fn inner_mut(s: &mut Cursive) -> MutexGuard<'_, UIInner> { + s.user_data::>>().unwrap().lock() } fn setup_colors(siv: &mut CursiveRunnable, inner: &mut UIInner, settings: &Settings) { @@ -425,7 +419,7 @@ impl UI { "Detaching" => None, _ => None, }; - let mut cmdproc = Self::command_processor(s); + let cmdproc = Self::command_processor(s); if let Some(a) = action { if a { cmdproc.attach(); @@ -707,7 +701,7 @@ impl UI { //////////////////////////////////////////////////////////////////////////// // Public functions - pub fn new(node_log_scrollback: usize, settings: &Settings) -> Self { + pub fn new(node_log_scrollback: usize, settings: &Settings) -> (Self, UISender) { cursive_flexi_logger_view::resize(node_log_scrollback); // Instantiate the cursive runnable @@ -726,9 +720,9 @@ impl UI { let cb_sink = runnable.cb_sink().clone(); // Create the UI object - let this = Self { - siv: Rc::new(RefCell::new(runnable)), - inner: Rc::new(RefCell::new(UIInner { + let mut this = Self { + siv: runnable, + inner: Arc::new(Mutex::new(UIInner { ui_state: UIState::new(), log_colors: Default::default(), cmdproc: None, @@ -740,15 +734,13 @@ impl UI { cmd_history_position: 0, cmd_history_max_size: settings.interface.command_line.history_size, connection_dialog_state: None, - cb_sink, })), }; - let mut siv = this.siv.borrow_mut(); - let mut inner = this.inner.borrow_mut(); + let mut inner = this.inner.lock(); // Make the inner object accessible in callbacks easily - siv.set_user_data(this.inner.clone()); + this.siv.set_user_data(this.inner.clone()); // Create layouts @@ -831,87 +823,44 @@ impl UI { .child(TextView::new(version)), ); - siv.add_fullscreen_layer(mainlayout); + this.siv.add_fullscreen_layer(mainlayout); - UI::setup_colors(&mut siv, &mut inner, settings); - UI::setup_quit_handler(&mut siv); - siv.set_global_callback(cursive::event::Event::CtrlChar('k'), UI::clear_handler); + UI::setup_colors(&mut this.siv, &mut inner, settings); + UI::setup_quit_handler(&mut this.siv); + this.siv + .set_global_callback(cursive::event::Event::CtrlChar('k'), UI::clear_handler); drop(inner); - drop(siv); - this + let inner = this.inner.clone(); + (this, UISender { inner, cb_sink }) } pub fn cursive_flexi_logger(&self) -> Box { - let mut flv = - cursive_flexi_logger_view::cursive_flexi_logger(self.siv.borrow().cb_sink().clone()); - flv.set_colors(self.inner.borrow().log_colors.clone()); + let mut flv = cursive_flexi_logger_view::cursive_flexi_logger(self.siv.cb_sink().clone()); + flv.set_colors(self.inner.lock().log_colors.clone()); flv } pub fn set_command_processor(&mut self, cmdproc: CommandProcessor) { - let mut inner = self.inner.borrow_mut(); + let mut inner = self.inner.lock(); inner.cmdproc = Some(cmdproc); - let _ = inner.cb_sink.send(Box::new(UI::update_cb)); - } - pub fn set_attachment_state( - &mut self, - state: AttachmentState, - public_internet_ready: bool, - local_network_ready: bool, - ) { - let mut inner = self.inner.borrow_mut(); - inner.ui_state.attachment_state.set(state); - inner - .ui_state - .public_internet_ready - .set(public_internet_ready); - inner.ui_state.local_network_ready.set(local_network_ready); - - let _ = inner.cb_sink.send(Box::new(UI::update_cb)); - } - pub fn set_network_status( - &mut self, - started: bool, - bps_down: u64, - bps_up: u64, - peers: Vec, - ) { - let mut inner = self.inner.borrow_mut(); - inner.ui_state.network_started.set(started); - inner.ui_state.network_down_up.set(( - ((bps_down as f64) / 1000.0f64) as f32, - ((bps_up as f64) / 1000.0f64) as f32, - )); - inner.ui_state.peers_state.set(peers); - let _ = inner.cb_sink.send(Box::new(UI::update_cb)); - } - pub fn set_config(&mut self, config: VeilidConfigInner) { - let mut inner = self.inner.borrow_mut(); - - inner - .ui_state - .node_id - .set(config.network.routing_table.node_id.to_string()); - } - pub fn set_connection_state(&mut self, state: ConnectionState) { - let mut inner = self.inner.borrow_mut(); - inner.ui_state.connection_state.set(state); - let _ = inner.cb_sink.send(Box::new(UI::update_cb)); } - pub fn add_node_event(&self, event: String) { - let inner = self.inner.borrow(); - let color = *inner.log_colors.get(&Level::Info).unwrap(); - let mut starting_style: Style = color.into(); - for line in event.lines() { - let (spanned_string, end_style) = - cursive::utils::markup::ansi::parse_with_starting_style(starting_style, line); - cursive_flexi_logger_view::push_to_log(spanned_string); - starting_style = end_style; - } - let _ = inner.cb_sink.send(Box::new(UI::update_cb)); + // Note: Cursive is not re-entrant, can't borrow_mut self.siv again after this + pub async fn run_async(&mut self) { + self.siv.run_async().await; } + // pub fn run(&mut self) { + // self.siv.run(); + // } +} +#[derive(Clone)] +pub struct UISender { + inner: Arc>, + cb_sink: Sender>, +} + +impl UISender { pub fn display_string_dialog( &self, title: T, @@ -920,31 +869,84 @@ impl UI { ) { let title = title.to_string(); let text = text.to_string(); - let inner = self.inner.borrow(); - let _ = inner.cb_sink.send(Box::new(move |s| { + let _ = self.cb_sink.send(Box::new(move |s| { UI::display_string_dialog_cb(s, title, text, close_cb) })); } pub fn quit(&self) { - let inner = self.inner.borrow(); - let _ = inner.cb_sink.send(Box::new(|s| { + let _ = self.cb_sink.send(Box::new(|s| { s.quit(); })); } pub fn send_callback(&self, callback: UICallback) { - let inner = self.inner.borrow(); - let _ = inner.cb_sink.send(Box::new(move |s| callback(s))); + let _ = self.cb_sink.send(Box::new(move |s| callback(s))); + } + pub fn set_attachment_state( + &mut self, + state: String, + public_internet_ready: bool, + local_network_ready: bool, + ) { + { + let mut inner = self.inner.lock(); + inner.ui_state.attachment_state.set(state); + inner + .ui_state + .public_internet_ready + .set(public_internet_ready); + inner.ui_state.local_network_ready.set(local_network_ready); + } + + let _ = self.cb_sink.send(Box::new(UI::update_cb)); + } + pub fn set_network_status( + &mut self, + started: bool, + bps_down: u64, + bps_up: u64, + peers: Vec, + ) { + { + let mut inner = self.inner.lock(); + inner.ui_state.network_started.set(started); + inner.ui_state.network_down_up.set(( + ((bps_down as f64) / 1000.0f64) as f32, + ((bps_up as f64) / 1000.0f64) as f32, + )); + inner.ui_state.peers_state.set(peers); + } + let _ = self.cb_sink.send(Box::new(UI::update_cb)); + } + pub fn set_config(&mut self, config: &json::JsonValue) { + let mut inner = self.inner.lock(); + + inner + .ui_state + .node_id + .set(config["network"]["routing_table"]["node_id"].to_string()); + } + pub fn set_connection_state(&mut self, state: ConnectionState) { + { + let mut inner = self.inner.lock(); + inner.ui_state.connection_state.set(state); + } + let _ = self.cb_sink.send(Box::new(UI::update_cb)); } - // Note: Cursive is not re-entrant, can't borrow_mut self.siv again after this - pub async fn run_async(&mut self) { - let mut siv = self.siv.borrow_mut(); - siv.run_async().await; + pub fn add_node_event(&self, event: String) { + { + let inner = self.inner.lock(); + let color = *inner.log_colors.get(&Level::Info).unwrap(); + let mut starting_style: Style = color.into(); + for line in event.lines() { + let (spanned_string, end_style) = + cursive::utils::markup::ansi::parse_with_starting_style(starting_style, line); + cursive_flexi_logger_view::push_to_log(spanned_string); + starting_style = end_style; + } + } + let _ = self.cb_sink.send(Box::new(UI::update_cb)); } - // pub fn run(&mut self) { - // let mut siv = self.siv.borrow_mut(); - // siv.run(); - // } } diff --git a/veilid-core/src/intf/native/protected_store.rs b/veilid-core/src/intf/native/protected_store.rs index d8e46918..380a0c34 100644 --- a/veilid-core/src/intf/native/protected_store.rs +++ b/veilid-core/src/intf/native/protected_store.rs @@ -69,7 +69,7 @@ impl ProtectedStore { )); // Ensure permissions are correct - ensure_file_private_owner(&insecure_keyring_file)?; + ensure_file_private_owner(&insecure_keyring_file).map_err(|e| eyre!("{}", e))?; // Open the insecure keyring inner.keyring_manager = Some( diff --git a/veilid-core/src/lib.rs b/veilid-core/src/lib.rs index ff7a311d..f49417d9 100644 --- a/veilid-core/src/lib.rs +++ b/veilid-core/src/lib.rs @@ -41,15 +41,6 @@ pub use self::veilid_config::*; pub use self::veilid_layer_filter::*; pub use veilid_tools as tools; -use enumset::*; -use rkyv::{ - bytecheck, bytecheck::CheckBytes, de::deserializers::SharedDeserializeMap, with::Skip, - Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, -}; -type RkyvDefaultValidator<'t> = rkyv::validation::validators::DefaultValidator<'t>; -use schemars::{schema_for, JsonSchema}; -use serde::*; - pub mod veilid_capnp { include!(concat!(env!("OUT_DIR"), "/proto/veilid_capnp.rs")); } @@ -94,4 +85,20 @@ pub static DEFAULT_LOG_IGNORE_LIST: [&str; 21] = [ "attohttpc", ]; +use cfg_if::*; +use enumset::*; +use eyre::{bail, eyre, Report as EyreReport, Result as EyreResult, WrapErr}; +use parking_lot::*; +use rkyv::{ + bytecheck, bytecheck::CheckBytes, de::deserializers::SharedDeserializeMap, with::Skip, + Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, +}; +use tracing::*; use veilid_tools::*; +type RkyvDefaultValidator<'t> = rkyv::validation::validators::DefaultValidator<'t>; +use futures_util::stream::FuturesUnordered; +use owo_colors::OwoColorize; +use schemars::{schema_for, JsonSchema}; +use serde::*; +use stop_token::*; +use thiserror::Error as ThisError; diff --git a/veilid-core/src/network_manager/native/start_protocols.rs b/veilid-core/src/network_manager/native/start_protocols.rs index 36f285e4..76cdaf68 100644 --- a/veilid-core/src/network_manager/native/start_protocols.rs +++ b/veilid-core/src/network_manager/native/start_protocols.rs @@ -212,7 +212,8 @@ impl Network { } else { // If no address is specified, but the port is, use ipv4 and ipv6 unspecified // If the address is specified, only use the specified port and fail otherwise - let sockaddrs = listen_address_to_socket_addrs(&listen_address)?; + let sockaddrs = + listen_address_to_socket_addrs(&listen_address).map_err(|e| eyre!("{}", e))?; if sockaddrs.is_empty() { bail!("No valid listen address: {}", listen_address); } @@ -236,7 +237,8 @@ impl Network { } else { // If no address is specified, but the port is, use ipv4 and ipv6 unspecified // If the address is specified, only use the specified port and fail otherwise - let sockaddrs = listen_address_to_socket_addrs(&listen_address)?; + let sockaddrs = + listen_address_to_socket_addrs(&listen_address).map_err(|e| eyre!("{}", e))?; if sockaddrs.is_empty() { bail!("No valid listen address: {}", listen_address); } diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index ededea30..e63b5e19 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -167,20 +167,20 @@ impl ClientApi { if args.len() == 0 { apibail_generic!("no control request specified"); } - if args[0] == "shutdown" { + if args[0] == "Shutdown" { if args.len() != 1 { apibail_generic!("wrong number of arguments"); } self.shutdown(); Ok("".to_owned()) - } else if args[0] == "change_log_level" { + } else if args[0] == "ChangeLogLevel" { if args.len() != 3 { apibail_generic!("wrong number of arguments"); } let log_level: VeilidConfigLogLevel = deserialize_json(&args[2])?; self.change_log_level(args[1].clone(), log_level)?; Ok("".to_owned()) - } else if args[0] == "get_server_settings" { + } else if args[0] == "GetServerSettings" { if args.len() != 1 { apibail_generic!("wrong number of arguments"); } @@ -194,7 +194,7 @@ impl ClientApi { settings_json["core"]["protected_store"].remove("new_device_encryption_key_password"); let safe_settings_json = settings_json.to_string(); Ok(safe_settings_json) - } else if args[0] == "emit_schema" { + } else if args[0] == "EmitSchema" { if args.len() != 2 { apibail_generic!("wrong number of arguments"); } diff --git a/veilid-server/src/main.rs b/veilid-server/src/main.rs index b228a68e..e359855a 100644 --- a/veilid-server/src/main.rs +++ b/veilid-server/src/main.rs @@ -14,14 +14,10 @@ mod veilid_logs; #[cfg(windows)] mod windows; -use cfg_if::*; -#[allow(unused_imports)] -use color_eyre::eyre::{bail, ensure, eyre, Result as EyreResult, WrapErr}; use server::*; use std::collections::HashMap; use std::str::FromStr; use tools::*; -use tracing::*; use veilid_logs::*; #[instrument(err)] diff --git a/veilid-server/src/server.rs b/veilid-server/src/server.rs index 2a16f91e..e918fc2f 100644 --- a/veilid-server/src/server.rs +++ b/veilid-server/src/server.rs @@ -1,5 +1,6 @@ use crate::client_api; use crate::settings::*; +use crate::tools::*; use crate::veilid_logs::*; use flume::{unbounded, Receiver, Sender}; use futures_util::select; diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index ab7e36ef..d6ee6f3b 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -2,6 +2,7 @@ use directories::*; +use crate::tools::*; use serde_derive::*; use std::ffi::OsStr; use std::net::SocketAddr; diff --git a/veilid-server/src/tools.rs b/veilid-server/src/tools.rs index 06d87e1d..65f296d5 100644 --- a/veilid-server/src/tools.rs +++ b/veilid-server/src/tools.rs @@ -1,5 +1,8 @@ -use cfg_if::*; -use core::future::Future; +pub use cfg_if::*; +pub use color_eyre::eyre::{bail, ensure, eyre, Result as EyreResult, WrapErr}; +pub use core::future::Future; +pub use parking_lot::*; +pub use tracing::*; cfg_if! { if #[cfg(feature="rt-async-std")] { diff --git a/veilid-tools/src/lib.rs b/veilid-tools/src/lib.rs index 788e143d..253e6c8b 100644 --- a/veilid-tools/src/lib.rs +++ b/veilid-tools/src/lib.rs @@ -28,27 +28,6 @@ mod tools; #[cfg(target_arch = "wasm32")] mod wasm; -pub use cfg_if::*; -#[allow(unused_imports)] -pub use eyre::{bail, ensure, eyre, Report as EyreReport, Result as EyreResult, WrapErr}; -pub use futures_util::future::{select, Either}; -pub use futures_util::select; -pub use futures_util::stream::FuturesUnordered; -pub use futures_util::{AsyncRead, AsyncWrite}; -pub use log_thru::*; -pub use owo_colors::OwoColorize; -pub use parking_lot::*; -pub use split_url::*; -pub use static_assertions::*; -pub use stop_token::*; -pub use thiserror::Error as ThisError; -cfg_if! { - if #[cfg(feature = "tracing")] { - pub use tracing::*; - } else { - pub use log::*; - } -} pub type PinBox = Pin>; pub type PinBoxFuture = PinBox + 'static>; pub type PinBoxFutureLifetime<'a, T> = PinBox + 'a>; @@ -120,6 +99,7 @@ pub use eventual_value_clone::*; pub use interval::*; pub use ip_addr_port::*; pub use ip_extra::*; +pub use log_thru::*; pub use must_join_handle::*; pub use must_join_single_future::*; pub use mutable_future::*; @@ -128,17 +108,32 @@ pub use random::*; pub use single_shot_eventual::*; pub use sleep::*; pub use spawn::*; +pub use split_url::*; pub use tick_task::*; pub use timeout::*; pub use timeout_or::*; pub use timestamp::*; pub use tools::*; + #[cfg(target_arch = "wasm32")] pub use wasm::*; // Tests must be public for wasm-pack tests pub mod tests; +cfg_if! { + if #[cfg(feature = "tracing")] { + use tracing::*; + } else { + use log::*; + } +} +use cfg_if::*; +use futures_util::{AsyncRead, AsyncWrite}; +use parking_lot::*; +use stop_token::*; +use thiserror::Error as ThisError; + // For iOS tests #[no_mangle] pub extern "C" fn main_rs() {} diff --git a/veilid-tools/src/network_result.rs b/veilid-tools/src/network_result.rs index 0ddb3973..437000bc 100644 --- a/veilid-tools/src/network_result.rs +++ b/veilid-tools/src/network_result.rs @@ -337,19 +337,13 @@ macro_rules! network_result_value_or_log { ($r: expr => $f:tt) => { match $r { NetworkResult::Timeout => { - log_network_result!( - "{} at {}@{}:{}", - "Timeout".cyan(), - file!(), - line!(), - column!() - ); + log_network_result!("{} at {}@{}:{}", "Timeout", file!(), line!(), column!()); $f } NetworkResult::ServiceUnavailable => { log_network_result!( "{} at {}@{}:{}", - "ServiceUnavailable".cyan(), + "ServiceUnavailable", file!(), line!(), column!() @@ -359,7 +353,7 @@ macro_rules! network_result_value_or_log { NetworkResult::NoConnection(e) => { log_network_result!( "{}({}) at {}@{}:{}", - "No connection".cyan(), + "No connection", e.to_string(), file!(), line!(), @@ -370,7 +364,7 @@ macro_rules! network_result_value_or_log { NetworkResult::AlreadyExists(e) => { log_network_result!( "{}({}) at {}@{}:{}", - "Already exists".cyan(), + "Already exists", e.to_string(), file!(), line!(), @@ -381,7 +375,7 @@ macro_rules! network_result_value_or_log { NetworkResult::InvalidMessage(s) => { log_network_result!( "{}({}) at {}@{}:{}", - "Invalid message".cyan(), + "Invalid message", s, file!(), line!(), diff --git a/veilid-tools/src/tools.rs b/veilid-tools/src/tools.rs index 4aba1f00..92ce1444 100644 --- a/veilid-tools/src/tools.rs +++ b/veilid-tools/src/tools.rs @@ -97,11 +97,13 @@ cfg_if! { ////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub fn split_port(name: &str) -> EyreResult<(String, Option)> { +pub fn split_port(name: &str) -> Result<(String, Option), String> { if let Some(split) = name.rfind(':') { let hoststr = &name[0..split]; let portstr = &name[split + 1..]; - let port: u16 = portstr.parse::().wrap_err("invalid port")?; + let port: u16 = portstr + .parse::() + .map_err(|e| format!("invalid port: {}", e))?; Ok((hoststr.to_string(), Some(port))) } else { @@ -130,8 +132,8 @@ pub fn ms_to_us(ms: u32) -> u64 { (ms as u64) * 1000u64 } -pub fn us_to_ms(us: u64) -> EyreResult { - u32::try_from(us / 1000u64).wrap_err("could not convert microseconds") +pub fn us_to_ms(us: u64) -> Result { + u32::try_from(us / 1000u64).map_err(|e| format!("could not convert microseconds: {}", e)) } // Calculate retry attempt with logarhythmic falloff @@ -224,7 +226,7 @@ pub fn compatible_unspecified_socket_addr(socket_addr: &SocketAddr) -> SocketAdd } } -pub fn listen_address_to_socket_addrs(listen_address: &str) -> EyreResult> { +pub fn listen_address_to_socket_addrs(listen_address: &str) -> Result, String> { // If no address is specified, but the port is, use ipv4 and ipv6 unspecified // If the address is specified, only use the specified port and fail otherwise let ip_addrs = vec![ @@ -235,7 +237,7 @@ pub fn listen_address_to_socket_addrs(listen_address: &str) -> EyreResult() - .wrap_err("Invalid port format in udp listen address")?; + .map_err(|e| format!("Invalid port format in udp listen address: {}", e))?; ip_addrs.iter().map(|a| SocketAddr::new(*a, port)).collect() } else if let Ok(port) = listen_address.parse::() { ip_addrs.iter().map(|a| SocketAddr::new(*a, port)).collect() @@ -243,11 +245,11 @@ pub fn listen_address_to_socket_addrs(listen_address: &str) -> EyreResult>(path: P) -> EyreResult<()> + pub fn ensure_file_private_owner>(path: P) -> Result<(), String> { let path = path.as_ref(); if !path.exists() { @@ -286,13 +288,13 @@ cfg_if::cfg_if! { let uid = Uid::effective(); let gid = Gid::effective(); - let meta = std::fs::metadata(path).wrap_err("unable to get metadata for path")?; + let meta = std::fs::metadata(path).map_err(|e| format!("unable to get metadata for path: {}", e))?; if meta.mode() != 0o600 { - std::fs::set_permissions(path,std::fs::Permissions::from_mode(0o600)).wrap_err("unable to set correct permissions on path")?; + std::fs::set_permissions(path,std::fs::Permissions::from_mode(0o600)).map_err(|e| format!("unable to set correct permissions on path: {}", e))?; } if meta.uid() != uid.as_raw() || meta.gid() != gid.as_raw() { - bail!("path has incorrect owner/group"); + return Err("path has incorrect owner/group".to_owned()); } Ok(()) } @@ -300,7 +302,7 @@ cfg_if::cfg_if! { //use std::os::windows::fs::MetadataExt; //use windows_permissions::*; - pub fn ensure_file_private_owner>(path: P) -> EyreResult<()> + pub fn ensure_file_private_owner>(path: P) -> Result<(), String> { let path = path.as_ref(); if !path.exists() { From 532bcf2e2a61d0e1fceefa8b7856866c4d332eba Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 9 Jun 2023 19:08:49 -0400 Subject: [PATCH 10/23] json api cli working --- veilid-cli/src/client_api_connection.rs | 164 +++-------- veilid-cli/src/command_processor.rs | 16 +- .../src/veilid_api/types/veilid_log.rs | 26 +- veilid-core/src/veilid_config.rs | 29 ++ veilid-server/src/client_api.rs | 276 ++++++++++-------- 5 files changed, 254 insertions(+), 257 deletions(-) diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index b7e12670..e790dcd9 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -1,6 +1,5 @@ use crate::command_processor::*; use crate::tools::*; -use core::str::FromStr; use futures::stream::FuturesUnordered; use futures::StreamExt; use std::net::SocketAddr; @@ -23,7 +22,6 @@ struct ClientApiConnectionInner { comproc: CommandProcessor, connect_addr: Option, request_sender: Option>, - server_settings: Option, disconnector: Option, disconnect_requested: bool, reply_channels: HashMap>, @@ -42,7 +40,6 @@ impl ClientApiConnection { comproc, connect_addr: None, request_sender: None, - server_settings: None, disconnector: None, disconnect_requested: false, reply_channels: HashMap::new(), @@ -56,28 +53,19 @@ impl ClientApiConnection { inner.reply_channels.clear(); } - // async fn process_veilid_state<'a>( - // &'a mut self, - // veilid_state: VeilidState, - // ) -> Result<(), String> { - // let mut inner = self.inner.borrow_mut(); - // inner.comproc.update_attachment(veilid_state.attachment); - // inner.comproc.update_network_status(veilid_state.network); - // inner.comproc.update_config(veilid_state.config); - // Ok(()) - // } + async fn process_veilid_state<'a>(&self, state: &json::JsonValue) { + let comproc = self.inner.lock().comproc.clone(); + comproc.update_attachment(&state["attachment"]); + comproc.update_network_status(&state["network"]); + comproc.update_config(&state["config"]); + } async fn process_response(&self, response: json::JsonValue) { // find the operation id and send the response to the channel for it - let Some(id_str) = response["id"].as_str() else { - error!("missing id: {}", response); - return; - }; - let Ok(id) = u32::from_str(id_str) else { + let Some(id) = response["id"].as_u32() else { error!("invalid id: {}", response); return; }; - let reply_channel = { let mut inner = self.inner.lock(); inner.reply_channels.remove(&id) @@ -92,7 +80,7 @@ impl ClientApiConnection { } } - async fn process_update(&self, update: json::JsonValue) { + async fn process_veilid_update(&self, update: json::JsonValue) { let comproc = self.inner.lock().comproc.clone(); let Some(kind) = update["kind"].as_str() else { comproc.log_message(format!("missing update kind: {}", update)); @@ -100,29 +88,29 @@ impl ClientApiConnection { }; match kind { "Log" => { - comproc.update_log(update); + comproc.update_log(&update); } "AppMessage" => { - comproc.update_app_message(update); + comproc.update_app_message(&update); } "AppCall" => { - comproc.update_app_call(update); + comproc.update_app_call(&update); } "Attachment" => { - comproc.update_attachment(update); + comproc.update_attachment(&update); } "Network" => { - comproc.update_network_status(update); + comproc.update_network_status(&update); } "Config" => { - comproc.update_config(update); + comproc.update_config(&update); } "RouteChange" => { - comproc.update_route(update); + comproc.update_route(&update); } "Shutdown" => comproc.update_shutdown(), "ValueChange" => { - comproc.update_value_change(update); + comproc.update_value_change(&update); } _ => { comproc.log_message(format!("unknown update kind: {}", update)); @@ -130,97 +118,6 @@ impl ClientApiConnection { } } - // async fn spawn_rpc_system( - // &self, - // connect_addr: SocketAddr, - // mut rpc_system: RpcSystem, - // ) -> Result<(), String> { - // let mut request; - // { - // let mut inner = self.inner.borrow_mut(); - - // // Get the bootstrap server connection object - // inner.server = Some(Rc::new(RefCell::new( - // rpc_system.bootstrap(rpc_twoparty_capnp::Side::Server), - // ))); - - // // Store our disconnector future for later (must happen after bootstrap, contrary to documentation) - // inner.disconnector = Some(rpc_system.get_disconnector()); - - // // Get a client object to pass to the server for status update callbacks - // let client = capnp_rpc::new_client(VeilidClientImpl::new(inner.comproc.clone())); - - // // Register our client and get a registration object back - // request = inner - // .server - // .as_ref() - // .unwrap() - // .borrow_mut() - // .register_request(); - // request.get().set_veilid_client(client); - - // inner - // .comproc - // .set_connection_state(ConnectionState::Connected( - // connect_addr, - // std::time::SystemTime::now(), - // )); - // } - - // let rpc_jh = spawn_local(rpc_system); - - // let reg_res: Result = (async { - // // Send the request and get the state object and the registration object - // let response = request - // .send() - // .promise - // .await - // .map_err(|e| format!("failed to send register request: {}", e))?; - // let response = response - // .get() - // .map_err(|e| format!("failed to get register response: {}", e))?; - - // // Get the registration object, which drops our connection when it is dropped - // let registration = response - // .get_registration() - // .map_err(|e| format!("failed to get registration object: {}", e))?; - - // // Get the initial veilid state - // let veilid_state = response - // .get_state() - // .map_err(|e| format!("failed to get initial veilid state: {}", e))?; - - // // Set up our state for the first time - // let veilid_state: VeilidState = deserialize_json(veilid_state) - // .map_err(|e| format!("failed to get deserialize veilid state: {}", e))?; - // self.process_veilid_state(veilid_state).await?; - - // // Save server settings - // let server_settings = response - // .get_settings() - // .map_err(|e| format!("failed to get initial veilid server settings: {}", e))? - // .to_owned(); - // self.inner.borrow_mut().server_settings = Some(server_settings.clone()); - - // // Don't drop the registration, doing so will remove the client - // // object mapping from the server which we need for the update backchannel - // Ok(registration) - // }) - // .await; - - // let _registration = match reg_res { - // Ok(v) => v, - // Err(e) => { - // rpc_jh.abort().await; - // return Err(e); - // } - // }; - - // // Wait until rpc system completion or disconnect was requested - // let res = rpc_jh.await; - // res.map_err(|e| format!("client RPC system error: {}", e)) - // } - async fn handle_connection(&self, connect_addr: SocketAddr) -> Result<(), String> { trace!("ClientApiConnection::handle_connection"); @@ -251,6 +148,7 @@ impl ClientApiConnection { // Requests to send let (requests_tx, requests_rx) = flume::unbounded(); + // Create disconnection mechanism let stop_token = { let stop_source = StopSource::new(); let token = stop_source.token(); @@ -267,16 +165,19 @@ impl ClientApiConnection { // Process lines let this = self.clone(); let recv_messages_future = async move { - let mut line = String::new(); - while let Ok(size) = reader.read_line(&mut line).await { + let mut linebuf = String::new(); + while let Ok(size) = reader.read_line(&mut linebuf).await { // Exit on EOF if size == 0 { // Disconnected break; } + let line = linebuf.trim().to_owned(); + linebuf.clear(); + // Unmarshal json - let j = match json::parse(line.trim()) { + let j = match json::parse(&line) { Ok(v) => v, Err(e) => { error!("failed to parse server response: {}", e); @@ -285,7 +186,7 @@ impl ClientApiConnection { }; if j["type"] == "Update" { - this.process_update(j).await; + this.process_veilid_update(j).await; } else if j["type"] == "Response" { this.process_response(j).await; } @@ -306,13 +207,28 @@ impl ClientApiConnection { }; unord.push(system_boxed(send_requests_future)); + // Request initial server state + let capi = self.clone(); + spawn_detached_local(async move { + let mut req = json::JsonValue::new_object(); + req["op"] = "GetState".into(); + let Some(resp) = capi.perform_request(req).await else { + error!("failed to get state"); + return; + }; + if resp.has_key("error") { + error!("failed to get state: {}", resp["error"]); + return; + } + capi.process_veilid_state(&resp["value"]).await; + }); + // Send and receive until we're done or a stop is requested while let Ok(Some(())) = unord.next().timeout_at(stop_token.clone()).await {} // // Drop the server and disconnector too (if we still have it) let mut inner = self.inner.lock(); let disconnect_requested = inner.disconnect_requested; - inner.server_settings = None; inner.request_sender = None; inner.disconnector = None; inner.disconnect_requested = false; diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index 6cbc6331..636561d1 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -387,7 +387,7 @@ reply - reply to an AppCall not handled directly by the server self.inner().ui_sender.add_node_event(message); } - pub fn update_attachment(&self, attachment: json::JsonValue) { + pub fn update_attachment(&self, attachment: &json::JsonValue) { self.inner_mut().ui_sender.set_attachment_state( attachment["state"].as_str().unwrap_or_default().to_owned(), attachment["public_internet_ready"] @@ -399,7 +399,7 @@ reply - reply to an AppCall not handled directly by the server ); } - pub fn update_network_status(&self, network: json::JsonValue) { + pub fn update_network_status(&self, network: &json::JsonValue) { self.inner_mut().ui_sender.set_network_status( network["started"].as_bool().unwrap_or_default(), json_str_u64(&network["bps_down"]), @@ -410,10 +410,10 @@ reply - reply to an AppCall not handled directly by the server .collect::>(), ); } - pub fn update_config(&self, config: json::JsonValue) { + pub fn update_config(&self, config: &json::JsonValue) { self.inner_mut().ui_sender.set_config(&config["config"]) } - pub fn update_route(&self, route: json::JsonValue) { + pub fn update_route(&self, route: &json::JsonValue) { let mut out = String::new(); if route["dead_routes"].len() != 0 { out.push_str(&format!("Dead routes: {:?}", route["dead_routes"])); @@ -431,12 +431,12 @@ reply - reply to an AppCall not handled directly by the server self.inner().ui_sender.add_node_event(out); } } - pub fn update_value_change(&self, value_change: json::JsonValue) { + pub fn update_value_change(&self, value_change: &json::JsonValue) { let out = format!("Value change: {:?}", value_change.as_str().unwrap_or("???")); self.inner().ui_sender.add_node_event(out); } - pub fn update_log(&self, log: json::JsonValue) { + pub fn update_log(&self, log: &json::JsonValue) { self.inner().ui_sender.add_node_event(format!( "{}: {}{}", log["log_level"].as_str().unwrap_or("???"), @@ -449,7 +449,7 @@ reply - reply to an AppCall not handled directly by the server )); } - pub fn update_app_message(&self, msg: json::JsonValue) { + pub fn update_app_message(&self, msg: &json::JsonValue) { let message = json_str_vec_u8(&msg["message"]); // check is message body is ascii printable @@ -471,7 +471,7 @@ reply - reply to an AppCall not handled directly by the server .add_node_event(format!("AppMessage ({:?}): {}", msg["sender"], strmsg)); } - pub fn update_app_call(&self, call: json::JsonValue) { + pub fn update_app_call(&self, call: &json::JsonValue) { let message = json_str_vec_u8(&call["message"]); // check is message body is ascii printable diff --git a/veilid-core/src/veilid_api/types/veilid_log.rs b/veilid-core/src/veilid_api/types/veilid_log.rs index 66852e14..f377115c 100644 --- a/veilid-core/src/veilid_api/types/veilid_log.rs +++ b/veilid-core/src/veilid_api/types/veilid_log.rs @@ -64,19 +64,33 @@ impl VeilidLogLevel { } } +impl FromStr for VeilidLogLevel { + type Err = VeilidAPIError; + fn from_str(s: &str) -> Result { + Ok(match s { + "Error" => Self::Error, + "Warn" => Self::Warn, + "Info" => Self::Info, + "Debug" => Self::Debug, + "Trace" => Self::Trace, + _ => { + apibail_invalid_argument!("Can't convert str", "s", s); + } + }) + } +} impl fmt::Display for VeilidLogLevel { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { let text = match self { - Self::Error => "ERROR", - Self::Warn => "WARN", - Self::Info => "INFO", - Self::Debug => "DEBUG", - Self::Trace => "TRACE", + Self::Error => "Error", + Self::Warn => "Warn", + Self::Info => "Info", + Self::Debug => "Debug", + Self::Trace => "Trace", }; write!(f, "{}", text) } } - /// A VeilidCore log message with optional backtrace #[derive( Debug, diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index 8b3c3e7e..98badf13 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -545,6 +545,35 @@ impl Default for VeilidConfigLogLevel { Self::Off } } +impl FromStr for VeilidConfigLogLevel { + type Err = VeilidAPIError; + fn from_str(s: &str) -> Result { + Ok(match s { + "Off" => Self::Off, + "Error" => Self::Error, + "Warn" => Self::Warn, + "Info" => Self::Info, + "Debug" => Self::Debug, + "Trace" => Self::Trace, + _ => { + apibail_invalid_argument!("Can't convert str", "s", s); + } + }) + } +} +impl fmt::Display for VeilidConfigLogLevel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + let text = match self { + Self::Off => "Off", + Self::Error => "Error", + Self::Warn => "Warn", + Self::Info => "Info", + Self::Debug => "Debug", + Self::Trace => "Trace", + }; + write!(f, "{}", text) + } +} #[derive( Default, diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index e63b5e19..821d06ef 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -3,14 +3,14 @@ use crate::tools::*; use crate::veilid_logs::VeilidLogs; use cfg_if::*; use futures_util::{future::try_join_all, stream::FuturesUnordered, StreamExt}; - use parking_lot::Mutex; use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; -use stop_token::future::FutureExt; +use stop_token::future::FutureExt as _; use stop_token::*; use tracing::*; +use veilid_core::json_api::JsonRequestProcessor; use veilid_core::tools::*; use veilid_core::*; use wg::AsyncWaitGroup; @@ -24,34 +24,18 @@ cfg_if! { use tokio::io::AsyncWriteExt; } } -// struct VeilidServerImpl { -// veilid_api: veilid_core::VeilidAPI, -// veilid_logs: VeilidLogs, -// settings: Settings, -// next_id: u64, -// } - -// impl VeilidServerImpl { -// #[instrument(level = "trace", skip_all)] -// pub fn new( -// veilid_api: veilid_core::VeilidAPI, -// veilid_logs: VeilidLogs, -// settings: Settings, -// ) -> Self { -// Self { -// next_id: 0, -// veilid_api, -// veilid_logs, -// settings, -// } -// } - -// } // --- Client API Server-Side --------------------------------- type ClientApiAllFuturesJoinHandle = MustJoinHandle>>; +struct RequestLine { + // Request to process + line: String, + // Where to send the response + responses_tx: flume::Sender, +} + struct ClientApiInner { veilid_api: veilid_core::VeilidAPI, veilid_logs: VeilidLogs, @@ -177,7 +161,7 @@ impl ClientApi { if args.len() != 3 { apibail_generic!("wrong number of arguments"); } - let log_level: VeilidConfigLogLevel = deserialize_json(&args[2])?; + let log_level = VeilidConfigLogLevel::from_str(&args[2])?; self.change_log_level(args[1].clone(), log_level)?; Ok("".to_owned()) } else if args[0] == "GetServerSettings" { @@ -212,6 +196,107 @@ impl ClientApi { } } + async fn process_request_line( + self, + jrp: JsonRequestProcessor, + request_line: RequestLine, + ) -> VeilidAPIResult> { + let line = request_line.line; + let responses_tx = request_line.responses_tx; + + // Unmarshal NDJSON - newline => json + // (trim all whitespace around input lines just to make things more permissive for API users) + let request: json_api::Request = deserialize_json(&line)?; + + // See if this is a control message or a veilid-core message + let response = if let json_api::RequestOp::Control { args } = request.op { + // Process control messages + json_api::Response { + id: request.id, + op: json_api::ResponseOp::Control { + result: json_api::to_json_api_result(self.process_control(args).await), + }, + } + } else { + // Process with ndjson api + jrp.clone().process_request(request).await + }; + + // Marshal json + newline => NDJSON + let response_string = serialize_json(json_api::RecvMessage::Response(response)) + "\n"; + if let Err(e) = responses_tx.send_async(response_string).await { + warn!("response not sent: {}", e) + } + VeilidAPIResult::Ok(None) + } + + async fn next_request_line( + requests_rx: flume::Receiver>, + ) -> VeilidAPIResult> { + Ok(requests_rx.recv_async().await.ok().flatten()) + } + + async fn receive_requests( + self, + conn_tuple: (SocketAddr, SocketAddr), + mut reader: R, + requests_tx: flume::Sender>, + responses_tx: flume::Sender, + ) -> VeilidAPIResult> { + // responses_tx becomes owned by recv_requests_future + // Start sending updates + self.inner + .lock() + .update_channels + .insert(conn_tuple, responses_tx.clone()); + + let mut linebuf = String::new(); + while let Ok(size) = reader.read_line(&mut linebuf).await { + // Eof? + if size == 0 { + break; + } + + // Put the processing in the async queue + let line = linebuf.trim().to_owned(); + linebuf.clear(); + + // Ignore newlines + if line.len() == 0 { + continue; + } + + // Enqueue the line for processing in parallel + let request_line = RequestLine { + line, + responses_tx: responses_tx.clone(), + }; + if let Err(e) = requests_tx.send_async(Some(request_line)).await { + error!("failed to enqueue request: {}", e); + break; + } + } + + // Stop sending updates + // Will cause send_responses_future to stop because we drop the responses_tx + self.inner.lock().update_channels.remove(&conn_tuple); + + VeilidAPIResult::Ok(None) + } + + async fn send_responses( + self, + responses_rx: flume::Receiver, + mut writer: W, + ) -> VeilidAPIResult> { + while let Ok(resp) = responses_rx.recv_async().await { + if let Err(e) = writer.write_all(resp.as_bytes()).await { + error!("failed to write response: {}", e) + } + } + VeilidAPIResult::Ok(None) + } + pub async fn handle_connection(self, stream: TcpStream, awg: AsyncWaitGroup) { // Get address of peer let peer_addr = match stream.peer_addr() { @@ -246,10 +331,10 @@ impl ClientApi { if #[cfg(feature="rt-async-std")] { use futures_util::AsyncReadExt; let (reader, mut writer) = stream.split(); - let mut reader = BufReader::new(reader); + let reader = BufReader::new(reader); } else if #[cfg(feature="rt-tokio")] { - let (reader, mut writer) = stream.into_split(); - let mut reader = BufReader::new(reader); + let (reader, writer) = stream.into_split(); + let reader = BufReader::new(reader); } } @@ -259,105 +344,58 @@ impl ClientApi { // Futures to process unordered let mut unord = FuturesUnordered::new(); - let (more_futures_tx, more_futures_rx) = flume::unbounded(); - // Output to serialize + // Requests and responses are done serially to the socket + // but the requests are processed in parallel by the FuturesUnordered + let (requests_tx, requests_rx) = flume::unbounded(); let (responses_tx, responses_rx) = flume::unbounded(); - // Request receive processor - let this = self.clone(); - let recv_requests_future = async move { - // Start sending updates - this.inner - .lock() - .update_channels - .insert(conn_tuple, responses_tx.clone()); - - let mut line = String::new(); - while let Ok(size) = reader.read_line(&mut line).await { - // Eof? - if size == 0 { - break; - } - - // Put the processing in the async queue - let jrp = jrp.clone(); - let line = line.trim().to_owned(); - // Ignore newlines - if line.len() == 0 { - continue; - } - let responses_tx = responses_tx.clone(); - let this2 = this.clone(); - let process_request = async move { - // Unmarshal NDJSON - newline => json - // (trim all whitespace around input lines just to make things more permissive for API users) - let request: json_api::Request = deserialize_json(&line)?; - - // See if this is a control message or a veilid-core message - let response = if let json_api::RequestOp::Control { args } = request.op { - // Process control messages - json_api::Response { - id: request.id, - op: json_api::ResponseOp::Control { - result: json_api::to_json_api_result( - this2.process_control(args).await, - ), - }, - } - } else { - // Process with ndjson api - jrp.clone().process_request(request).await - }; - - // Marshal json + newline => NDJSON - let response_string = - serialize_json(json_api::RecvMessage::Response(response)) + "\n"; - if let Err(e) = responses_tx.send_async(response_string).await { - warn!("response not sent: {}", e) - } - VeilidAPIResult::Ok(()) - }; - if let Err(e) = more_futures_tx - .send_async(system_boxed(process_request)) - .await - { - warn!("request dropped: {}", e) - } - } - - // Stop sending updates - // Will cause send_responses_future to stop because we drop the responses_tx - this.inner.lock().update_channels.remove(&conn_tuple); - - VeilidAPIResult::Ok(()) - }; - unord.push(system_boxed(recv_requests_future)); + // Request receive processor future + // Receives from socket and enqueues RequestLines + // Completes when the connection is closed or there is a failure + unord.push(system_boxed(self.clone().receive_requests( + conn_tuple, + reader, + requests_tx, + responses_tx, + ))); // Response send processor - let send_responses_future = async move { - while let Ok(resp) = responses_rx.recv_async().await { - if let Err(e) = writer.write_all(resp.as_bytes()).await { - error!("failed to write response: {}", e) - } - } - VeilidAPIResult::Ok(()) - }; - unord.push(system_boxed(send_responses_future)); + // Sends finished response strings out the socket + // Completes when the responses channel is closed + unord.push(system_boxed( + self.clone().send_responses(responses_rx, writer), + )); + + // Add future to process first request + unord.push(system_boxed(Self::next_request_line(requests_rx.clone()))); // Send and receive until we're done or a stop is requested while let Ok(Some(r)) = unord.next().timeout_at(stop_token.clone()).await { - match r { - Ok(()) => {} - Err(e) => { - warn!("JSON API Failure: {}", e); + // See if we got some work to do + let request_line = match r { + Ok(Some(request_line)) => { + // Add future to process next request + unord.push(system_boxed(Self::next_request_line(requests_rx.clone()))); + + // Socket receive future returned something to process + request_line } - } - // Add more futures if we had one that completed - // Allows processing requests in an async fashion - for fut in more_futures_rx.drain() { - unord.push(fut); - } + Ok(None) => { + // Non-request future finished + continue; + } + Err(e) => { + // Connection processing failure, abort + error!("Connection processing failure: {}", e); + break; + } + }; + + // Enqueue unordered future to process request line in parallel + unord.push(system_boxed( + self.clone().process_request_line(jrp.clone(), request_line), + )); } debug!( From 88466db03fd69aa59128dd916298ceae2c4179f7 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 9 Jun 2023 22:42:03 -0400 Subject: [PATCH 11/23] better statusq failure recording --- veilid-core/src/routing_table/bucket_entry.rs | 2 +- veilid-core/src/rpc_processor/rpc_status.rs | 31 +++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 46206d0c..f133f4e5 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -603,7 +603,7 @@ impl BucketEntryInner { } } pub(super) fn check_dead(&self, cur_ts: Timestamp) -> bool { - // If we have failured to send NEVER_REACHED_PING_COUNT times in a row, the node is dead + // If we have failed to send NEVER_REACHED_PING_COUNT times in a row, the node is dead if self.peer_stats.rpc_stats.failed_to_send >= NEVER_REACHED_PING_COUNT { return true; } diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index f45ea125..179508ce 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -30,9 +30,19 @@ impl RPCProcessor { let routing_domain = match target.best_routing_domain() { Some(rd) => rd, None => { + // Because this exits before calling 'question()', + // a failure to find a routing domain constitutes a send failure + let send_ts = get_aligned_timestamp(); + self.record_send_failure( + RPCKind::Question, + send_ts, + target.clone(), + None, + None, + ); return Ok(NetworkResult::no_connection_other( "no routing domain for target", - )) + )); } }; (Some(target.clone()), routing_domain) @@ -45,9 +55,26 @@ impl RPCProcessor { let routing_domain = match relay.best_routing_domain() { Some(rd) => rd, None => { + // Because this exits before calling 'question()', + // a failure to find a routing domain constitutes a send failure for both the target and its relay + let send_ts = get_aligned_timestamp(); + self.record_send_failure( + RPCKind::Question, + send_ts, + relay.clone(), + None, + None, + ); + self.record_send_failure( + RPCKind::Question, + send_ts, + target.clone(), + None, + None, + ); return Ok(NetworkResult::no_connection_other( "no routing domain for peer", - )) + )); } }; (Some(target.clone()), routing_domain) From b6c39ef042b16691d2769571e4824e3fcd4e7641 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Jun 2023 11:02:48 -0400 Subject: [PATCH 12/23] fix firewalling/fragmentation issue --- veilid-core/src/network_manager/mod.rs | 11 ++++++++++- veilid-core/src/rpc_processor/mod.rs | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 9456c625..d11f9b73 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1077,8 +1077,17 @@ impl NetworkManager { // Dial info filter comes from the target node ref let dial_info_filter = target_node_ref.dial_info_filter(); - let sequencing = target_node_ref.sequencing(); + let mut sequencing = target_node_ref.sequencing(); + + // If the node has had lost questions or failures to send, prefer sequencing + // to improve reliability. The node may be experiencing UDP fragmentation drops + // or other firewalling issues and may perform better with TCP. + let unreliable = target_node_ref.peer_stats().rpc_stats.failed_to_send > 0 || target_node_ref.peer_stats().rpc_stats.recent_lost_answers > 0; + if unreliable && sequencing < Sequencing::PreferOrdered { + sequencing = Sequencing::PreferOrdered; + } + // Get the best contact method with these parameters from the routing domain let cm = routing_table.get_contact_method( routing_domain, &peer_a, diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index ba24517e..daf8ff80 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -844,6 +844,10 @@ impl RPCProcessor { // Record for node if this was not sent via a route if safety_route.is_none() && remote_private_route.is_none() { node_ref.stats_failed_to_send(send_ts, wants_answer); + + // Also clear the last_connections for the entry so we make a new connection next time + node_ref.clear_last_connections(); + return; } @@ -872,6 +876,10 @@ impl RPCProcessor { // Record for node if this was not sent via a route if safety_route.is_none() && remote_private_route.is_none() { node_ref.stats_question_lost(); + + // Also clear the last_connections for the entry so we make a new connection next time + node_ref.clear_last_connections(); + return; } // Get route spec store From cd04a8a74c8b849d28e7fbe1ef2f55007c8b1452 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sun, 11 Jun 2023 21:41:13 -0400 Subject: [PATCH 13/23] python work --- veilid-flutter/lib/veilid_config.dart | 3 +- veilid-flutter/lib/veilid_state.dart | 7 +- veilid-python/.gitignore | 160 + veilid-python/README.md | 20 + veilid-python/poetry.lock | 959 ++++++ veilid-python/pyproject.toml | 20 + veilid-python/schema/RecvMessage.json | 3924 +++++++++++++++++++++++ veilid-python/schema/Request.json | 1585 +++++++++ veilid-python/tests/__init__.py | 0 veilid-python/update_schema.sh | 17 + veilid-python/veilid_python/__init__.py | 1 + veilid-python/veilid_python/api.py | 11 + veilid-python/veilid_python/config.py | 254 ++ veilid-python/veilid_python/error.py | 133 + veilid-python/veilid_python/request.py | 4 + veilid-python/veilid_python/state.py | 205 ++ veilid-python/veilid_python/types.py | 10 + 17 files changed, 7309 insertions(+), 4 deletions(-) create mode 100644 veilid-python/.gitignore create mode 100644 veilid-python/README.md create mode 100644 veilid-python/poetry.lock create mode 100644 veilid-python/pyproject.toml create mode 100644 veilid-python/schema/RecvMessage.json create mode 100644 veilid-python/schema/Request.json create mode 100644 veilid-python/tests/__init__.py create mode 100755 veilid-python/update_schema.sh create mode 100644 veilid-python/veilid_python/__init__.py create mode 100644 veilid-python/veilid_python/api.py create mode 100644 veilid-python/veilid_python/config.py create mode 100644 veilid-python/veilid_python/error.py create mode 100644 veilid-python/veilid_python/request.py create mode 100644 veilid-python/veilid_python/state.py create mode 100644 veilid-python/veilid_python/types.py diff --git a/veilid-flutter/lib/veilid_config.dart b/veilid-flutter/lib/veilid_config.dart index 8633d3e2..86a32406 100644 --- a/veilid-flutter/lib/veilid_config.dart +++ b/veilid-flutter/lib/veilid_config.dart @@ -856,7 +856,8 @@ class VeilidConfigProtectedStore { directory = json['directory'], delete = json['delete'], deviceEncryptionKeyPassword = json['device_encryption_key_password'], - newDeviceEncryptionKeyPassword = json['new_device_encryption_key_password']; + newDeviceEncryptionKeyPassword = + json['new_device_encryption_key_password']; } //////////// diff --git a/veilid-flutter/lib/veilid_state.dart b/veilid-flutter/lib/veilid_state.dart index fea12cdf..40c263c5 100644 --- a/veilid-flutter/lib/veilid_state.dart +++ b/veilid-flutter/lib/veilid_state.dart @@ -511,16 +511,17 @@ class VeilidStateNetwork { /// VeilidStateConfig class VeilidStateConfig { - final Map config; + final VeilidConfig config; VeilidStateConfig({ required this.config, }); - VeilidStateConfig.fromJson(dynamic json) : config = json['config']; + VeilidStateConfig.fromJson(dynamic json) + : config = VeilidConfig.fromJson(json['config']); Map toJson() { - return {'config': config}; + return {'config': config.toJson()}; } } diff --git a/veilid-python/.gitignore b/veilid-python/.gitignore new file mode 100644 index 00000000..68bc17f9 --- /dev/null +++ b/veilid-python/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/veilid-python/README.md b/veilid-python/README.md new file mode 100644 index 00000000..dabe4ad4 --- /dev/null +++ b/veilid-python/README.md @@ -0,0 +1,20 @@ +# Veilid Bindings for Python + +## Usage + +To use: +``` +poetry add veilid_python +``` +or +``` +pip3 install veilid_python +``` + + +## Development + +To update schema for validation: +``` +./update_schema.sh +``` diff --git a/veilid-python/poetry.lock b/veilid-python/poetry.lock new file mode 100644 index 00000000..9e4d4a9c --- /dev/null +++ b/veilid-python/poetry.lock @@ -0,0 +1,959 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "argcomplete" +version = "3.1.0" +description = "Bash tab completion for argparse" +optional = false +python-versions = ">=3.6" +files = [ + {file = "argcomplete-3.1.0-py3-none-any.whl", hash = "sha256:faea1cba05479b66628d7b216fcd23f2b9ad0117b13280aad8ae0b30872cd1f5"}, + {file = "argcomplete-3.1.0.tar.gz", hash = "sha256:3a81445fa51c9875c36789ae5386d5e95c5c43d503963a0029a747aea74de1e3"}, +] + +[package.extras] +test = ["coverage", "mypy", "pexpect", "ruff", "wheel"] + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "black" +version = "23.3.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.7" +files = [ + {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, + {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, + {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, + {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, + {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, + {file = "black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, + {file = "black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, + {file = "black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, + {file = "black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, + {file = "black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, + {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, + {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, + {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, + {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "certifi" +version = "2023.5.7" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, +] + +[[package]] +name = "chardet" +version = "5.1.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.1.0-py3-none-any.whl", hash = "sha256:362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"}, + {file = "chardet-5.1.0.tar.gz", hash = "sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "datamodel-code-generator" +version = "0.20.0" +description = "Datamodel Code Generator" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "datamodel_code_generator-0.20.0-py3-none-any.whl", hash = "sha256:fdb8dc18fd3a5d2c92e5e3ac473c5a48f3540066941e56fd947d2975cfebd281"}, + {file = "datamodel_code_generator-0.20.0.tar.gz", hash = "sha256:84dc7d6ae64ca67834b414c107adf3510f46a474ac21467a683aca0a4a8f0806"}, +] + +[package.dependencies] +argcomplete = ">=1.10,<4.0" +black = ">=19.10b0" +genson = ">=1.2.1,<2.0" +inflect = ">=4.1.0,<6.0" +isort = ">=4.3.21,<6.0" +jinja2 = ">=2.10.1,<4.0" +openapi-spec-validator = ">=0.2.8,<=0.5.2" +packaging = "*" +prance = ">=0.18.2,<1.0" +pydantic = {version = ">=1.10.0,<2.0.0", extras = ["email"], markers = "python_version >= \"3.11\" and python_version < \"4.0\""} +PySnooper = ">=0.4.1,<2.0.0" +toml = ">=0.10.0,<1.0.0" + +[package.extras] +http = ["httpx"] + +[[package]] +name = "dnspython" +version = "2.3.0" +description = "DNS toolkit" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, + {file = "dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, +] + +[package.extras] +curio = ["curio (>=1.2,<2.0)", "sniffio (>=1.1,<2.0)"] +dnssec = ["cryptography (>=2.6,<40.0)"] +doh = ["h2 (>=4.1.0)", "httpx (>=0.21.1)", "requests (>=2.23.0,<3.0.0)", "requests-toolbelt (>=0.9.1,<0.11.0)"] +doq = ["aioquic (>=0.9.20)"] +idna = ["idna (>=2.1,<4.0)"] +trio = ["trio (>=0.14,<0.23)"] +wmi = ["wmi (>=1.5.1,<2.0.0)"] + +[[package]] +name = "email-validator" +version = "2.0.0.post2" +description = "A robust email address syntax and deliverability validation library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "email_validator-2.0.0.post2-py3-none-any.whl", hash = "sha256:2466ba57cda361fb7309fd3d5a225723c788ca4bbad32a0ebd5373b99730285c"}, + {file = "email_validator-2.0.0.post2.tar.gz", hash = "sha256:1ff6e86044200c56ae23595695c54e9614f4a9551e0e393614f764860b3d7900"}, +] + +[package.dependencies] +dnspython = ">=2.0.0" +idna = ">=2.0.0" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "inflect" +version = "5.6.2" +description = "Correctly generate plurals, singular nouns, ordinals, indefinite articles; convert numbers to words" +optional = false +python-versions = ">=3.7" +files = [ + {file = "inflect-5.6.2-py3-none-any.whl", hash = "sha256:b45d91a4a28a4e617ff1821117439b06eaa86e2a4573154af0149e9be6687238"}, + {file = "inflect-5.6.2.tar.gz", hash = "sha256:aadc7ed73928f5e014129794bbac03058cca35d0a973a5fc4eb45c7fa26005f9"}, +] + +[package.extras] +docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] +testing = ["pygments", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "isort" +version = "5.12.0" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, + {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, +] + +[package.extras] +colors = ["colorama (>=0.4.3)"] +pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] +plugins = ["setuptools"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "json-ref-dict" +version = "0.7.1" +description = "Python dict-like object which abstracts resolution of JSONSchema references" +optional = false +python-versions = "*" +files = [ + {file = "json-ref-dict-0.7.1.tar.gz", hash = "sha256:b1a0f7acad8a022bdd5e2f1a8943fe4dde6f429ec40966e29c992ee8f6349b72"}, + {file = "json_ref_dict-0.7.1-py3-none-any.whl", hash = "sha256:0cfdd3967594bc24e74a9b37f17cbda597f5a38755c35ace52fef92b7be266cf"}, +] + +[package.dependencies] +jsonpointer = ">=2.0,<3.0" + +[[package]] +name = "jsonpointer" +version = "2.3" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "jsonpointer-2.3-py2.py3-none-any.whl", hash = "sha256:51801e558539b4e9cd268638c078c6c5746c9ac96bc38152d443400e4f3793e9"}, + {file = "jsonpointer-2.3.tar.gz", hash = "sha256:97cba51526c829282218feb99dab1b1e6bdf8efd1c43dc9d57be093c0d69c99a"}, +] + +[[package]] +name = "jsonschema" +version = "4.17.3" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, + {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "jsonschema-spec" +version = "0.1.6" +description = "JSONSchema Spec with object-oriented paths" +optional = false +python-versions = ">=3.7.0,<4.0.0" +files = [ + {file = "jsonschema_spec-0.1.6-py3-none-any.whl", hash = "sha256:f2206d18c89d1824c1f775ba14ed039743b41a9167bd2c5bdb774b66b3ca0bbf"}, + {file = "jsonschema_spec-0.1.6.tar.gz", hash = "sha256:90215863b56e212086641956b20127ccbf6d8a3a38343dad01d6a74d19482f76"}, +] + +[package.dependencies] +jsonschema = ">=4.0.0,<4.18.0" +pathable = ">=0.4.1,<0.5.0" +PyYAML = ">=5.1" +requests = ">=2.31.0,<3.0.0" + +[[package]] +name = "lazy-object-proxy" +version = "1.9.0" +description = "A fast and thorough lazy object proxy." +optional = false +python-versions = ">=3.7" +files = [ + {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, +] + +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "openapi-schema-validator" +version = "0.4.4" +description = "OpenAPI schema validation for Python" +optional = false +python-versions = ">=3.7.0,<4.0.0" +files = [ + {file = "openapi_schema_validator-0.4.4-py3-none-any.whl", hash = "sha256:79f37f38ef9fd5206b924ed7a6f382cea7b649b3b56383c47f1906082b7b9015"}, + {file = "openapi_schema_validator-0.4.4.tar.gz", hash = "sha256:c573e2be2c783abae56c5a1486ab716ca96e09d1c3eab56020d1dc680aa57bf8"}, +] + +[package.dependencies] +jsonschema = ">=4.0.0,<4.18.0" +rfc3339-validator = "*" + +[package.extras] +docs = ["sphinx (>=5.3.0,<6.0.0)", "sphinx-immaterial (>=0.11.0,<0.12.0)"] + +[[package]] +name = "openapi-spec-validator" +version = "0.5.2" +description = "OpenAPI 2.0 (aka Swagger) and OpenAPI 3 spec validator" +optional = false +python-versions = ">=3.7.0,<4.0.0" +files = [ + {file = "openapi_spec_validator-0.5.2-py3-none-any.whl", hash = "sha256:1f8db08ecbcf4ec8c558d65b65b3b7b428f81da6642f2f163e992ae3e17b229c"}, + {file = "openapi_spec_validator-0.5.2.tar.gz", hash = "sha256:ebed7f1c567780859402ad64b128e17f519d15f605f1b41d1e9a4a7a1690be07"}, +] + +[package.dependencies] +jsonschema = ">=4.0.0,<5.0.0" +jsonschema-spec = ">=0.1.1,<0.2.0" +lazy-object-proxy = ">=1.7.1,<2.0.0" +openapi-schema-validator = ">=0.3.2,<0.5" + +[package.extras] +requests = ["requests"] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pathable" +version = "0.4.3" +description = "Object-oriented paths" +optional = false +python-versions = ">=3.7.0,<4.0.0" +files = [ + {file = "pathable-0.4.3-py3-none-any.whl", hash = "sha256:cdd7b1f9d7d5c8b8d3315dbf5a86b2596053ae845f056f57d97c0eefff84da14"}, + {file = "pathable-0.4.3.tar.gz", hash = "sha256:5c869d315be50776cc8a993f3af43e0c60dc01506b399643f919034ebf4cdcab"}, +] + +[[package]] +name = "pathspec" +version = "0.11.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, +] + +[[package]] +name = "platformdirs" +version = "3.5.3" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.5.3-py3-none-any.whl", hash = "sha256:0ade98a4895e87dc51d47151f7d2ec290365a585151d97b4d8d6312ed6132fed"}, + {file = "platformdirs-3.5.3.tar.gz", hash = "sha256:e48fabd87db8f3a7df7150a4a5ea22c546ee8bc39bc2473244730d4b56d2cc4e"}, +] + +[package.extras] +docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] + +[[package]] +name = "prance" +version = "0.22.2.22.0" +description = "Resolving Swagger/OpenAPI 2.0 and 3.0.0 Parser" +optional = false +python-versions = ">=3.8" +files = [ + {file = "prance-0.22.2.22.0-py3-none-any.whl", hash = "sha256:57deeb67b7e93ef27c1c17845bf3ccb4af288ccfb5748c7e01779c01a8507f27"}, + {file = "prance-0.22.2.22.0.tar.gz", hash = "sha256:9a83f8a4f5fe0f2d896d238d4bec6b5788b10b94155414b3d88c21c1579b85bf"}, +] + +[package.dependencies] +chardet = ">=3.0" +packaging = ">=21.3" +requests = ">=2.25" +"ruamel.yaml" = ">=0.17.10" +six = ">=1.15,<2.0" + +[package.extras] +cli = ["click (>=7.0)"] +dev = ["bumpversion (>=0.6)", "pytest (>=6.1)", "pytest-cov (>=2.11)", "sphinx (>=3.4)", "towncrier (>=19.2)", "tox (>=3.4)"] +flex = ["flex (>=6.13,<7.0)"] +icu = ["PyICU (>=2.4,<3.0)"] +osv = ["openapi-spec-validator (>=0.5.1,<0.6.0)"] +ssv = ["swagger-spec-validator (>=2.4,<3.0)"] + +[[package]] +name = "pydantic" +version = "1.10.9" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e692dec4a40bfb40ca530e07805b1208c1de071a18d26af4a2a0d79015b352ca"}, + {file = "pydantic-1.10.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c52eb595db83e189419bf337b59154bdcca642ee4b2a09e5d7797e41ace783f"}, + {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:939328fd539b8d0edf244327398a667b6b140afd3bf7e347cf9813c736211896"}, + {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b48d3d634bca23b172f47f2335c617d3fcb4b3ba18481c96b7943a4c634f5c8d"}, + {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f0b7628fb8efe60fe66fd4adadd7ad2304014770cdc1f4934db41fe46cc8825f"}, + {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e1aa5c2410769ca28aa9a7841b80d9d9a1c5f223928ca8bec7e7c9a34d26b1d4"}, + {file = "pydantic-1.10.9-cp310-cp310-win_amd64.whl", hash = "sha256:eec39224b2b2e861259d6f3c8b6290d4e0fbdce147adb797484a42278a1a486f"}, + {file = "pydantic-1.10.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d111a21bbbfd85c17248130deac02bbd9b5e20b303338e0dbe0faa78330e37e0"}, + {file = "pydantic-1.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e9aec8627a1a6823fc62fb96480abe3eb10168fd0d859ee3d3b395105ae19a7"}, + {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07293ab08e7b4d3c9d7de4949a0ea571f11e4557d19ea24dd3ae0c524c0c334d"}, + {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee829b86ce984261d99ff2fd6e88f2230068d96c2a582f29583ed602ef3fc2c"}, + {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4b466a23009ff5cdd7076eb56aca537c745ca491293cc38e72bf1e0e00de5b91"}, + {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7847ca62e581e6088d9000f3c497267868ca2fa89432714e21a4fb33a04d52e8"}, + {file = "pydantic-1.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:7845b31959468bc5b78d7b95ec52fe5be32b55d0d09983a877cca6aedc51068f"}, + {file = "pydantic-1.10.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:517a681919bf880ce1dac7e5bc0c3af1e58ba118fd774da2ffcd93c5f96eaece"}, + {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67195274fd27780f15c4c372f4ba9a5c02dad6d50647b917b6a92bf00b3d301a"}, + {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2196c06484da2b3fded1ab6dbe182bdabeb09f6318b7fdc412609ee2b564c49a"}, + {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6257bb45ad78abacda13f15bde5886efd6bf549dd71085e64b8dcf9919c38b60"}, + {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3283b574b01e8dbc982080d8287c968489d25329a463b29a90d4157de4f2baaf"}, + {file = "pydantic-1.10.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5f8bbaf4013b9a50e8100333cc4e3fa2f81214033e05ac5aa44fa24a98670a29"}, + {file = "pydantic-1.10.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9cd67fb763248cbe38f0593cd8611bfe4b8ad82acb3bdf2b0898c23415a1f82"}, + {file = "pydantic-1.10.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f50e1764ce9353be67267e7fd0da08349397c7db17a562ad036aa7c8f4adfdb6"}, + {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73ef93e5e1d3c8e83f1ff2e7fdd026d9e063c7e089394869a6e2985696693766"}, + {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:128d9453d92e6e81e881dd7e2484e08d8b164da5507f62d06ceecf84bf2e21d3"}, + {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ad428e92ab68798d9326bb3e5515bc927444a3d71a93b4a2ca02a8a5d795c572"}, + {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fab81a92f42d6d525dd47ced310b0c3e10c416bbfae5d59523e63ea22f82b31e"}, + {file = "pydantic-1.10.9-cp38-cp38-win_amd64.whl", hash = "sha256:963671eda0b6ba6926d8fc759e3e10335e1dc1b71ff2a43ed2efd6996634dafb"}, + {file = "pydantic-1.10.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:970b1bdc6243ef663ba5c7e36ac9ab1f2bfecb8ad297c9824b542d41a750b298"}, + {file = "pydantic-1.10.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7e1d5290044f620f80cf1c969c542a5468f3656de47b41aa78100c5baa2b8276"}, + {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83fcff3c7df7adff880622a98022626f4f6dbce6639a88a15a3ce0f96466cb60"}, + {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0da48717dc9495d3a8f215e0d012599db6b8092db02acac5e0d58a65248ec5bc"}, + {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0a2aabdc73c2a5960e87c3ffebca6ccde88665616d1fd6d3db3178ef427b267a"}, + {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9863b9420d99dfa9c064042304868e8ba08e89081428a1c471858aa2af6f57c4"}, + {file = "pydantic-1.10.9-cp39-cp39-win_amd64.whl", hash = "sha256:e7c9900b43ac14110efa977be3da28931ffc74c27e96ee89fbcaaf0b0fe338e1"}, + {file = "pydantic-1.10.9-py3-none-any.whl", hash = "sha256:6cafde02f6699ce4ff643417d1a9223716ec25e228ddc3b436fe7e2d25a1f305"}, + {file = "pydantic-1.10.9.tar.gz", hash = "sha256:95c70da2cd3b6ddf3b9645ecaa8d98f3d80c606624b6d245558d202cd23ea3be"}, +] + +[package.dependencies] +email-validator = {version = ">=1.0.3", optional = true, markers = "extra == \"email\""} +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.19.3" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"}, + {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"}, + {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"}, + {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"}, + {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"}, + {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"}, +] + +[[package]] +name = "pysnooper" +version = "1.1.1" +description = "A poor man's debugger for Python." +optional = false +python-versions = "*" +files = [ + {file = "PySnooper-1.1.1-py2.py3-none-any.whl", hash = "sha256:378f13d731a3e04d3d0350e5f295bdd0f1b49fc8a8b8bf2067fe1e5290bd20be"}, + {file = "PySnooper-1.1.1.tar.gz", hash = "sha256:d17dc91cca1593c10230dce45e46b1d3ff0f8910f0c38e941edf6ba1260b3820"}, +] + +[package.extras] +tests = ["pytest"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +description = "A pure python RFC3339 validator" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"}, + {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "ruamel-yaml" +version = "0.17.31" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +optional = false +python-versions = ">=3" +files = [ + {file = "ruamel.yaml-0.17.31-py3-none-any.whl", hash = "sha256:3cf153f0047ced526e723097ac615d3009371779432e304dbd5596b6f3a4c777"}, + {file = "ruamel.yaml-0.17.31.tar.gz", hash = "sha256:098ed1eb6d338a684891a72380277c1e6fc4d4ae0e120de9a447275056dda335"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.12\""} + +[package.extras] +docs = ["ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.7" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +optional = false +python-versions = ">=3.5" +files = [ + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:370445fd795706fd291ab00c9df38a0caed0f17a6fb46b0f607668ecb16ce763"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win32.whl", hash = "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:40d030e2329ce5286d6b231b8726959ebbe0404c92f0a578c0e2482182e38282"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c3ca1fbba4ae962521e5eb66d72998b51f0f4d0f608d3c0347a48e1af262efa7"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3243f48ecd450eddadc2d11b5feb08aca941b5cd98c9b1db14b2fd128be8c697"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win32.whl", hash = "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf9a6bc4a0221538b1a7de3ed7bca4c93c02346853f44e1cd764be0023cd3640"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win32.whl", hash = "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5"}, + {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, +] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "statham-schema" +version = "0.14.0" +description = "Tools for generating Python models from JSON Schema documents." +optional = false +python-versions = "*" +files = [ + {file = "statham-schema-0.14.0.tar.gz", hash = "sha256:2570642c0c3f0864d86659cafa6293e011a9e113953376e7d79c4c28009c066a"}, + {file = "statham_schema-0.14.0-py3-none-any.whl", hash = "sha256:2e922c21f3ac2ebf0ac79a142df272bd30b0fd16c8235f6e2628240c7fa2bfd9"}, +] + +[package.dependencies] +json-ref-dict = ">=0.6.2,<0.8.0" +jsonpointer = ">=2.0,<3.0" +python-dateutil = ">=2.8,<3.0" +PyYAML = ">=5.4,<6.0" +typing-extensions = ">=3.7.0,<5.0.0" + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.6.3" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.6.3-py3-none-any.whl", hash = "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26"}, + {file = "typing_extensions-4.6.3.tar.gz", hash = "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5"}, +] + +[[package]] +name = "urllib3" +version = "2.0.3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, + {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "f4368567ec647c3c626e34fb7b1115dca196b68a2290bce7f223260a24a9c54c" diff --git a/veilid-python/pyproject.toml b/veilid-python/pyproject.toml new file mode 100644 index 00000000..c4b15dd9 --- /dev/null +++ b/veilid-python/pyproject.toml @@ -0,0 +1,20 @@ +[tool.poetry] +name = "veilid-python" +version = "0.1.0" +description = "" +authors = ["Christien Rioux "] +readme = "README.md" +packages = [{include = "veilid_python"}] + +[tool.poetry.scripts] +update_bindings = "veilid_python.update_bindings:main" + +[tool.poetry.dependencies] +python = "^3.11" + +[tool.poetry.group.dev.dependencies] +datamodel-code-generator = "^0.20.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/veilid-python/schema/RecvMessage.json b/veilid-python/schema/RecvMessage.json new file mode 100644 index 00000000..b84dc8e4 --- /dev/null +++ b/veilid-python/schema/RecvMessage.json @@ -0,0 +1,3924 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RecvMessage", + "oneOf": [ + { + "type": "object", + "oneOf": [ + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "Control" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "$ref": "#/definitions/VeilidState" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "GetState" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "Attach" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "Detach" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "$ref": "#/definitions/NewPrivateRouteResult" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "NewPrivateRoute" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "$ref": "#/definitions/NewPrivateRouteResult" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "NewCustomPrivateRoute" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "ImportRemotePrivateRoute" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "ReleasePrivateRoute" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "AppCallReply" + ] + } + } + }, + { + "type": "object", + "required": [ + "op", + "value" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "NewRoutingContext" + ] + }, + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "InvalidId" + ] + } + } + }, + { + "type": "object", + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "Release" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "WithPrivacy" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "WithCustomPrivacy" + ] + } + } + }, + { + "type": "object", + "required": [ + "rc_op", + "value" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "WithSequencing" + ] + }, + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "AppCall" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "AppMessage" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "$ref": "#/definitions/DHTRecordDescriptor" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "CreateDhtRecord" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "$ref": "#/definitions/DHTRecordDescriptor" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "OpenDhtRecord" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "CloseDhtRecord" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "DeleteDhtRecord" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "$ref": "#/definitions/ValueData" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "GetDhtValue" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "$ref": "#/definitions/ValueData" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "SetDhtValue" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "WatchDhtValues" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "boolean" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "CancelDhtWatch" + ] + } + } + } + ], + "required": [ + "op", + "rc_id" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "RoutingContext" + ] + }, + "rc_id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "OpenTableDb" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "boolean" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "DeleteTableDb" + ] + } + } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "InvalidId" + ] + } + } + }, + { + "type": "object", + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "Release" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "GetColumnCount" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "GetKeys" + ] + } + } + }, + { + "type": "object", + "required": [ + "db_op", + "value" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "Transact" + ] + }, + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "Store" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "properties": { + "value": { + "type": [ + "string", + "null" + ] + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "Load" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "properties": { + "value": { + "type": [ + "string", + "null" + ] + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "Delete" + ] + } + } + } + ], + "required": [ + "db_id", + "op" + ], + "properties": { + "db_id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "TableDb" + ] + } + } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "tx_op" + ], + "properties": { + "tx_op": { + "type": "string", + "enum": [ + "InvalidId" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "tx_op" + ], + "properties": { + "tx_op": { + "type": "string", + "enum": [ + "Commit" + ] + } + } + }, + { + "type": "object", + "required": [ + "tx_op" + ], + "properties": { + "tx_op": { + "type": "string", + "enum": [ + "Rollback" + ] + } + } + }, + { + "type": "object", + "required": [ + "tx_op" + ], + "properties": { + "tx_op": { + "type": "string", + "enum": [ + "Store" + ] + } + } + }, + { + "type": "object", + "required": [ + "tx_op" + ], + "properties": { + "tx_op": { + "type": "string", + "enum": [ + "Delete" + ] + } + } + } + ], + "required": [ + "op", + "tx_id" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "TableDbTransaction" + ] + }, + "tx_id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "GetCryptoSystem" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "BestCryptoSystem" + ] + } + } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "InvalidId" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "Release" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "CachedDh" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "ComputeDh" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "RandomBytes" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "DefaultSaltLength" + ] + }, + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "HashPassword" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "boolean" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "VerifyPassword" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "DeriveSharedSecret" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "RandomNonce" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "RandomSharedSecret" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "GenerateKeyPair" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "GenerateHash" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "ValidateKeyPair" + ] + }, + "value": { + "type": "boolean" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "ValidateHash" + ] + }, + "value": { + "type": "boolean" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "Distance" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "Sign" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "Verify" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "AeadOverhead" + ] + }, + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "DecryptAead" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "EncryptAead" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "value" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "CryptNoAuth" + ] + }, + "value": { + "type": "string" + } + } + } + ], + "required": [ + "cs_id", + "op" + ], + "properties": { + "cs_id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "CryptoSystem" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "VerifySignatures" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "GenerateSignatures" + ] + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "GenerateKeyPair" + ] + } + } + }, + { + "type": "object", + "required": [ + "op", + "value" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "Now" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "Debug" + ] + } + } + }, + { + "type": "object", + "required": [ + "op", + "value" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "VeilidVersionString" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "major", + "minor", + "op", + "patch" + ], + "properties": { + "major": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "minor": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "VeilidVersion" + ] + }, + "patch": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + } + ], + "required": [ + "type" + ], + "properties": { + "id": { + "description": "Operation Id (pairs with Request, or empty if unidirectional)", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "type": { + "type": "string", + "enum": [ + "Response" + ] + } + } + }, + { + "type": "object", + "oneOf": [ + { + "description": "A VeilidCore log message with optional backtrace", + "type": "object", + "required": [ + "kind", + "log_level", + "message" + ], + "properties": { + "backtrace": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "type": "string", + "enum": [ + "Log" + ] + }, + "log_level": { + "$ref": "#/definitions/VeilidLogLevel" + }, + "message": { + "type": "string" + } + } + }, + { + "description": "Direct statement blob passed to hosting application for processing", + "type": "object", + "required": [ + "kind", + "message" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "AppMessage" + ] + }, + "message": { + "description": "The content of the message to deliver to the application", + "type": "string" + }, + "sender": { + "description": "Some(sender) if the message was sent directly, None if received via a private/safety route", + "type": [ + "string", + "null" + ] + } + } + }, + { + "description": "Direct question blob passed to hosting application for processing to send an eventual AppReply", + "type": "object", + "required": [ + "id", + "kind", + "message" + ], + "properties": { + "id": { + "description": "The id to reply to", + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "AppCall" + ] + }, + "message": { + "description": "The content of the request to deliver to the application", + "type": "string" + }, + "sender": { + "description": "Some(sender) if the request was sent directly, None if received via a private/safety route", + "type": [ + "string", + "null" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind", + "local_network_ready", + "public_internet_ready", + "state" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "Attachment" + ] + }, + "local_network_ready": { + "type": "boolean" + }, + "public_internet_ready": { + "type": "boolean" + }, + "state": { + "$ref": "#/definitions/AttachmentState" + } + } + }, + { + "type": "object", + "required": [ + "bps_down", + "bps_up", + "kind", + "peers", + "started" + ], + "properties": { + "bps_down": { + "type": "string" + }, + "bps_up": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "Network" + ] + }, + "peers": { + "type": "array", + "items": { + "$ref": "#/definitions/PeerTableData" + } + }, + "started": { + "type": "boolean" + } + } + }, + { + "type": "object", + "required": [ + "config", + "kind" + ], + "properties": { + "config": { + "$ref": "#/definitions/VeilidConfigInner" + }, + "kind": { + "type": "string", + "enum": [ + "Config" + ] + } + } + }, + { + "type": "object", + "required": [ + "dead_remote_routes", + "dead_routes", + "kind" + ], + "properties": { + "dead_remote_routes": { + "type": "array", + "items": { + "type": "string" + } + }, + "dead_routes": { + "type": "array", + "items": { + "type": "string" + } + }, + "kind": { + "type": "string", + "enum": [ + "RouteChange" + ] + } + } + }, + { + "type": "object", + "required": [ + "count", + "key", + "kind", + "subkeys", + "value" + ], + "properties": { + "count": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "key": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "ValueChange" + ] + }, + "subkeys": { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + }, + "value": { + "$ref": "#/definitions/ValueData" + } + } + }, + { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "Shutdown" + ] + } + } + } + ], + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "Update" + ] + } + } + } + ], + "definitions": { + "AttachmentState": { + "description": "Attachment abstraction for network 'signal strength'", + "type": "string", + "enum": [ + "Detached", + "Attaching", + "AttachedWeak", + "AttachedGood", + "AttachedStrong", + "FullyAttached", + "OverAttached", + "Detaching" + ] + }, + "DHTRecordDescriptor": { + "description": "DHT Record Descriptor", + "type": "object", + "required": [ + "key", + "owner", + "schema" + ], + "properties": { + "key": { + "description": "DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ]", + "type": "string" + }, + "owner": { + "description": "The public key of the owner", + "type": "string" + }, + "owner_secret": { + "description": "If this key is being created: Some(the secret key of the owner) If this key is just being opened: None", + "type": [ + "string", + "null" + ] + }, + "schema": { + "description": "The schema in use associated with the key", + "allOf": [ + { + "$ref": "#/definitions/DHTSchema" + } + ] + } + } + }, + "DHTSchema": { + "description": "Enum over all the supported DHT Schemas", + "oneOf": [ + { + "description": "Default DHT Schema (DFLT)", + "type": "object", + "required": [ + "kind", + "o_cnt" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "DFLT" + ] + }, + "o_cnt": { + "description": "Owner subkey count", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + } + } + }, + { + "description": "Simple DHT Schema (SMPL)", + "type": "object", + "required": [ + "kind", + "members", + "o_cnt" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "SMPL" + ] + }, + "members": { + "description": "Members", + "type": "array", + "items": { + "$ref": "#/definitions/DHTSchemaSMPLMember" + } + }, + "o_cnt": { + "description": "Owner subkey count", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + } + } + } + ] + }, + "DHTSchemaSMPLMember": { + "description": "Simple DHT Schema (SMPL) Member", + "type": "object", + "required": [ + "m_cnt", + "m_key" + ], + "properties": { + "m_cnt": { + "description": "Member subkey count", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "m_key": { + "description": "Member key", + "type": "string" + } + } + }, + "LatencyStats": { + "type": "object", + "required": [ + "average", + "fastest", + "slowest" + ], + "properties": { + "average": { + "type": "string" + }, + "fastest": { + "type": "string" + }, + "slowest": { + "type": "string" + } + } + }, + "NewPrivateRouteResult": { + "type": "object", + "required": [ + "blob", + "route_id" + ], + "properties": { + "blob": { + "type": "string" + }, + "route_id": { + "type": "string" + } + } + }, + "PeerStats": { + "type": "object", + "required": [ + "rpc_stats", + "time_added", + "transfer" + ], + "properties": { + "latency": { + "anyOf": [ + { + "$ref": "#/definitions/LatencyStats" + }, + { + "type": "null" + } + ] + }, + "rpc_stats": { + "$ref": "#/definitions/RPCStats" + }, + "time_added": { + "type": "string" + }, + "transfer": { + "$ref": "#/definitions/TransferStatsDownUp" + } + } + }, + "PeerTableData": { + "type": "object", + "required": [ + "node_ids", + "peer_address", + "peer_stats" + ], + "properties": { + "node_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "peer_address": { + "type": "string" + }, + "peer_stats": { + "$ref": "#/definitions/PeerStats" + } + } + }, + "RPCStats": { + "type": "object", + "required": [ + "failed_to_send", + "messages_rcvd", + "messages_sent", + "questions_in_flight", + "recent_lost_answers" + ], + "properties": { + "failed_to_send": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "first_consecutive_seen_ts": { + "type": [ + "string", + "null" + ] + }, + "last_question_ts": { + "type": [ + "string", + "null" + ] + }, + "last_seen_ts": { + "type": [ + "string", + "null" + ] + }, + "messages_rcvd": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "messages_sent": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "questions_in_flight": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "recent_lost_answers": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "TransferStats": { + "type": "object", + "required": [ + "average", + "maximum", + "minimum", + "total" + ], + "properties": { + "average": { + "type": "string" + }, + "maximum": { + "type": "string" + }, + "minimum": { + "type": "string" + }, + "total": { + "type": "string" + } + } + }, + "TransferStatsDownUp": { + "type": "object", + "required": [ + "down", + "up" + ], + "properties": { + "down": { + "$ref": "#/definitions/TransferStats" + }, + "up": { + "$ref": "#/definitions/TransferStats" + } + } + }, + "ValueData": { + "type": "object", + "required": [ + "data", + "seq", + "writer" + ], + "properties": { + "data": { + "description": "The contents of a DHT Record", + "type": "string" + }, + "seq": { + "description": "An increasing sequence number to time-order the DHT record changes", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "writer": { + "description": "The public identity key of the writer of the data", + "type": "string" + } + } + }, + "VeilidAPIError": { + "oneOf": [ + { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "NotInitialized" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "AlreadyInitialized" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "Timeout" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "TryAgain" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "Shutdown" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "InvalidTarget" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind", + "message" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "NoConnection" + ] + }, + "message": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "key", + "kind" + ], + "properties": { + "key": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "KeyNotFound" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind", + "message" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "Internal" + ] + }, + "message": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "message" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "Unimplemented" + ] + }, + "message": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "message", + "value" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "ParseError" + ] + }, + "message": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "argument", + "context", + "kind", + "value" + ], + "properties": { + "argument": { + "type": "string" + }, + "context": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "InvalidArgument" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "argument", + "context", + "kind" + ], + "properties": { + "argument": { + "type": "string" + }, + "context": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "MissingArgument" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind", + "message" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "Generic" + ] + }, + "message": { + "type": "string" + } + } + } + ] + }, + "VeilidConfigApplication": { + "description": "Application configuration\n\nConfigure web access to the Prograssive Web App (PWA)\n\nTo be implemented...", + "type": "object", + "required": [ + "http", + "https" + ], + "properties": { + "http": { + "$ref": "#/definitions/VeilidConfigHTTP" + }, + "https": { + "$ref": "#/definitions/VeilidConfigHTTPS" + } + } + }, + "VeilidConfigBlockStore": { + "type": "object", + "required": [ + "delete", + "directory" + ], + "properties": { + "delete": { + "type": "boolean" + }, + "directory": { + "type": "string" + } + } + }, + "VeilidConfigCapabilities": { + "type": "object", + "required": [ + "protocol_accept_tcp", + "protocol_accept_ws", + "protocol_accept_wss", + "protocol_connect_tcp", + "protocol_connect_ws", + "protocol_connect_wss", + "protocol_udp" + ], + "properties": { + "protocol_accept_tcp": { + "type": "boolean" + }, + "protocol_accept_ws": { + "type": "boolean" + }, + "protocol_accept_wss": { + "type": "boolean" + }, + "protocol_connect_tcp": { + "type": "boolean" + }, + "protocol_connect_ws": { + "type": "boolean" + }, + "protocol_connect_wss": { + "type": "boolean" + }, + "protocol_udp": { + "type": "boolean" + } + } + }, + "VeilidConfigDHT": { + "description": "Configure the Distributed Hash Table (DHT)", + "type": "object", + "required": [ + "get_value_count", + "get_value_fanout", + "get_value_timeout_ms", + "local_max_subkey_cache_memory_mb", + "local_subkey_cache_size", + "max_find_node_count", + "min_peer_count", + "min_peer_refresh_time_ms", + "remote_max_records", + "remote_max_storage_space_mb", + "remote_max_subkey_cache_memory_mb", + "remote_subkey_cache_size", + "resolve_node_count", + "resolve_node_fanout", + "resolve_node_timeout_ms", + "set_value_count", + "set_value_fanout", + "set_value_timeout_ms", + "validate_dial_info_receipt_time_ms" + ], + "properties": { + "get_value_count": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "get_value_fanout": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "get_value_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "local_max_subkey_cache_memory_mb": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "local_subkey_cache_size": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "max_find_node_count": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "min_peer_count": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "min_peer_refresh_time_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "remote_max_records": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "remote_max_storage_space_mb": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "remote_max_subkey_cache_memory_mb": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "remote_subkey_cache_size": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "resolve_node_count": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "resolve_node_fanout": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "resolve_node_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "set_value_count": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "set_value_fanout": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "set_value_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "validate_dial_info_receipt_time_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "VeilidConfigHTTP": { + "description": "Enable and configure HTTP access to the Veilid node\n\n```yaml http: enabled: false listen_address: ':5150' path: 'app\" url: 'https://localhost:5150' ```", + "type": "object", + "required": [ + "enabled", + "listen_address", + "path" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "listen_address": { + "type": "string" + }, + "path": { + "type": "string" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "VeilidConfigHTTPS": { + "description": "Enable and configure HTTPS access to the Veilid node\n\n```yaml https: enabled: false listen_address: ':5150' path: 'app' url: 'https://localhost:5150' ```", + "type": "object", + "required": [ + "enabled", + "listen_address", + "path" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "listen_address": { + "type": "string" + }, + "path": { + "type": "string" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "VeilidConfigInner": { + "type": "object", + "required": [ + "block_store", + "capabilities", + "namespace", + "network", + "program_name", + "protected_store", + "table_store" + ], + "properties": { + "block_store": { + "$ref": "#/definitions/VeilidConfigBlockStore" + }, + "capabilities": { + "$ref": "#/definitions/VeilidConfigCapabilities" + }, + "namespace": { + "type": "string" + }, + "network": { + "$ref": "#/definitions/VeilidConfigNetwork" + }, + "program_name": { + "type": "string" + }, + "protected_store": { + "$ref": "#/definitions/VeilidConfigProtectedStore" + }, + "table_store": { + "$ref": "#/definitions/VeilidConfigTableStore" + } + } + }, + "VeilidConfigNetwork": { + "type": "object", + "required": [ + "application", + "client_whitelist_timeout_ms", + "connection_inactivity_timeout_ms", + "connection_initial_timeout_ms", + "detect_address_changes", + "dht", + "hole_punch_receipt_time_ms", + "max_connection_frequency_per_min", + "max_connections_per_ip4", + "max_connections_per_ip6_prefix", + "max_connections_per_ip6_prefix_size", + "protocol", + "restricted_nat_retries", + "reverse_connection_receipt_time_ms", + "routing_table", + "rpc", + "tls", + "upnp" + ], + "properties": { + "application": { + "$ref": "#/definitions/VeilidConfigApplication" + }, + "client_whitelist_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "connection_inactivity_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "connection_initial_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "detect_address_changes": { + "type": "boolean" + }, + "dht": { + "$ref": "#/definitions/VeilidConfigDHT" + }, + "hole_punch_receipt_time_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "max_connection_frequency_per_min": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "max_connections_per_ip4": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "max_connections_per_ip6_prefix": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "max_connections_per_ip6_prefix_size": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "protocol": { + "$ref": "#/definitions/VeilidConfigProtocol" + }, + "restricted_nat_retries": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "reverse_connection_receipt_time_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "routing_table": { + "$ref": "#/definitions/VeilidConfigRoutingTable" + }, + "rpc": { + "$ref": "#/definitions/VeilidConfigRPC" + }, + "tls": { + "$ref": "#/definitions/VeilidConfigTLS" + }, + "upnp": { + "type": "boolean" + } + } + }, + "VeilidConfigProtectedStore": { + "type": "object", + "required": [ + "allow_insecure_fallback", + "always_use_insecure_storage", + "delete", + "device_encryption_key_password", + "directory" + ], + "properties": { + "allow_insecure_fallback": { + "type": "boolean" + }, + "always_use_insecure_storage": { + "type": "boolean" + }, + "delete": { + "type": "boolean" + }, + "device_encryption_key_password": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "new_device_encryption_key_password": { + "type": [ + "string", + "null" + ] + } + } + }, + "VeilidConfigProtocol": { + "description": "Configure Network Protocols\n\nVeilid can communicate over UDP, TCP, and Web Sockets.\n\nAll protocols are available by default, and the Veilid node will sort out which protocol is used for each peer connection.", + "type": "object", + "required": [ + "tcp", + "udp", + "ws", + "wss" + ], + "properties": { + "tcp": { + "$ref": "#/definitions/VeilidConfigTCP" + }, + "udp": { + "$ref": "#/definitions/VeilidConfigUDP" + }, + "ws": { + "$ref": "#/definitions/VeilidConfigWS" + }, + "wss": { + "$ref": "#/definitions/VeilidConfigWSS" + } + } + }, + "VeilidConfigRPC": { + "description": "Configure RPC", + "type": "object", + "required": [ + "concurrency", + "default_route_hop_count", + "max_route_hop_count", + "queue_size", + "timeout_ms" + ], + "properties": { + "concurrency": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "default_route_hop_count": { + "type": "integer", + "format": "uint8", + "minimum": 0.0 + }, + "max_route_hop_count": { + "type": "integer", + "format": "uint8", + "minimum": 0.0 + }, + "max_timestamp_ahead_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "max_timestamp_behind_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "queue_size": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "VeilidConfigRoutingTable": { + "description": "Configure the network routing table", + "type": "object", + "required": [ + "bootstrap", + "limit_attached_good", + "limit_attached_strong", + "limit_attached_weak", + "limit_fully_attached", + "limit_over_attached", + "node_id", + "node_id_secret" + ], + "properties": { + "bootstrap": { + "type": "array", + "items": { + "type": "string" + } + }, + "limit_attached_good": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "limit_attached_strong": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "limit_attached_weak": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "limit_fully_attached": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "limit_over_attached": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "node_id": { + "type": "array", + "items": { + "type": "string" + } + }, + "node_id_secret": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "VeilidConfigTCP": { + "description": "Enable and configure TCP\n\n```yaml tcp: connect: true listen: true max_connections: 32 listen_address: ':5150' public_address: ''", + "type": "object", + "required": [ + "connect", + "listen", + "listen_address", + "max_connections" + ], + "properties": { + "connect": { + "type": "boolean" + }, + "listen": { + "type": "boolean" + }, + "listen_address": { + "type": "string" + }, + "max_connections": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "public_address": { + "type": [ + "string", + "null" + ] + } + } + }, + "VeilidConfigTLS": { + "description": "Configure TLS\n\n```yaml tls: certificate_path: /path/to/cert private_key_path: /path/to/private/key connection_initial_timeout_ms: 2000", + "type": "object", + "required": [ + "certificate_path", + "connection_initial_timeout_ms", + "private_key_path" + ], + "properties": { + "certificate_path": { + "type": "string" + }, + "connection_initial_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "private_key_path": { + "type": "string" + } + } + }, + "VeilidConfigTableStore": { + "type": "object", + "required": [ + "delete", + "directory" + ], + "properties": { + "delete": { + "type": "boolean" + }, + "directory": { + "type": "string" + } + } + }, + "VeilidConfigUDP": { + "description": "Enable and configure UDP\n\n```yaml udp: enabled: true socket_pool_size: 0 listen_address: ':5150' public_address: '' ```", + "type": "object", + "required": [ + "enabled", + "listen_address", + "socket_pool_size" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "listen_address": { + "type": "string" + }, + "public_address": { + "type": [ + "string", + "null" + ] + }, + "socket_pool_size": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "VeilidConfigWS": { + "description": "Enable and configure Web Sockets\n\n```yaml ws: connect: true listen: true max_connections: 16 listen_address: ':5150' path: 'ws' url: 'ws://localhost:5150/ws'", + "type": "object", + "required": [ + "connect", + "listen", + "listen_address", + "max_connections", + "path" + ], + "properties": { + "connect": { + "type": "boolean" + }, + "listen": { + "type": "boolean" + }, + "listen_address": { + "type": "string" + }, + "max_connections": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "path": { + "type": "string" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "VeilidConfigWSS": { + "description": "Enable and configure Secure Web Sockets\n\n```yaml wss: connect: true listen: false max_connections: 16 listen_address: ':5150' path: 'ws' url: ''", + "type": "object", + "required": [ + "connect", + "listen", + "listen_address", + "max_connections", + "path" + ], + "properties": { + "connect": { + "type": "boolean" + }, + "listen": { + "type": "boolean" + }, + "listen_address": { + "type": "string" + }, + "max_connections": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "path": { + "type": "string" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "VeilidLogLevel": { + "description": "Log level for VeilidCore", + "type": "string", + "enum": [ + "Error", + "Warn", + "Info", + "Debug", + "Trace" + ] + }, + "VeilidState": { + "type": "object", + "required": [ + "attachment", + "config", + "network" + ], + "properties": { + "attachment": { + "$ref": "#/definitions/VeilidStateAttachment" + }, + "config": { + "$ref": "#/definitions/VeilidStateConfig" + }, + "network": { + "$ref": "#/definitions/VeilidStateNetwork" + } + } + }, + "VeilidStateAttachment": { + "type": "object", + "required": [ + "local_network_ready", + "public_internet_ready", + "state" + ], + "properties": { + "local_network_ready": { + "type": "boolean" + }, + "public_internet_ready": { + "type": "boolean" + }, + "state": { + "$ref": "#/definitions/AttachmentState" + } + } + }, + "VeilidStateConfig": { + "type": "object", + "required": [ + "config" + ], + "properties": { + "config": { + "$ref": "#/definitions/VeilidConfigInner" + } + } + }, + "VeilidStateNetwork": { + "type": "object", + "required": [ + "bps_down", + "bps_up", + "peers", + "started" + ], + "properties": { + "bps_down": { + "type": "string" + }, + "bps_up": { + "type": "string" + }, + "peers": { + "type": "array", + "items": { + "$ref": "#/definitions/PeerTableData" + } + }, + "started": { + "type": "boolean" + } + } + } + } +} diff --git a/veilid-python/schema/Request.json b/veilid-python/schema/Request.json new file mode 100644 index 00000000..82119da0 --- /dev/null +++ b/veilid-python/schema/Request.json @@ -0,0 +1,1585 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Request", + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "args", + "op" + ], + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "op": { + "type": "string", + "enum": [ + "Control" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "GetState" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "Attach" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "Detach" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "NewPrivateRoute" + ] + } + } + }, + { + "type": "object", + "required": [ + "kinds", + "op" + ], + "properties": { + "kinds": { + "type": "array", + "items": { + "type": "string" + } + }, + "op": { + "type": "string", + "enum": [ + "NewCustomPrivateRoute" + ] + }, + "sequencing": { + "default": "NoPreference", + "allOf": [ + { + "$ref": "#/definitions/Sequencing" + } + ] + }, + "stability": { + "default": "LowLatency", + "allOf": [ + { + "$ref": "#/definitions/Stability" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "blob", + "op" + ], + "properties": { + "blob": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "ImportRemotePrivateRoute" + ] + } + } + }, + { + "type": "object", + "required": [ + "op", + "route_id" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "ReleasePrivateRoute" + ] + }, + "route_id": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "call_id", + "message", + "op" + ], + "properties": { + "call_id": { + "type": "string" + }, + "message": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "AppCallReply" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "NewRoutingContext" + ] + } + } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "Release" + ] + } + } + }, + { + "type": "object", + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "WithPrivacy" + ] + } + } + }, + { + "type": "object", + "required": [ + "rc_op", + "stability" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "WithCustomPrivacy" + ] + }, + "stability": { + "$ref": "#/definitions/Stability" + } + } + }, + { + "type": "object", + "required": [ + "rc_op", + "sequencing" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "WithSequencing" + ] + }, + "sequencing": { + "$ref": "#/definitions/Sequencing" + } + } + }, + { + "type": "object", + "required": [ + "rc_op", + "request", + "target" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "AppCall" + ] + }, + "request": { + "type": "string" + }, + "target": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "message", + "rc_op", + "target" + ], + "properties": { + "message": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "AppMessage" + ] + }, + "target": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "rc_op", + "schema" + ], + "properties": { + "kind": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "CreateDhtRecord" + ] + }, + "schema": { + "$ref": "#/definitions/DHTSchema" + } + } + }, + { + "type": "object", + "required": [ + "key", + "rc_op" + ], + "properties": { + "key": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "OpenDhtRecord" + ] + }, + "writer": { + "type": [ + "string", + "null" + ] + } + } + }, + { + "type": "object", + "required": [ + "key", + "rc_op" + ], + "properties": { + "key": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "CloseDhtRecord" + ] + } + } + }, + { + "type": "object", + "required": [ + "key", + "rc_op" + ], + "properties": { + "key": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "DeleteDhtRecord" + ] + } + } + }, + { + "type": "object", + "required": [ + "force_refresh", + "key", + "rc_op", + "subkey" + ], + "properties": { + "force_refresh": { + "type": "boolean" + }, + "key": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "GetDhtValue" + ] + }, + "subkey": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "data", + "key", + "rc_op", + "subkey" + ], + "properties": { + "data": { + "type": "string" + }, + "key": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "SetDhtValue" + ] + }, + "subkey": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "count", + "expiration", + "key", + "rc_op", + "subkeys" + ], + "properties": { + "count": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "expiration": { + "type": "string" + }, + "key": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "WatchDhtValues" + ] + }, + "subkeys": { + "type": "array", + "items": { + "type": "array", + "items": [ + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + ], + "maxItems": 2, + "minItems": 2 + } + } + } + }, + { + "type": "object", + "required": [ + "key", + "rc_op", + "subkeys" + ], + "properties": { + "key": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "CancelDhtWatch" + ] + }, + "subkeys": { + "type": "array", + "items": { + "type": "array", + "items": [ + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + ], + "maxItems": 2, + "minItems": 2 + } + } + } + } + ], + "required": [ + "op", + "rc_id" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "RoutingContext" + ] + }, + "rc_id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "column_count", + "name", + "op" + ], + "properties": { + "column_count": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "name": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "OpenTableDb" + ] + } + } + }, + { + "type": "object", + "required": [ + "name", + "op" + ], + "properties": { + "name": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "DeleteTableDb" + ] + } + } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "Release" + ] + } + } + }, + { + "type": "object", + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "GetColumnCount" + ] + } + } + }, + { + "type": "object", + "required": [ + "col", + "db_op" + ], + "properties": { + "col": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "db_op": { + "type": "string", + "enum": [ + "GetKeys" + ] + } + } + }, + { + "type": "object", + "required": [ + "db_op" + ], + "properties": { + "db_op": { + "type": "string", + "enum": [ + "Transact" + ] + } + } + }, + { + "type": "object", + "required": [ + "col", + "db_op", + "key", + "value" + ], + "properties": { + "col": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "db_op": { + "type": "string", + "enum": [ + "Store" + ] + }, + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "col", + "db_op", + "key" + ], + "properties": { + "col": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "db_op": { + "type": "string", + "enum": [ + "Load" + ] + }, + "key": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "col", + "db_op", + "key" + ], + "properties": { + "col": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "db_op": { + "type": "string", + "enum": [ + "Delete" + ] + }, + "key": { + "type": "string" + } + } + } + ], + "required": [ + "db_id", + "op" + ], + "properties": { + "db_id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "TableDb" + ] + } + } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "tx_op" + ], + "properties": { + "tx_op": { + "type": "string", + "enum": [ + "Commit" + ] + } + } + }, + { + "type": "object", + "required": [ + "tx_op" + ], + "properties": { + "tx_op": { + "type": "string", + "enum": [ + "Rollback" + ] + } + } + }, + { + "type": "object", + "required": [ + "col", + "key", + "tx_op", + "value" + ], + "properties": { + "col": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "key": { + "type": "string" + }, + "tx_op": { + "type": "string", + "enum": [ + "Store" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "col", + "key", + "tx_op" + ], + "properties": { + "col": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "key": { + "type": "string" + }, + "tx_op": { + "type": "string", + "enum": [ + "Delete" + ] + } + } + } + ], + "required": [ + "op", + "tx_id" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "TableDbTransaction" + ] + }, + "tx_id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "kind", + "op" + ], + "properties": { + "kind": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "GetCryptoSystem" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "BestCryptoSystem" + ] + } + } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "Release" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "key", + "secret" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "CachedDh" + ] + }, + "key": { + "type": "string" + }, + "secret": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "key", + "secret" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "ComputeDh" + ] + }, + "key": { + "type": "string" + }, + "secret": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "len" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "RandomBytes" + ] + }, + "len": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + { + "type": "object", + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "DefaultSaltLength" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "password", + "salt" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "HashPassword" + ] + }, + "password": { + "type": "string" + }, + "salt": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "password", + "password_hash" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "VerifyPassword" + ] + }, + "password": { + "type": "string" + }, + "password_hash": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "password", + "salt" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "DeriveSharedSecret" + ] + }, + "password": { + "type": "string" + }, + "salt": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "RandomNonce" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "RandomSharedSecret" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "GenerateKeyPair" + ] + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "data" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "GenerateHash" + ] + }, + "data": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "key", + "secret" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "ValidateKeyPair" + ] + }, + "key": { + "type": "string" + }, + "secret": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "data", + "hash_digest" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "ValidateHash" + ] + }, + "data": { + "type": "string" + }, + "hash_digest": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "key1", + "key2" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "Distance" + ] + }, + "key1": { + "type": "string" + }, + "key2": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "data", + "key", + "secret" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "Sign" + ] + }, + "data": { + "type": "string" + }, + "key": { + "type": "string" + }, + "secret": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op", + "data", + "key", + "secret" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "Verify" + ] + }, + "data": { + "type": "string" + }, + "key": { + "type": "string" + }, + "secret": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "cs_op" + ], + "properties": { + "cs_op": { + "type": "string", + "enum": [ + "AeadOverhead" + ] + } + } + }, + { + "type": "object", + "required": [ + "body", + "cs_op", + "nonce", + "shared_secret" + ], + "properties": { + "associated_data": { + "type": [ + "string", + "null" + ] + }, + "body": { + "type": "string" + }, + "cs_op": { + "type": "string", + "enum": [ + "DecryptAead" + ] + }, + "nonce": { + "type": "string" + }, + "shared_secret": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "body", + "cs_op", + "nonce", + "shared_secret" + ], + "properties": { + "associated_data": { + "type": [ + "string", + "null" + ] + }, + "body": { + "type": "string" + }, + "cs_op": { + "type": "string", + "enum": [ + "EncryptAead" + ] + }, + "nonce": { + "type": "string" + }, + "shared_secret": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "body", + "cs_op", + "nonce", + "shared_secret" + ], + "properties": { + "body": { + "type": "string" + }, + "cs_op": { + "type": "string", + "enum": [ + "CryptNoAuth" + ] + }, + "nonce": { + "type": "string" + }, + "shared_secret": { + "type": "string" + } + } + } + ], + "required": [ + "cs_id", + "op" + ], + "properties": { + "cs_id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "CryptoSystem" + ] + } + } + }, + { + "type": "object", + "required": [ + "data", + "node_ids", + "op", + "signatures" + ], + "properties": { + "data": { + "type": "string" + }, + "node_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "op": { + "type": "string", + "enum": [ + "VerifySignatures" + ] + }, + "signatures": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "required": [ + "data", + "key_pairs", + "op" + ], + "properties": { + "data": { + "type": "string" + }, + "key_pairs": { + "type": "array", + "items": { + "type": "string" + } + }, + "op": { + "type": "string", + "enum": [ + "GenerateSignatures" + ] + } + } + }, + { + "type": "object", + "required": [ + "kind", + "op" + ], + "properties": { + "kind": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "GenerateKeyPair" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "Now" + ] + } + } + }, + { + "type": "object", + "required": [ + "command", + "op" + ], + "properties": { + "command": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "Debug" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "VeilidVersionString" + ] + } + } + }, + { + "type": "object", + "required": [ + "op" + ], + "properties": { + "op": { + "type": "string", + "enum": [ + "VeilidVersion" + ] + } + } + } + ], + "properties": { + "id": { + "description": "Operation Id (pairs with Response, or empty if unidirectional)", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + }, + "definitions": { + "DHTSchema": { + "description": "Enum over all the supported DHT Schemas", + "oneOf": [ + { + "description": "Default DHT Schema (DFLT)", + "type": "object", + "required": [ + "kind", + "o_cnt" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "DFLT" + ] + }, + "o_cnt": { + "description": "Owner subkey count", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + } + } + }, + { + "description": "Simple DHT Schema (SMPL)", + "type": "object", + "required": [ + "kind", + "members", + "o_cnt" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "SMPL" + ] + }, + "members": { + "description": "Members", + "type": "array", + "items": { + "$ref": "#/definitions/DHTSchemaSMPLMember" + } + }, + "o_cnt": { + "description": "Owner subkey count", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + } + } + } + ] + }, + "DHTSchemaSMPLMember": { + "description": "Simple DHT Schema (SMPL) Member", + "type": "object", + "required": [ + "m_cnt", + "m_key" + ], + "properties": { + "m_cnt": { + "description": "Member subkey count", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "m_key": { + "description": "Member key", + "type": "string" + } + } + }, + "Sequencing": { + "type": "string", + "enum": [ + "NoPreference", + "PreferOrdered", + "EnsureOrdered" + ] + }, + "Stability": { + "type": "string", + "enum": [ + "LowLatency", + "Reliable" + ] + } + } +} diff --git a/veilid-python/tests/__init__.py b/veilid-python/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/veilid-python/update_schema.sh b/veilid-python/update_schema.sh new file mode 100755 index 00000000..b00e14e8 --- /dev/null +++ b/veilid-python/update_schema.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -eo pipefail +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +VEILID_SERVER=$SCRIPTDIR/../target/debug/veilid-server + +# Ensure executable exists +if [ ! -f "$VEILID_SERVER" ]; then + echo "$VEILID_SERVER does not exist. Build with cargo build." + exit 1 +fi + +# Produce schema from veilid-server +$VEILID_SERVER --emit-schema Request > $SCRIPTDIR/schema/Request.json +$VEILID_SERVER --emit-schema RecvMessage > $SCRIPTDIR/schema/RecvMessage.json + + diff --git a/veilid-python/veilid_python/__init__.py b/veilid-python/veilid_python/__init__.py new file mode 100644 index 00000000..cc512528 --- /dev/null +++ b/veilid-python/veilid_python/__init__.py @@ -0,0 +1 @@ +from .error import * diff --git a/veilid-python/veilid_python/api.py b/veilid-python/veilid_python/api.py new file mode 100644 index 00000000..abd9a10f --- /dev/null +++ b/veilid-python/veilid_python/api.py @@ -0,0 +1,11 @@ +from abc import ABC, abstractmethod +from .state import VeilidState + +class VeilidAPI(ABC): + @abstractmethod + def control(self, args: list[str]) -> str: + pass + @abstractmethod + def get_state(self) -> VeilidState: + pass + \ No newline at end of file diff --git a/veilid-python/veilid_python/config.py b/veilid-python/veilid_python/config.py new file mode 100644 index 00000000..ba520cc6 --- /dev/null +++ b/veilid-python/veilid_python/config.py @@ -0,0 +1,254 @@ +from typing import Self, Optional +from enum import StrEnum + +class VeilidConfigLogLevel(StrEnum): + OFF = 'Off' + ERROR = 'Error' + WARN = 'Warn' + INFO = 'Info' + DEBUG = 'Debug' + TRACE = 'Trace' + +class VeilidConfigCapabilities: + protocol_udp: bool + protocol_connect_tcp: bool + protocol_accept_tcp: bool + protocol_connect_ws: bool + protocol_accept_ws: bool + protocol_connect_wss: bool + protocol_accept_wss: bool + + def __init__(self, protocol_udp: bool, protocol_connect_tcp: bool, protocol_accept_tcp: bool, + protocol_connect_ws: bool, protocol_accept_ws: bool, protocol_connect_wss: bool, protocol_accept_wss: bool): + + self.protocol_udp = protocol_udp + self.protocol_connect_tcp = protocol_connect_tcp + self.protocol_accept_tcp = protocol_accept_tcp + self.protocol_connect_ws = protocol_connect_ws + self.protocol_accept_ws = protocol_accept_ws + self.protocol_connect_wss = protocol_connect_wss + self.protocol_accept_wss = protocol_accept_wss + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigCapabilities(j['protocol_udp'], + j['protocol_connect_tcp'], + j['protocol_accept_tcp'], + j['protocol_connect_ws'], + j['protocol_accept_ws'], + j['protocol_connect_wss'], + j['protocol_accept_wss']) + +class VeilidConfigProtectedStore: + allow_insecure_fallback: bool + always_use_insecure_storage: bool + directory: str + delete: bool + device_encryption_key_password: str + new_device_encryption_key_password: Optional[str] + + def __init__(self, allow_insecure_fallback: bool, always_use_insecure_storage: bool, + directory: str, delete: bool, device_encryption_key_password: str, new_device_encryption_key_password: Optional[str]): + + self.allow_insecure_fallback = allow_insecure_fallback + self.always_use_insecure_storage = always_use_insecure_storage + self.directory = directory + self.delete = delete + self.device_encryption_key_password = device_encryption_key_password + self.new_device_encryption_key_password = new_device_encryption_key_password + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigProtectedStore(j['allow_insecure_fallback'], j['always_use_insecure_storage'], + j['directory'], j['delete'], j['device_encryption_key_password'], j['new_device_encryption_key_password']) + +class VeilidConfigTableStore: + directory: str + delete: bool + + def __init__(self, directory: str, delete: bool): + self.directory = directory + self.delete = delete + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigTableStore(j['directory'], j['delete']) + +class VeilidConfigBlockStore: + directory: str + delete: bool + + def __init__(self, directory: str, delete: bool): + self.directory = directory + self.delete = delete + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigBlockStore(j['directory'], j['delete']) + +class VeilidConfigRoutingTable: + node_id: list[str] + node_id_secret: list[str] + bootstrap: list[str] + limit_over_attached: int + limit_fully_attached: int + limit_attached_strong: int + limit_attached_good: int + limit_attached_weak: int + + def __init__(self, node_id: list[str], node_id_secret: list[str], bootstrap: list[str], limit_over_attached: int, + limit_fully_attached: int, limit_attached_strong: int, limit_attached_good: int, limit_attached_weak: int): + + self.node_id = node_id + self.node_id_secret = node_id_secret + self.bootstrap = bootstrap + self.limit_over_attached = limit_over_attached + self.limit_fully_attached = limit_fully_attached + self.limit_attached_strong = limit_attached_strong + self.limit_attached_good = limit_attached_good + self.limit_attached_weak = limit_attached_weak + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigRoutingTable( + j['node_id'], + j['node_id_secret'], + j['bootstrap'], + j['limit_over_attached'], + j['limit_fully_attached'], + j['limit_attached_strong'], + j['limit_attached_good'], + j['limit_attached_weak']) + + +class VeilidConfigRPC: + concurrency: int + queue_size: int + max_timestamp_behind_ms: Optional[int] + max_timestamp_ahead_ms: Optional[int] + timeout_ms: int + max_route_hop_count: int + default_route_hop_count: int + + def __init__(self, concurrency: int, queue_size: int, max_timestamp_behind_ms: Optional[int], max_timestamp_ahead_ms: Optional[int], + timeout_ms: int, max_route_hop_count: int, default_route_hop_count: int): + + self.concurrency = concurrency + self.queue_size = queue_size + self.max_timestamp_behind_ms = max_timestamp_behind_ms + self.max_timestamp_ahead_ms = max_timestamp_ahead_ms + self.timeout_ms = timeout_ms + self.max_route_hop_count = max_route_hop_count + self.default_route_hop_count = default_route_hop_count + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigRPC( + j['concurrency'], + j['queue_size'], + j['max_timestamp_behind_ms'], + j['max_timestamp_ahead_ms'], + j['timeout_ms'], + j['max_route_hop_count'], + j['default_route_hop_count']) + + +class VeilidConfigNetwork: + connection_initial_timeout_ms: int + connection_inactivity_timeout_ms: int + max_connections_per_ip4: int + max_connections_per_ip6_prefix: int + max_connections_per_ip6_prefix_size: int + max_connection_frequency_per_min: int + client_whitelist_timeout_ms: int + reverse_connection_receipt_time_ms: int + hole_punch_receipt_time_ms: int + routing_table: VeilidConfigRoutingTable + rpc: VeilidConfigRPC + dht: VeilidConfigDHT + upnp: bool + detect_address_changes: bool + restricted_nat_retries: int + tls: VeilidConfigTLS + application: VeilidConfigApplication + protocol: VeilidConfigProtocol + + def __init__(self, connection_initial_timeout_ms: int, connection_inactivity_timeout_ms: int, + max_connections_per_ip4: int, max_connections_per_ip6_prefix: int, + max_connections_per_ip6_prefix_size: int, max_connection_frequency_per_min: int, + client_whitelist_timeout_ms: int, reverse_connection_receipt_time_ms: int, + hole_punch_receipt_time_ms: int, routing_table: VeilidConfigRoutingTable, + rpc: VeilidConfigRPC, dht: VeilidConfigDHT, upnp: bool, detect_address_changes: bool, + restricted_nat_retries: int, tls: VeilidConfigTLS, application: VeilidConfigApplication, protocol: VeilidConfigProtocol): + + self.connection_initial_timeout_ms = connection_initial_timeout_ms + self.connection_inactivity_timeout_ms = connection_inactivity_timeout_ms + self.max_connections_per_ip4 = max_connections_per_ip4 + self.max_connections_per_ip6_prefix = max_connections_per_ip6_prefix + self.max_connections_per_ip6_prefix_size = max_connections_per_ip6_prefix_size + self.max_connection_frequency_per_min = max_connection_frequency_per_min + self.client_whitelist_timeout_ms = client_whitelist_timeout_ms + self.reverse_connection_receipt_time_ms = reverse_connection_receipt_time_ms + self.hole_punch_receipt_time_ms = hole_punch_receipt_time_ms + self.routing_table = routing_table + self.rpc = rpc + self.dht = dht + self.upnp = upnp + self.detect_address_changes = detect_address_changes + self.restricted_nat_retries = restricted_nat_retries + self.tls = tls + self.application = application + self.protocol = protocol + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigNetwork( + j['connection_initial_timeout_ms'], + j['connection_inactivity_timeout_ms'], + j['max_connections_per_ip4'], + j['max_connections_per_ip6_prefix'], + j['max_connections_per_ip6_prefix_size'], + j['max_connection_frequency_per_min'], + j['client_whitelist_timeout_ms'], + j['reverse_connection_receipt_time_ms'], + j['hole_punch_receipt_time_ms'], + VeilidConfigRoutingTable.from_json(j['routing_table']), + VeilidConfigRPC.from_json(j['rpc']), + VeilidConfigDHT.from_json(j['dht']), + j['upnp'], + j['detect_address_changes'], + j['restricted_nat_retries'], + VeilidConfigTLS.from_json(j['tls']), + VeilidConfigApplication.from_json(j['application']), + VeilidConfigProtocol.from_json(j['protocol'])) + +class VeilidConfig: + program_name: str + namespace: str + capabilities: VeilidConfigCapabilities + protected_store: VeilidConfigProtectedStore + table_store: VeilidConfigTableStore + block_store: VeilidConfigBlockStore + network: VeilidConfigNetwork + + def __init__(self, program_name: str, namespace: str, capabilities: VeilidConfigCapabilities, + protected_store: VeilidConfigProtectedStore, table_store: VeilidConfigTableStore, + block_store: VeilidConfigBlockStore, network: VeilidConfigNetwork): + + self.program_name = program_name + self.namespace = namespace + self.capabilities = capabilities + self.protected_store = protected_store + self.table_store = table_store + self.block_store = block_store + self.network = network + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfig(j['program_name'], j['namespace'], + VeilidConfigCapabilities.from_json(j['capabilities']), + VeilidConfigProtectedStore.from_json(j['protected_store']), + VeilidConfigTableStore.from_json(j['table_store']), + VeilidConfigBlockStore.from_json(j['block_store']), + VeilidConfigNetwork.from_json(j['network'])) + diff --git a/veilid-python/veilid_python/error.py b/veilid-python/veilid_python/error.py new file mode 100644 index 00000000..d3ded7da --- /dev/null +++ b/veilid-python/veilid_python/error.py @@ -0,0 +1,133 @@ +from typing import Self + +class VeilidAPIError(Exception): + """Veilid API error exception base class""" + pass + @staticmethod + def from_json(j: dict) -> Self: + match j['kind']: + case 'NotInitialized': + return VeilidAPIErrorNotInitialized() + case 'AlreadyInitialized': + return VeilidAPIErrorAlreadyInitialized() + case 'Timeout': + return VeilidAPIErrorTimeout() + case 'TryAgain': + return VeilidAPIErrorTryAgain() + case 'Shutdown': + return VeilidAPIErrorShutdown() + case 'InvalidTarget': + return VeilidAPIErrorInvalidTarget() + case 'NoConnection': + return VeilidAPIErrorNoConnection(j['message']) + case 'KeyNotFound': + return VeilidAPIErrorKeyNotFound(j['key']) + case 'Internal': + return VeilidAPIErrorInternal(j['message']) + case 'Unimplemented': + return VeilidAPIErrorUnimplemented(j['message']) + case 'ParseError': + return VeilidAPIErrorParseError(j['message'], j['value']) + case 'InvalidArgument': + return VeilidAPIErrorInvalidArgument(j['context'], j['argument'], j['value']) + case 'MissingArgument': + return VeilidAPIErrorMissingArgument(j['context'], j['argument']) + case 'Generic': + return VeilidAPIErrorGeneric(j['message']) + case _: + return VeilidAPIError("Unknown exception type: {}".format(j['kind'])) + + +class VeilidAPIErrorNotInitialized(VeilidAPIError): + """Veilid was not initialized""" + def __init__(self): + super().__init__("Not initialized") + +class VeilidAPIErrorAlreadyInitialized(VeilidAPIError): + """Veilid was already initialized""" + def __init__(self): + super().__init__("Already initialized") + +class VeilidAPIErrorTimeout(VeilidAPIError): + """Veilid operation timed out""" + def __init__(self): + super().__init__("Timeout") + +class VeilidAPIErrorTryAgain(VeilidAPIError): + """Operation could not be performed at this time, retry again later""" + def __init__(self): + super().__init__("Try again") + +class VeilidAPIErrorShutdown(VeilidAPIError): + """Veilid was already shut down""" + def __init__(self): + super().__init__("Shutdown") + +class VeilidAPIErrorInvalidTarget(VeilidAPIError): + """Target of operation is not valid""" + def __init__(self): + super().__init__("Invalid target") + +class VeilidAPIErrorNoConnection(VeilidAPIError): + """Connection could not be established""" + message: str + def __init__(self, message: str): + super().__init__("No connection") + self.message = message + +class VeilidAPIErrorKeyNotFound(VeilidAPIError): + """Key was not found""" + key: str + def __init__(self, key: str): + super().__init__("Key not found") + self.key = key + +class VeilidAPIErrorInternal(VeilidAPIError): + """Veilid experienced an internal failure""" + message: str + def __init__(self, message: str): + super().__init__("Internal") + self.message = message + +class VeilidAPIErrorUnimplemented(VeilidAPIError): + """Functionality is not yet implemented""" + message: str + def __init__(self, message: str): + super().__init__("Unimplemented") + self.message = message + +class VeilidAPIErrorParseError(VeilidAPIError): + """Value was not in a parseable format""" + message: str + value: str + def __init__(self, message: str, value: str): + super().__init__("Parse error") + self.message = message + self.value = value + +class VeilidAPIErrorInvalidArgument(VeilidAPIError): + """Argument is not valid in this context""" + context: str + argument: str + value: str + def __init__(self, context: str, argument: str, value: str): + super().__init__("Invalid argument") + self.context = context + self.argument = argument + self.value = value + +class VeilidAPIErrorMissingArgument(VeilidAPIError): + """Required argument was missing""" + context: str + argument: str + def __init__(self, context: str, argument: str): + super().__init__("Missing argument") + self.context = context + self.argument = argument + +class VeilidAPIErrorGeneric(VeilidAPIError): + """Generic error message""" + message: str + def __init__(self, message: str): + super().__init__("Generic") + self.message = message diff --git a/veilid-python/veilid_python/request.py b/veilid-python/veilid_python/request.py new file mode 100644 index 00000000..19d59ae2 --- /dev/null +++ b/veilid-python/veilid_python/request.py @@ -0,0 +1,4 @@ +class Request: + + def __init__(self, id: int): + self.id = id \ No newline at end of file diff --git a/veilid-python/veilid_python/state.py b/veilid-python/veilid_python/state.py new file mode 100644 index 00000000..f5ac2abb --- /dev/null +++ b/veilid-python/veilid_python/state.py @@ -0,0 +1,205 @@ +from typing import Self, Optional +from enum import StrEnum +from .types import Timestamp, TimestampDuration, ByteCount +from .config import VeilidConfig + +class AttachmentState(StrEnum): + DETACHED = 'Detached' + ATTACHING = 'Attaching' + ATTACHED_WEAK = 'AttachedWeak' + ATTACHED_GOOD = 'AttachedGood' + ATTACHED_STRONG = 'AttachedStrong' + FULLY_ATTACHED = 'FullyAttached' + OVER_ATTACHED = 'OverAttached' + DETACHING = 'Detaching' + +class VeilidStateAttachment: + state: AttachmentState + public_internet_ready: bool + local_network_ready: bool + + def __init__(self, state: AttachmentState, public_internet_ready: bool, local_network_ready: bool): + self.state = state + self.public_internet_ready = public_internet_ready + self.local_network_ready = local_network_ready + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidStateAttachment( + AttachmentState(j['state']), + j['public_internet_ready'], + j['local_network_ready']) + +class RPCStats: + messages_sent: int + messages_rcvd: int + questions_in_flight: int + last_question_ts: Optional[Timestamp] + last_seen_ts: Optional[Timestamp] + first_consecutive_seen_ts: Optional[Timestamp] + recent_lost_answers: int + failed_to_send: int + + def __init__(self, messages_sent: int, messages_rcvd: int, questions_in_flight: int, + last_question_ts: Optional[Timestamp], last_seen_ts: Optional[Timestamp], + first_consecutive_seen_ts: Optional[Timestamp], recent_lost_answers: int, failed_to_send: int): + self.messages_sent = messages_sent + self.messages_rcvd = messages_rcvd + self.questions_in_flight = questions_in_flight + self.last_question_ts = last_question_ts + self.last_seen_ts = last_seen_ts + self.first_consecutive_seen_ts = first_consecutive_seen_ts + self.recent_lost_answers = recent_lost_answers + self.failed_to_send = failed_to_send + + @staticmethod + def from_json(j: dict) -> Self: + return RPCStats( + j['messages_sent'], + j['messages_rcvd'], + j['questions_in_flight'], + None if j['last_question_ts'] is None else Timestamp(j['last_question_ts']), + None if j['last_seen_ts'] is None else Timestamp(j['last_seen_ts']), + None if j['first_consecutive_seen_ts'] is None else Timestamp(j['first_consecutive_seen_ts']), + j['recent_lost_answers'], + j['failed_to_send']) + +class LatencyStats: + fastest: TimestampDuration + average: TimestampDuration + slowest: TimestampDuration + + def __init__(self, fastest: TimestampDuration, average: TimestampDuration, slowest: TimestampDuration): + self.fastest = fastest + self.average = average + self.slowest = slowest + + @staticmethod + def from_json(j: dict) -> Self: + return LatencyStats( + TimestampDuration(j['fastest']), + TimestampDuration(j['average']), + TimestampDuration(j['slowest'])) + + +class TransferStats: + total: ByteCount + maximum: ByteCount + average: ByteCount + minimum: ByteCount + + def __init__(self, total: ByteCount, maximum: ByteCount, average: ByteCount, minimum: ByteCount): + self.total = total + self.maximum = maximum + self.average = average + self.minimum = minimum + + @staticmethod + def from_json(j: dict) -> Self: + return TransferStats( + ByteCount(j['total']), + ByteCount(j['maximum']), + ByteCount(j['average']), + ByteCount(j['minimum'])) + + +class TransferStatsDownUp: + down: TransferStats + up: TransferStats + + def __init__(self, down: TransferStats, up: TransferStats): + self.down = down + self.up = up + + @staticmethod + def from_json(j: dict) -> Self: + return TransferStatsDownUp( + TransferStats.from_json(j['down']), + TransferStats.from_json(j['up'])) + +class PeerStats: + time_added: Timestamp + rpc_stats: RPCStats + latency: Optional[LatencyStats] + transfer: TransferStatsDownUp + + def __init__(self, time_added: Timestamp, rpc_stats: RPCStats, latency: Optional[LatencyStats], transfer: TransferStatsDownUp): + self.time_added = time_added + self.rpc_stats = rpc_stats + self.latency = latency + self.transfer = transfer + + @staticmethod + def from_json(j: dict) -> Self: + return PeerStats( + j['time_added'], + RPCStats.from_json(j['rpc_stats']), + None if j['latency'] is None else LatencyStats.from_json(j['latency']), + TransferStatsDownUp.from_json(j['transfer'])) + +class PeerTableData: + node_ids: list[str] + peer_address: str + peer_stats: PeerStats + + def __init__(self, node_ids: list[str], peer_address: str, peer_stats: PeerStats): + self.node_ids = node_ids + self.peer_address = peer_address + self.peer_stats = peer_stats + + @staticmethod + def from_json(j: dict) -> Self: + return PeerTableData( + j['node_ids'], + j['peer_address'], + PeerStats.from_json(j['peer_stats'])) + +class VeilidStateNetwork: + started: bool + bps_down: ByteCount + bps_up: ByteCount + peers: list[PeerTableData] + + def __init__(self, started: bool, bps_down: ByteCount, bps_up: ByteCount, peers: list[PeerTableData]): + self.started = started + self.bps_down = bps_down + self.bps_up = bps_up + self.peers = peers + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidStateNetwork( + j['started'], + ByteCount(j['bps_down']), + ByteCount(j['bps_up']), + map(lambda x: PeerTableData.from_json(x), j['peers'])) + +class VeilidStateConfig: + config: VeilidConfig + + def __init__(self, config: VeilidConfig): + self.config = config + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidStateConfig( + j['config']) + + +class VeilidState: + attachment: VeilidStateAttachment + network: VeilidStateNetwork + config: VeilidStateConfig + + def __init__(self, attachment: VeilidStateAttachment, network: VeilidStateNetwork, config: VeilidStateConfig): + self.attachment = attachment + self.network = network + self.config = config + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidState( + VeilidStateAttachment.from_json(j['attachment']), + VeilidStateNetwork.from_json(j['network']), + VeilidStateConfig.from_json(j['config'])) + diff --git a/veilid-python/veilid_python/types.py b/veilid-python/veilid_python/types.py new file mode 100644 index 00000000..e1938e04 --- /dev/null +++ b/veilid-python/veilid_python/types.py @@ -0,0 +1,10 @@ +import time + +class Timestamp(int): + pass + +class TimestampDuration(int): + pass + +class ByteCount(int): + pass \ No newline at end of file From df0b06bf3c1c6c177b1f8696b0541849c1fe1940 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 13 Jun 2023 23:17:45 -0400 Subject: [PATCH 14/23] python work --- veilid-core/proto/veilid.capnp | 5 +- veilid-python/veilid_python/__init__.py | 1 - veilid-python/veilid_python/api.py | 208 +++++++++++++- veilid-python/veilid_python/config.py | 299 +++++++++++++++++++- veilid-python/veilid_python/error.py | 11 +- veilid-python/veilid_python/json_api.py | 314 ++++++++++++++++++++++ veilid-python/veilid_python/operations.py | 91 +++++++ veilid-python/veilid_python/request.py | 4 - veilid-python/veilid_python/state.py | 150 ++++++++++- veilid-python/veilid_python/types.py | 227 +++++++++++++++- 10 files changed, 1293 insertions(+), 17 deletions(-) create mode 100644 veilid-python/veilid_python/json_api.py create mode 100644 veilid-python/veilid_python/operations.py delete mode 100644 veilid-python/veilid_python/request.py diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index b7f1a359..59af6722 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -230,7 +230,7 @@ struct NodeInfo @0xe125d847e3f9f419 { outboundProtocols @1 :ProtocolTypeSet; # protocols that can go outbound addressTypes @2 :AddressTypeSet; # address types supported envelopeSupport @3 :List(UInt8); # supported rpc envelope/receipt versions - cryptoSupport @4 :List(CryptoKind); # cryptography systems supported + cryptoSupport @4 :List( ); # cryptography systems supported dialInfoDetailList @5 :List(DialInfoDetail); # inbound dial info details for this node } @@ -534,7 +534,8 @@ struct Answer @0xacacb8b6988c1058 { appCallA @2 :OperationAppCallA; getValueA @3 :OperationGetValueA; setValueA @4 :OperationSetValueA; - watchValueA @5 :OperationWatchValueA; + watchValueA @5 :OperationWatchValueA; + # #[cfg(feature="unstable-blockstore")] #supplyBlockA @6 :OperationSupplyBlockA; #findBlockA @7 :OperationFindBlockA; diff --git a/veilid-python/veilid_python/__init__.py b/veilid-python/veilid_python/__init__.py index cc512528..e69de29b 100644 --- a/veilid-python/veilid_python/__init__.py +++ b/veilid-python/veilid_python/__init__.py @@ -1 +0,0 @@ -from .error import * diff --git a/veilid-python/veilid_python/api.py b/veilid-python/veilid_python/api.py index abd9a10f..356866d8 100644 --- a/veilid-python/veilid_python/api.py +++ b/veilid-python/veilid_python/api.py @@ -1,11 +1,211 @@ from abc import ABC, abstractmethod -from .state import VeilidState +from typing import Self + +from .state import * +from .config import * +from .error import * +from .types import * + +class RoutingContext(ABC): + @abstractmethod + async def with_privacy(self) -> Self: + pass + @abstractmethod + async def with_custom_privacy(self, stability: Stability) -> Self: + pass + @abstractmethod + async def with_sequencing(self, sequencing: Sequencing) -> Self: + pass + @abstractmethod + async def app_call(self, target: TypedKey | RouteId, request: bytes) -> bytes: + pass + @abstractmethod + async def app_message(self, target: TypedKey | RouteId, message: bytes): + pass + @abstractmethod + async def create_dht_record(self, kind: CryptoKind, schema: DHTSchema) -> DHTRecordDescriptor: + pass + @abstractmethod + async def open_dht_record(self, key: TypedKey, writer: Optional[KeyPair]) -> DHTRecordDescriptor: + pass + @abstractmethod + async def close_dht_record(self, key: TypedKey): + pass + @abstractmethod + async def delete_dht_record(self, key: TypedKey): + pass + @abstractmethod + async def get_dht_value(self, key: TypedKey, subkey: ValueSubkey, force_refresh: bool) -> Optional[ValueData]: + pass + @abstractmethod + async def set_dht_value(self, key: TypedKey, subkey: ValueSubkey, data: bytes) -> Optional[ValueData]: + pass + @abstractmethod + async def watch_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)], expiration: Timestamp, count: int) -> Timestamp: + pass + @abstractmethod + async def cancel_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)]) -> bool: + pass + + +class TableDBTransaction(ABC): + @abstractmethod + async def commit(self): + pass + @abstractmethod + async def rollback(self): + pass + @abstractmethod + async def store(self, col: int, key: bytes, value: bytes): + pass + @abstractmethod + async def delete(self, col: int, key: bytes): + pass + +class TableDB(ABC): + @abstractmethod + async def get_column_count(self) -> int: + pass + @abstractmethod + async def get_keys(self, col: int) -> list[str]: + pass + @abstractmethod + async def transact(self) -> TableDBTransaction: + pass + @abstractmethod + async def store(self, col: int, key: bytes, value: bytes): + pass + @abstractmethod + async def load(self, col: int, key: bytes) -> Optional[bytes]: + pass + @abstractmethod + async def delete(self, col: int, key: bytes) -> Optional[bytes]: + pass + +class CryptoSystem(ABC): + @abstractmethod + async def cached_dh(self, key: PublicKey, secret: SecretKey) -> SharedSecret: + pass + @abstractmethod + async def compute_dh(self, key: PublicKey, secret: SecretKey) -> SharedSecret: + pass + @abstractmethod + async def random_bytes(self, len: int) -> bytes: + pass + @abstractmethod + async def default_salt_length(self) -> int: + pass + @abstractmethod + async def hash_password(self, password: bytes, salt: bytes) -> str: + pass + @abstractmethod + async def verify_password(self, password: bytes, password_hash: str) -> bool: + pass + @abstractmethod + async def derive_shared_secret(self, password: bytes, salt: bytes) -> SharedSecret: + pass + @abstractmethod + async def random_nonce(self) -> Nonce: + pass + @abstractmethod + async def random_shared_secret(self) -> SharedSecret: + pass + @abstractmethod + async def generate_key_pair(self) -> KeyPair: + pass + @abstractmethod + async def generate_hash(self, data: bytes) -> HashDigest: + pass + @abstractmethod + async def validate_key_pair(self, key: PublicKey, secret: SecretKey) -> bool: + pass + @abstractmethod + async def validate_hash(self, data: bytes, hash_digest: HashDigest) -> bool: + pass + @abstractmethod + async def distance(self, key1: CryptoKey, key2: CryptoKey) -> CryptoKeyDistance: + pass + @abstractmethod + async def sign(self, key: PublicKey, secret: SecretKey, data: bytes) -> Signature: + pass + @abstractmethod + async def verify(self, key: PublicKey, data: bytes, signature: Signature): + pass + @abstractmethod + async def aead_overhead(self) -> int: + pass + @abstractmethod + async def decrypt_aead(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret, associated_data: Optional[bytes]) -> bytes: + pass + @abstractmethod + async def encrypt_aead(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret, associated_data: Optional[bytes]) -> bytes: + pass + @abstractmethod + async def crypt_no_auth(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret) -> bytes: + pass + class VeilidAPI(ABC): @abstractmethod - def control(self, args: list[str]) -> str: + async def control(self, args: list[str]) -> str: pass @abstractmethod - def get_state(self) -> VeilidState: + async def get_state(self) -> VeilidState: + pass + @abstractmethod + async def attach(self): + pass + @abstractmethod + async def detach(self): + pass + @abstractmethod + async def new_private_route(self) -> NewPrivateRouteResult: + pass + @abstractmethod + async def new_custom_private_route(self, kinds: list[CryptoKind], stability: Stability, sequencing: Sequencing) -> NewPrivateRouteResult: + pass + @abstractmethod + async def import_remote_private_route(self, blob: bytes) -> RouteId: + pass + @abstractmethod + async def release_private_route(self, route_id: RouteId): + pass + @abstractmethod + async def app_call_reply(self, call_id: OperationId, message: bytes): + pass + @abstractmethod + async def new_routing_context(self) -> RoutingContext: + pass + @abstractmethod + async def open_table_db(self, name: str, column_count: int) -> TableDB: + pass + @abstractmethod + async def delete_table_db(self, name: str): + pass + @abstractmethod + async def get_crypto_system(self, kind: CryptoKind) -> CryptoSystem: + pass + @abstractmethod + async def best_crypto_system(self) -> CryptoSystem: + pass + @abstractmethod + async def verify_signatures(self, node_ids: list[TypedKey], data: bytes, signatures: list[TypedSignature]) -> list[TypedKey]: + pass + @abstractmethod + async def generate_signatures(self, data: bytes, key_pairs: list[TypedKeyPair]) -> list[TypedSignature]: + pass + @abstractmethod + async def generate_key_pair(self, kind: CryptoKind) -> list[TypedKeyPair]: + pass + @abstractmethod + async def now(self) -> Timestamp: + pass + @abstractmethod + async def debug(self, command: str) -> str: + pass + @abstractmethod + async def veilid_version_string(self) -> str: + pass + @abstractmethod + async def veilid_version(self) -> VeilidVersion: pass - \ No newline at end of file diff --git a/veilid-python/veilid_python/config.py b/veilid-python/veilid_python/config.py index ba520cc6..d44bc11a 100644 --- a/veilid-python/veilid_python/config.py +++ b/veilid-python/veilid_python/config.py @@ -1,5 +1,6 @@ from typing import Self, Optional from enum import StrEnum +from json import dumps class VeilidConfigLogLevel(StrEnum): OFF = 'Off' @@ -38,6 +39,8 @@ class VeilidConfigCapabilities: j['protocol_accept_ws'], j['protocol_connect_wss'], j['protocol_accept_wss']) + def to_json(self) -> dict: + return self.__dict__ class VeilidConfigProtectedStore: allow_insecure_fallback: bool @@ -61,6 +64,8 @@ class VeilidConfigProtectedStore: def from_json(j: dict) -> Self: return VeilidConfigProtectedStore(j['allow_insecure_fallback'], j['always_use_insecure_storage'], j['directory'], j['delete'], j['device_encryption_key_password'], j['new_device_encryption_key_password']) + def to_json(self) -> dict: + return self.__dict__ class VeilidConfigTableStore: directory: str @@ -73,6 +78,8 @@ class VeilidConfigTableStore: @staticmethod def from_json(j: dict) -> Self: return VeilidConfigTableStore(j['directory'], j['delete']) + def to_json(self) -> dict: + return self.__dict__ class VeilidConfigBlockStore: directory: str @@ -85,6 +92,8 @@ class VeilidConfigBlockStore: @staticmethod def from_json(j: dict) -> Self: return VeilidConfigBlockStore(j['directory'], j['delete']) + def to_json(self) -> dict: + return self.__dict__ class VeilidConfigRoutingTable: node_id: list[str] @@ -119,6 +128,8 @@ class VeilidConfigRoutingTable: j['limit_attached_strong'], j['limit_attached_good'], j['limit_attached_weak']) + def to_json(self) -> dict: + return self.__dict__ class VeilidConfigRPC: @@ -151,6 +162,286 @@ class VeilidConfigRPC: j['timeout_ms'], j['max_route_hop_count'], j['default_route_hop_count']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigDHT: + max_find_node_count: int + resolve_node_timeout_ms: int + resolve_node_count: int + resolve_node_fanout: int + get_value_timeout_ms: int + get_value_count: int + get_value_fanout: int + set_value_timeout_ms: int + set_value_count: int + set_value_fanout: int + min_peer_count: int + min_peer_refresh_time_ms: int + validate_dial_info_receipt_time_ms: int + local_subkey_cache_size: int + local_max_subkey_cache_memory_mb: int + remote_subkey_cache_size: int + remote_max_records: int + remote_max_subkey_cache_memory_mb: int + remote_max_storage_space_mb: int + + def __init__(self, max_find_node_count: int, resolve_node_timeout_ms: int, resolve_node_count: int, + resolve_node_fanout: int, get_value_timeout_ms: int, get_value_count: int, get_value_fanout: int, + set_value_timeout_ms: int, set_value_count: int, set_value_fanout: int, + min_peer_count: int, min_peer_refresh_time_ms: int, validate_dial_info_receipt_time_ms: int, + local_subkey_cache_size: int, local_max_subkey_cache_memory_mb: int, + remote_subkey_cache_size: int, remote_max_records: int, remote_max_subkey_cache_memory_mb: int, remote_max_storage_space_mb: int): + + self.max_find_node_count = max_find_node_count + self.resolve_node_timeout_ms =resolve_node_timeout_ms + self.resolve_node_count = resolve_node_count + self.resolve_node_fanout = resolve_node_fanout + self.get_value_timeout_ms = get_value_timeout_ms + self.get_value_count = get_value_count + self.get_value_fanout = get_value_fanout + self.set_value_timeout_ms = set_value_timeout_ms + self.set_value_count = set_value_count + self.set_value_fanout = set_value_fanout + self.min_peer_count = min_peer_count + self.min_peer_refresh_time_ms = min_peer_refresh_time_ms + self.validate_dial_info_receipt_time_ms = validate_dial_info_receipt_time_ms + self.local_subkey_cache_size = local_subkey_cache_size + self.local_max_subkey_cache_memory_mb = local_max_subkey_cache_memory_mb + self.remote_subkey_cache_size = remote_subkey_cache_size + self.remote_max_records = remote_max_records + self.remote_max_subkey_cache_memory_mb = remote_max_subkey_cache_memory_mb + self.remote_max_storage_space_mb = remote_max_storage_space_mb + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigDHT( + j['max_find_node_count'], + j['resolve_node_timeout_ms'], + j['resolve_node_count'], + j['resolve_node_fanout'], + j['get_value_timeout_ms'], + j['get_value_count'], + j['get_value_fanout'], + j['set_value_timeout_ms'], + j['set_value_count'], + j['set_value_fanout'], + j['min_peer_count'], + j['min_peer_refresh_time_ms'], + j['validate_dial_info_receipt_time_ms'], + j['local_subkey_cache_size'], + j['local_max_subkey_cache_memory_mb'], + j['remote_subkey_cache_size'], + j['remote_max_records'], + j['remote_max_subkey_cache_memory_mb'], + j['remote_max_storage_space_mb']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigTLS: + certificate_path: str + private_key_path: str + connection_initial_timeout_ms: int + + def __init__(self, certificate_path: str, private_key_path: str, connection_initial_timeout_ms: int): + self.certificate_path = certificate_path + self.private_key_path = private_key_path + self.connection_initial_timeout_ms = connection_initial_timeout_ms + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigTLS( + j['certificate_path'], + j['private_key_path'], + j['connection_initial_timeout_ms']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigHTTPS: + enabled: bool + listen_address: str + path: str + url: Optional[str] + + def __init__(self, enabled: bool, listen_address: str, path: str, url: Optional[str]): + self.enabled = enabled + self.listen_address = listen_address + self.path = path + self.url = url + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigHTTPS( + j['enabled'], + j['listen_address'], + j['path'], + j['url']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigHTTP: + enabled: bool + listen_address: str + path: str + url: Optional[str] + + def __init__(self, enabled: bool, listen_address: str, path: str, url: Optional[str]): + self.enabled = enabled + self.listen_address = listen_address + self.path = path + self.url = url + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigHTTP( + j['enabled'], + j['listen_address'], + j['path'], + j['url']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigApplication: + https: VeilidConfigHTTPS + http: VeilidConfigHTTP + + def __init__(self, https: VeilidConfigHTTPS, http: VeilidConfigHTTP): + self.https = https + self.http = http + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigApplication( + VeilidConfigHTTPS.from_json(j['https']), + VeilidConfigHTTP.from_json(j['http'])) + def to_json(self) -> dict: + return self.__dict__ + + +class VeilidConfigUDP: + enabled: bool + socket_pool_size: int + listen_address: str + public_address: Optional[str] + + def __init__(self, enabled: bool, socket_pool_size: int, listen_address: str, public_address: Optional[str]): + self.enabled = enabled + self.socket_pool_size = socket_pool_size + self.listen_address = listen_address + self.public_address = public_address + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigUDP( + j['enabled'], + j['socket_pool_size'], + j['listen_address'], + j['public_address']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigTCP: + connect: bool + listen: bool + max_connections: int + listen_address: str + public_address: Optional[str] + + def __init__(self, connect: bool, listen: bool, max_connections: int, listen_address: str, public_address: Optional[str]): + self.connect = connect + self.listen = listen + self.max_connections = max_connections + self.listen_address = listen_address + self.public_address = public_address + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigTCP( + j['connect'], + j['listen'], + j['max_connections'], + j['listen_address'], + j['public_address']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigWS: + connect: bool + listen: bool + max_connections: int + listen_address: str + path: str + url: Optional[str] + + def __init__(self, connect: bool, listen: bool, max_connections: int, listen_address: str, path: str, url: Optional[str]): + self.connect = connect + self.listen = listen + self.max_connections = max_connections + self.listen_address = listen_address + self.path = path + self.url = url + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigWS( + j['connect'], + j['listen'], + j['max_connections'], + j['listen_address'], + j['path'], + j['url']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigWSS: + connect: bool + listen: bool + max_connections: int + listen_address: str + path: str + url: Optional[str] + + def __init__(self, connect: bool, listen: bool, max_connections: int, listen_address: str, path: str, url: Optional[str]): + self.connect = connect + self.listen = listen + self.max_connections = max_connections + self.listen_address = listen_address + self.path = path + self.url = url + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigWSS( + j['connect'], + j['listen'], + j['max_connections'], + j['listen_address'], + j['path'], + j['url']) + def to_json(self) -> dict: + return self.__dict__ + +class VeilidConfigProtocol: + udp: VeilidConfigUDP + tcp: VeilidConfigTCP + ws: VeilidConfigWS + wss: VeilidConfigWSS + + def __init__(self, udp: VeilidConfigUDP, tcp: VeilidConfigTCP, ws: VeilidConfigWS, wss: VeilidConfigWSS): + self.udp = udp + self.tcp = tcp + self.ws = ws + self.wss = wss + + @staticmethod + def from_json(j: dict) -> Self: + return VeilidConfigProtocol( + VeilidConfigUDP.from_json(j['udp']), + VeilidConfigTCP.from_json(j['tcp']), + VeilidConfigWS.from_json(j['ws']), + VeilidConfigWSS.from_json(j['wss'])) + def to_json(self) -> dict: + return self.__dict__ class VeilidConfigNetwork: @@ -221,6 +512,8 @@ class VeilidConfigNetwork: VeilidConfigTLS.from_json(j['tls']), VeilidConfigApplication.from_json(j['application']), VeilidConfigProtocol.from_json(j['protocol'])) + def to_json(self) -> dict: + return self.__dict__ class VeilidConfig: program_name: str @@ -245,10 +538,14 @@ class VeilidConfig: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return VeilidConfig(j['program_name'], j['namespace'], VeilidConfigCapabilities.from_json(j['capabilities']), VeilidConfigProtectedStore.from_json(j['protected_store']), VeilidConfigTableStore.from_json(j['table_store']), VeilidConfigBlockStore.from_json(j['block_store']), VeilidConfigNetwork.from_json(j['network'])) - + def to_json(self) -> dict: + return self.__dict__ + + \ No newline at end of file diff --git a/veilid-python/veilid_python/error.py b/veilid-python/veilid_python/error.py index d3ded7da..07a242f4 100644 --- a/veilid-python/veilid_python/error.py +++ b/veilid-python/veilid_python/error.py @@ -1,4 +1,4 @@ -from typing import Self +from typing import Self, Any class VeilidAPIError(Exception): """Veilid API error exception base class""" @@ -131,3 +131,12 @@ class VeilidAPIErrorGeneric(VeilidAPIError): def __init__(self, message: str): super().__init__("Generic") self.message = message + + +def raise_api_result(api_result: dict) -> Any: + if "value" in api_result: + return api_result["value"] + elif "error" in api_result: + raise VeilidAPIError.from_json(api_result["error"]) + else: + raise ValueError("Invalid format for ApiResult") \ No newline at end of file diff --git a/veilid-python/veilid_python/json_api.py b/veilid-python/veilid_python/json_api.py new file mode 100644 index 00000000..b68d3a76 --- /dev/null +++ b/veilid-python/veilid_python/json_api.py @@ -0,0 +1,314 @@ +import json; +import asyncio; +from typing import Callable, Awaitable + +from .api import *; +from .state import * +from .config import * +from .error import * +from .types import * +from .operations import * + +class _JsonRoutingContext(RoutingContext): + api: VeilidAPI + rc_id: int + + def __init__(self, api: VeilidAPI, rc_id: int): + self.api = api + self.rc_id = rc_id + + async def with_privacy(self) -> Self: + new_rc_id = raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.WITH_PRIVACY)) + return _JsonRoutingContext(self.api, new_rc_id) + + async def with_custom_privacy(self, stability: Stability) -> Self: + new_rc_id = raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.WITH_CUSTOM_PRIVACY, + stability = stability)) + return _JsonRoutingContext(self.api, new_rc_id) + async def with_sequencing(self, sequencing: Sequencing) -> Self: + new_rc_id = raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.WITH_SEQUENCING, + sequencing = sequencing)) + return _JsonRoutingContext(self.api, new_rc_id) + async def app_call(self, target: TypedKey | RouteId, request: bytes) -> bytes: + return urlsafe_b64decode_no_pad(raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.APP_CALL, + target = target, + request = request))) + async def app_message(self, target: TypedKey | RouteId, message: bytes): + raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.APP_MESSAGE, + target = target, + message = message)) + async def create_dht_record(self, kind: CryptoKind, schema: DHTSchema) -> DHTRecordDescriptor: + return DHTRecordDescriptor.from_json(raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.CREATE_DHT_RECORD, + kind = kind, + schema = schema))) + async def open_dht_record(self, key: TypedKey, writer: Optional[KeyPair]) -> DHTRecordDescriptor: + pass + async def close_dht_record(self, key: TypedKey): + pass + async def delete_dht_record(self, key: TypedKey): + pass + async def get_dht_value(self, key: TypedKey, subkey: ValueSubkey, force_refresh: bool) -> Optional[ValueData]: + pass + async def set_dht_value(self, key: TypedKey, subkey: ValueSubkey, data: bytes) -> Optional[ValueData]: + pass + async def watch_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)], expiration: Timestamp, count: int) -> Timestamp: + pass + async def cancel_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)]) -> bool: + pass + + +class _JsonTableDBTransaction(TableDBTransaction): + async def commit(self): + pass + async def rollback(self): + pass + async def store(self, col: int, key: bytes, value: bytes): + pass + async def delete(self, col: int, key: bytes): + pass + +class _JsonTableDB(TableDB): + async def get_column_count(self) -> int: + pass + async def get_keys(self, col: int) -> list[str]: + pass + async def transact(self) -> TableDBTransaction: + pass + async def store(self, col: int, key: bytes, value: bytes): + pass + async def load(self, col: int, key: bytes) -> Optional[bytes]: + pass + async def delete(self, col: int, key: bytes) -> Optional[bytes]: + pass + +class _JsonCryptoSystem(CryptoSystem): + async def cached_dh(self, key: PublicKey, secret: SecretKey) -> SharedSecret: + pass + async def compute_dh(self, key: PublicKey, secret: SecretKey) -> SharedSecret: + pass + async def random_bytes(self, len: int) -> bytes: + pass + async def default_salt_length(self) -> int: + pass + async def hash_password(self, password: bytes, salt: bytes) -> str: + pass + async def verify_password(self, password: bytes, password_hash: str) -> bool: + pass + async def derive_shared_secret(self, password: bytes, salt: bytes) -> SharedSecret: + pass + async def random_nonce(self) -> Nonce: + pass + async def random_shared_secret(self) -> SharedSecret: + pass + async def generate_key_pair(self) -> KeyPair: + pass + async def generate_hash(self, data: bytes) -> HashDigest: + pass + async def validate_key_pair(self, key: PublicKey, secret: SecretKey) -> bool: + pass + async def validate_hash(self, data: bytes, hash_digest: HashDigest) -> bool: + pass + async def distance(self, key1: CryptoKey, key2: CryptoKey) -> CryptoKeyDistance: + pass + async def sign(self, key: PublicKey, secret: SecretKey, data: bytes) -> Signature: + pass + async def verify(self, key: PublicKey, data: bytes, signature: Signature): + pass + async def aead_overhead(self) -> int: + pass + async def decrypt_aead(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret, associated_data: Optional[bytes]) -> bytes: + pass + async def encrypt_aead(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret, associated_data: Optional[bytes]) -> bytes: + pass + async def crypt_no_auth(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret) -> bytes: + pass + + +class _JsonVeilidAPI(VeilidAPI): + reader: asyncio.StreamReader + writer: asyncio.StreamWriter + update_callback: Callable[[VeilidUpdate], Awaitable] + handle_recv_messages_task: Optional[asyncio.Task] + # Shared Mutable State + lock: asyncio.Lock + next_id: int + in_flight_requests: dict + + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, update_callback: Callable[[VeilidUpdate], Awaitable]): + self.reader = reader + self.writer = writer + self.update_callback = update_callback + self.handle_recv_messages_task = None + self.lock = asyncio.Lock() + self.next_id = 1 + self.in_flight_requests = dict() + + @staticmethod + async def connect(host: str, port: int, update_callback: Callable[[VeilidUpdate], Awaitable]) -> Self: + reader, writer = await asyncio.open_connection(host, port) + veilid_api = _JsonVeilidAPI(reader, writer, update_callback) + veilid_api.handle_recv_messages_task = asyncio.create_task(veilid_api.handle_recv_messages(), name = "JsonVeilidAPI.handle_recv_messages") + return veilid_api + + async def handle_recv_message_response(self, j: dict): + id = j['id'] + await self.lock.acquire() + try: + # Get and remove the in-flight request + reqfuture = self.in_flight_requests.pop(id, None) + finally: + self.lock.release() + # Resolve the request's future to the response json + reqfuture.set_result(j) + + async def handle_recv_messages(self): + # Read lines until we're done + try: + while True: + linebytes = await self.reader.readline() + if not linebytes.endswith(b'\n'): + break + + # Parse line as ndjson + j = json.loads(linebytes.strip()) + + # Process the message + if j['type'] == "Response": + await self.handle_recv_message_response(j) + elif j['type'] == "Update": + await self.update_callback(VeilidUpdate.from_json(j)) + except: + pass + finally: + self.reader = None + self.writer.close() + await self.writer.wait_closed() + self.writer = None + + async def allocate_request_future(self, id: int) -> asyncio.Future: + reqfuture = asyncio.get_running_loop().create_future() + + await self.lock.acquire() + try: + self.in_flight_requests[id] = reqfuture + finally: + self.lock.release() + + return reqfuture + + async def cancel_request_future(self, id: int): + await self.lock.acquire() + try: + reqfuture = self.in_flight_requests.pop(id, None) + reqfuture.cancel() + finally: + self.lock.release() + + async def send_ndjson_request(self, op: Operation, **kwargs) -> dict: + + # Get next id + await self.lock.acquire() + try: + id = self.next_id + self.next_id += 1 + finally: + self.lock.release() + + # Make NDJSON string for request + req = { "id": id, "op": op } + for k, v in kwargs.items(): + setattr(req, k, v) + reqstr = VeilidJSONEncoder.dumps(req) + "\n" + reqbytes = reqstr.encode() + + # Allocate future for request + reqfuture = await self.allocate_request_future(id) + + # Send to socket + try: + self.writer.write(reqbytes) + await self.writer.drain() + finally: + # Send failed, release future + self.cancel_request_future(id) + + # Wait for response + response = await reqfuture + + return response + + async def control(self, args: list[str]) -> str: + return raise_api_result(await self.send_ndjson_request(Operation.CONTROL, args = args)) + async def get_state(self) -> VeilidState: + return VeilidState.from_json(raise_api_result(await self.send_ndjson_request(Operation.GET_STATE))) + async def attach(self): + raise_api_result(await self.send_ndjson_request(Operation.ATTACH)) + async def detach(self): + raise_api_result(await self.send_ndjson_request(Operation.DETACH)) + async def new_private_route(self) -> NewPrivateRouteResult: + return NewPrivateRouteResult.from_json(raise_api_result(await self.send_ndjson_request(Operation.NEW_PRIVATE_ROUTE))) + async def new_custom_private_route(self, kinds: list[CryptoKind], stability: Stability, sequencing: Sequencing) -> NewPrivateRouteResult: + return NewPrivateRouteResult.from_json(raise_api_result( + await self.send_ndjson_request(Operation.NEW_CUSTOM_PRIVATE_ROUTE, + kinds = kinds, + stability = stability, + sequencing = sequencing) + )) + async def import_remote_private_route(self, blob: bytes) -> RouteId: + return RouteId(raise_api_result( + await self.send_ndjson_request(Operation.IMPORT_REMOTE_PRIVATE_ROUTE, + blob = blob) + )) + async def release_private_route(self, route_id: RouteId): + raise_api_result( + await self.send_ndjson_request(Operation.RELEASE_PRIVATE_ROUTE, + route_id = route_id) + ) + async def app_call_reply(self, call_id: OperationId, message: bytes): + raise_api_result( + await self.send_ndjson_request(Operation.APP_CALL_REPLY, + call_id = call_id, + message = message) + ) + async def new_routing_context(self) -> RoutingContext: + rc_id = raise_api_result(await self.send_ndjson_request(Operation.NEW_ROUTING_CONTEXT)) + return RoutingContext(self, rc_id) + + async def open_table_db(self, name: str, column_count: int) -> TableDB: + pass + async def delete_table_db(self, name: str): + pass + async def get_crypto_system(self, kind: CryptoKind) -> CryptoSystem: + pass + async def best_crypto_system(self) -> CryptoSystem: + pass + async def verify_signatures(self, node_ids: list[TypedKey], data: bytes, signatures: list[TypedSignature]) -> list[TypedKey]: + pass + async def generate_signatures(self, data: bytes, key_pairs: list[TypedKeyPair]) -> list[TypedSignature]: + pass + async def generate_key_pair(self, kind: CryptoKind) -> list[TypedKeyPair]: + pass + async def now(self) -> Timestamp: + pass + async def debug(self, command: str) -> str: + pass + async def veilid_version_string(self) -> str: + pass + async def veilid_version(self) -> VeilidVersion: + pass + + +def json_api_connect(host:str, port:int) -> VeilidAPI: + return _JsonVeilidAPI.connect(host, port) \ No newline at end of file diff --git a/veilid-python/veilid_python/operations.py b/veilid-python/veilid_python/operations.py new file mode 100644 index 00000000..20dd11eb --- /dev/null +++ b/veilid-python/veilid_python/operations.py @@ -0,0 +1,91 @@ +from enum import StrEnum +from typing import Self + +class Operation(StrEnum): + CONTROL = "Control" + GET_STATE = "GetState" + ATTACH = "Attach" + DETACH = "Detach" + NEW_PRIVATE_ROUTE = "NewPrivateRoute" + NEW_CUSTOM_PRIVATE_ROUTE = "NewCustomPrivateRoute" + IMPORT_REMOTE_PRIVATE_ROUTE = "ImportRemotePrivateRoute" + RELEASE_PRIVATE_ROUTE = "ReleasePrivateRoute" + APP_CALL_REPLY = "AppCallReply" + NEW_ROUTING_CONTEXT = "NewRoutingContext" + ROUTING_CONTEXT = "RoutingContext" + OPEN_TABLE_DB = "OpenTableDb" + DELETE_TABLE_DB = "DeleteTableDb" + TABLE_DB = "TableDb" + TABLE_DB_TRANSACTION = "TableDbTransaction" + GET_CRYPTO_SYSTEM = "GetCryptoSystem" + BEST_CRYPTO_SYSTEM = "BestCryptoSystem" + CRYPTO_SYSTEM = "CryptoSystem" + VERIFY_SIGNATURES = "VerifySignatures" + GENERATE_SIGNATURES = "GenerateSignatures" + GENERATE_KEY_PAIR = "GenerateKeyPair" + NOW = "Now" + DEBUG = "Debug" + VEILID_VERSION_STRING = "VeilidVersionString" + VEILID_VERSION = "VeilidVersion" + +class RoutingContextOperation(StrEnum): + INVALID_ID = "InvalidId" + RELEASE = "Release" + WITH_PRIVACY = "WithPrivacy" + WITH_CUSTOM_PRIVACY = "WithCustomPrivacy" + WITH_SEQUENCING = "WithSequencing" + APP_CALL = "AppCall" + APP_MESSAGE = "AppMessage" + CREATE_DHT_RECORD = "CreateDhtRecord" + OPEN_DHT_RECORD = "OpenDhtRecord" + CLOSE_DHT_RECORD = "CloseDhtRecord" + DELETE_DHT_RECORD = "DeleteDhtRecord" + GET_DHT_VALUE = "GetDhtValue" + SET_DHT_VALUE = "SetDhtValue" + WATCH_DHT_VALUES = "WatchDhtValues" + CANCEL_DHT_WATCH = "CancelDhtWatch" + +class TableDbOperation(StrEnum): + INVALID_ID = "InvalidId" + RELEASE = "Release" + GET_COLUMN_COUNT = "GetColumnCount" + GET_KEYS = "GetKeys" + TRANSACT = "Transact" + STORE = "Store" + LOAD = "Load" + DELETE = "Delete" + +class TableDBTransactionOperation(StrEnum): + INVALID_ID = "InvalidId" + COMMIT = "Commit" + ROLLBACK = "Rollback" + STORE = "Store" + DELETE = "Delete" + +class CryptoSystemOperation(StrEnum): + INVALID_ID = "InvalidId" + RELEASE = "Release" + CACHED_DH = "CachedDh" + COMPUTE_DH = "ComputeDh" + RANDOM_BYTES = "RandomBytes" + DEFAULT_SALT_LENGTH = "DefaultSaltLength" + HASH_PASSWORD = "HashPassword" + VERIFY_PASSWORD = "VerifyPassword" + DERIVE_SHARED_SECRET = "DeriveSharedSecret" + RANDOM_NONCE = "RandomNonce" + RANDOM_SHARED_SECRET = "RandomSharedSecret" + GENERATE_KEY_PAIR = "GenerateKeyPair" + GENERATE_HASH = "GenerateHash" + VALIDATE_KEY_PAIR = "ValidateKeyPair" + VALIDATE_HASH = "ValidateHash" + DISTANCE = "Distance" + SIGN = "Sign" + VERIFY = "Verify" + AEAD_OVERHEAD = "AeadOverhead" + DECRYPT_AEAD = "DecryptAead" + ENCRYPT_AEAD = "EncryptAead" + CRYPT_NO_AUTH = "CryptNoAuth" + +class RecvMessageType(StrEnum): + RESPONSE = "Response" + UPDATE = "Update" diff --git a/veilid-python/veilid_python/request.py b/veilid-python/veilid_python/request.py deleted file mode 100644 index 19d59ae2..00000000 --- a/veilid-python/veilid_python/request.py +++ /dev/null @@ -1,4 +0,0 @@ -class Request: - - def __init__(self, id: int): - self.id = id \ No newline at end of file diff --git a/veilid-python/veilid_python/state.py b/veilid-python/veilid_python/state.py index f5ac2abb..a5fd9cd8 100644 --- a/veilid-python/veilid_python/state.py +++ b/veilid-python/veilid_python/state.py @@ -1,7 +1,8 @@ from typing import Self, Optional from enum import StrEnum -from .types import Timestamp, TimestampDuration, ByteCount -from .config import VeilidConfig + +from .types import * +from .config import * class AttachmentState(StrEnum): DETACHED = 'Detached' @@ -25,6 +26,7 @@ class VeilidStateAttachment: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return VeilidStateAttachment( AttachmentState(j['state']), j['public_internet_ready'], @@ -54,6 +56,7 @@ class RPCStats: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return RPCStats( j['messages_sent'], j['messages_rcvd'], @@ -76,6 +79,7 @@ class LatencyStats: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return LatencyStats( TimestampDuration(j['fastest']), TimestampDuration(j['average']), @@ -96,6 +100,7 @@ class TransferStats: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return TransferStats( ByteCount(j['total']), ByteCount(j['maximum']), @@ -113,6 +118,7 @@ class TransferStatsDownUp: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return TransferStatsDownUp( TransferStats.from_json(j['down']), TransferStats.from_json(j['up'])) @@ -131,6 +137,7 @@ class PeerStats: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return PeerStats( j['time_added'], RPCStats.from_json(j['rpc_stats']), @@ -149,6 +156,7 @@ class PeerTableData: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return PeerTableData( j['node_ids'], j['peer_address'], @@ -168,6 +176,7 @@ class VeilidStateNetwork: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return VeilidStateNetwork( j['started'], ByteCount(j['bps_down']), @@ -182,10 +191,10 @@ class VeilidStateConfig: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return VeilidStateConfig( j['config']) - class VeilidState: attachment: VeilidStateAttachment network: VeilidStateNetwork @@ -198,8 +207,143 @@ class VeilidState: @staticmethod def from_json(j: dict) -> Self: + '''JSON object hook''' return VeilidState( VeilidStateAttachment.from_json(j['attachment']), VeilidStateNetwork.from_json(j['network']), VeilidStateConfig.from_json(j['config'])) +class VeilidLog: + log_level: VeilidLogLevel + message: str + backtrace: Optional[str] + + def __init__(self, log_level: VeilidLogLevel, message: str, backtrace: Optional[str]): + self.log_level = log_level + self.message = message + self.backtrace = backtrace + + @staticmethod + def from_json(j: dict) -> Self: + '''JSON object hook''' + return VeilidLog( + VeilidLogLevel(j['attachment']), + j['message'], + j['backtrace']) + +class VeilidAppMessage: + sender: Optional[TypedKey] + message: bytes + + def __init__(self, sender: Optional[TypedKey], message: bytes): + self.sender = sender + self.message = message + + @staticmethod + def from_json(j: dict) -> Self: + '''JSON object hook''' + return VeilidAppMessage( + None if j['sender'] is None else TypedKey(j['sender']), + urlsafe_b64decode_no_pad(j['message'])) + +class VeilidAppCall: + sender: Optional[TypedKey] + message: bytes + operation_id: str + + def __init__(self, sender: Optional[TypedKey], message: bytes, operation_id: str): + self.sender = sender + self.message = message + self.operation_id = operation_id + + @staticmethod + def from_json(j: dict) -> Self: + '''JSON object hook''' + return VeilidAppCall( + None if j['sender'] is None else TypedKey(j['sender']), + urlsafe_b64decode_no_pad(j['message']), + j['operation_id']) + +class VeilidRouteChange: + dead_routes: list[RouteId] + dead_remote_routes: list[RouteId] + + def __init__(self, dead_routes: list[RouteId], dead_remote_routes: list[RouteId]): + self.dead_routes = dead_routes + self.dead_remote_routes = dead_remote_routes + + @staticmethod + def from_json(j: dict) -> Self: + '''JSON object hook''' + return VeilidRouteChange( + map(lambda x: RouteId(x), j['dead_routes']), + map(lambda x: RouteId(x), j['dead_remote_routes'])) + +class VeilidValueChange: + key: TypedKey + subkeys: list[ValueSubkey] + count: int + value: ValueData + + def __init__(self, key: TypedKey, subkeys: list[ValueSubkey], count: int, value: ValueData): + self.key = key + self.subkeys = subkeys + self.count = count + self.value = value + + @staticmethod + def from_json(j: dict) -> Self: + '''JSON object hook''' + return VeilidValueChange( + TypedKey(j['key']), + map(lambda x: ValueSubkey(x), j['subkeys']), + j['count'], + ValueData.from_json(j['value'])) + + +class VeilidUpdateKind(StrEnum): + LOG = "Log" + APP_MESSAGE = "AppMessage" + APP_CALL = "AppCall" + ATTACHMENT = "Attachment" + NETWORK = "Network" + CONFIG = "Config" + ROUTE_CHANGE = "RouteChange" + VALUE_CHANGE = "ValueChange" + SHUTDOWN = "Shutdown" + +class VeilidUpdate: + kind: VeilidUpdateKind + detail: Optional[VeilidLog | VeilidAppMessage | VeilidAppCall | VeilidStateAttachment | VeilidStateNetwork | VeilidStateConfig | VeilidRouteChange | VeilidValueChange] + + def __init__(self, kind: VeilidUpdateKind, detail: Optional[VeilidLog | VeilidAppMessage | VeilidAppCall | VeilidStateAttachment | VeilidStateNetwork | VeilidStateConfig | VeilidRouteChange | VeilidValueChange]): + self.kind = kind + self.detail = detail + + @staticmethod + def from_json(j: dict) -> Self: + '''JSON object hook''' + kind = VeilidUpdateKind(j['kind']) + detail = None + match kind: + case VeilidUpdateKind.LOG: + detail = VeilidLog.from_json(j) + case VeilidUpdateKind.APP_MESSAGE: + detail = VeilidAppMessage.from_json(j) + case VeilidUpdateKind.APP_CALL: + detail = VeilidAppCall.from_json(j) + case VeilidUpdateKind.ATTACHMENT: + detail = VeilidStateAttachment.from_json(j) + case VeilidUpdateKind.NETWORK: + detail = VeilidStateNetwork.from_json(j) + case VeilidUpdateKind.CONFIG: + detail = VeilidStateConfig.from_json(j) + case VeilidUpdateKind.ROUTE_CHANGE: + detail = VeilidRouteChange.from_json(j) + case VeilidUpdateKind.VALUE_CHANGE: + detail = VeilidValueChange.from_json(j) + case VeilidUpdateKind.SHUTDOWN: + detail = None + case _: + raise ValueError("Unknown VeilidUpdateKind") + diff --git a/veilid-python/veilid_python/types.py b/veilid-python/veilid_python/types.py index e1938e04..9b87670c 100644 --- a/veilid-python/veilid_python/types.py +++ b/veilid-python/veilid_python/types.py @@ -1,4 +1,38 @@ import time +import json +import base64 + +from enum import StrEnum +from typing import Self, Optional, Any + +def urlsafe_b64encode_no_pad(b: bytes) -> str: + """ + Removes any `=` used as padding from the encoded string. + """ + encoded = str(base64.urlsafe_b64encode(b)) + return encoded.rstrip("=") + + +def urlsafe_b64decode_no_pad(s: str) -> bytes: + """ + Adds back in the required padding before decoding. + """ + padding = 4 - (len(s) % 4) + string = string + ("=" * padding) + return base64.urlsafe_b64decode(s) + +class VeilidJSONEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, bytes): + return urlsafe_b64encode_no_pad(o) + if hasattr(o, "to_json") and callable(o.to_json): + return o.to_json() + return json.JSONEncoder.default(self, o) + + @staticmethod + def dumps(req: Any, *args, **kwargs) -> str: + return json.dumps(req, cls = VeilidJSONEncoder, *args, **kwargs) + class Timestamp(int): pass @@ -7,4 +41,195 @@ class TimestampDuration(int): pass class ByteCount(int): - pass \ No newline at end of file + pass + +class OperationId(int): + pass + +class RouteId(str): + pass + +class TypedKey(str): + pass + +class TypedKeyPair(str): + pass + +class TypedSignature(str): + pass + +class CryptoKey(str): + pass + +class CryptoKeyDistance(str): + pass + +class PublicKey(CryptoKey): + pass + +class SecretKey(CryptoKey): + pass + +class SharedSecret(CryptoKey): + pass + +class Signature(str): + pass + +class Nonce(str): + pass + +class KeyPair(str): + pass + +class HashDigest(CryptoKey): + pass + +class ValueSubkey(int): + pass + +class ValueSeqNum(int): + pass + +class VeilidVersion: + _major: int + _minor: int + _patch: int + def __init__(self, major: int, minor: int, patch: int): + self._major = major + self._minor = minor + self._patch = patch + @property + def major(self): + return self._major + @property + def minor(self): + return self._minor + @property + def patch(self): + return self._patch + +class VeilidLogLevel(StrEnum): + ERROR = 'Error' + WARN = 'Warn' + INFO = 'Info' + DEBUG = 'Debug' + TRACE = 'Trace' + +class NewPrivateRouteResult: + route_id: RouteId + blob: bytes + + def __init__(self, route_id: RouteId, blob: bytes): + self.route_id = route_id + self.blob = blob + + @staticmethod + def from_json(j: dict) -> Self: + return NewPrivateRouteResult( + RouteId(j['route_id']), + urlsafe_b64decode_no_pad(j['blob'])) + +class CryptoKind(StrEnum): + CRYPTO_KIND_NONE = "NONE" + CRYPTO_KIND_VLD0 = "VLD0" + +class Stability(StrEnum): + LOW_LATENCY = "LowLatency" + RELIABLE = "Reliable" + +class Sequencing(StrEnum): + NO_PREFERENCE = "NoPreference" + PREFER_ORDERED = "PreferOrdered" + ENSURE_ORDERED = "EnsureOrdered" + +class DHTSchemaKind(StrEnum): + DFLT = "DFLT" + SMPL = "SMPL" + +class DHTSchemaSMPLMember: + m_key: PublicKey + m_cnt: int + def __init__(self, m_key: PublicKey, m_cnt: int): + self.m_key = m_key + self.m_cnt = m_cnt + @staticmethod + def from_json(j: dict) -> Self: + return DHTSchemaSMPLMember( + PublicKey(j['m_key']), + j['m_cnt']) + def to_json(self) -> dict: + return self.__dict__ + +class DHTSchema: + kind: DHTSchemaKind + + def __init__(self, kind: DHTSchemaKind, **kwargs): + self.kind = kind + for k, v in kwargs.items(): + setattr(self, k, v) + + @staticmethod + def dflt(o_cnt: int) -> Self: + Self(DHTSchemaKind.DFLT, o_cnt = o_cnt) + + @staticmethod + def smpl(o_cnt: int, members: list[DHTSchemaSMPLMember]) -> Self: + Self(DHTSchemaKind.SMPL, o_cnt = o_cnt, members = members) + + @staticmethod + def from_json(j: dict) -> Self: + if DHTSchemaKind(j['kind']) == DHTSchemaKind.DFLT: + return DHTSchema.dflt(j['o_cnt']) + if DHTSchemaKind(j['kind']) == DHTSchemaKind.SMPL: + return DHTSchema.smpl( + j['o_cnt'], + map(lambda x: DHTSchemaSMPLMember.from_json(x), j['members'])) + raise Exception("Unknown DHTSchema kind", j['kind']) + + def to_json(self) -> dict: + return self.__dict__ + +class DHTRecordDescriptor: + key: TypedKey + owner: PublicKey + owner_secret: Optional[SecretKey] + schema: DHTSchema + + def __init__(self, key: TypedKey, owner: PublicKey, owner_secret: Optional[SecretKey], schema: DHTSchema): + self.key = key + self.owner = owner + self.owner_secret = owner_secret + self.schema = schema + + @staticmethod + def from_json(j: dict) -> Self: + DHTRecordDescriptor( + TypedKey(j['key']), + PublicKey(j['owner']), + None if j['owner_secret'] is None else SecretKey(j['owner_secret']), + DHTSchema.from_json(j['schema'])) + + def to_json(self) -> dict: + return self.__dict__ + +class ValueData: + seq: ValueSeqNum + data: bytes + writer: PublicKey + + def __init__(self, seq: ValueSeqNum, data: bytes, writer: PublicKey): + self.seq = seq + self.data = data + self.writer = writer + + @staticmethod + def from_json(j: dict) -> Self: + DHTRecordDescriptor( + ValueSeqNum(j['seq']), + urlsafe_b64decode_no_pad(j['data']), + PublicKey(j['writer'])) + + def to_json(self) -> dict: + return self.__dict__ + From d053e93e7292c269d0ef57f1fc0924f2849717fd Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 14 Jun 2023 16:33:14 -0400 Subject: [PATCH 15/23] python work --- veilid-core/proto/veilid.capnp | 2 +- veilid-python/README.md | 7 +- veilid-python/poetry.lock | 860 +----------------- veilid-python/pyproject.toml | 7 +- veilid-python/tests/test_basic.py | 44 + veilid-python/update_schema.sh | 4 +- veilid-python/veilid_python/__init__.py | 6 + veilid-python/veilid_python/api.py | 12 +- veilid-python/veilid_python/json_api.py | 591 +++++++++--- veilid-python/veilid_python/operations.py | 2 +- .../schema/RecvMessage.json | 0 .../{ => veilid_python}/schema/Request.json | 0 veilid-python/veilid_python/types.py | 160 +++- 13 files changed, 659 insertions(+), 1036 deletions(-) create mode 100644 veilid-python/tests/test_basic.py rename veilid-python/{ => veilid_python}/schema/RecvMessage.json (100%) rename veilid-python/{ => veilid_python}/schema/Request.json (100%) diff --git a/veilid-core/proto/veilid.capnp b/veilid-core/proto/veilid.capnp index 59af6722..3d813450 100644 --- a/veilid-core/proto/veilid.capnp +++ b/veilid-core/proto/veilid.capnp @@ -230,7 +230,7 @@ struct NodeInfo @0xe125d847e3f9f419 { outboundProtocols @1 :ProtocolTypeSet; # protocols that can go outbound addressTypes @2 :AddressTypeSet; # address types supported envelopeSupport @3 :List(UInt8); # supported rpc envelope/receipt versions - cryptoSupport @4 :List( ); # cryptography systems supported + cryptoSupport @4 :List(CryptoKind); # cryptography systems supported dialInfoDetailList @5 :List(DialInfoDetail); # inbound dial info details for this node } diff --git a/veilid-python/README.md b/veilid-python/README.md index dabe4ad4..868cd237 100644 --- a/veilid-python/README.md +++ b/veilid-python/README.md @@ -14,7 +14,12 @@ pip3 install veilid_python ## Development -To update schema for validation: +To run tests: +``` +poetry run pytest +``` + +To update schema for validation with the latest copy from a running `veilid-server`: ``` ./update_schema.sh ``` diff --git a/veilid-python/poetry.lock b/veilid-python/poetry.lock index 9e4d4a9c..85a37c9b 100644 --- a/veilid-python/poetry.lock +++ b/veilid-python/poetry.lock @@ -1,19 +1,5 @@ # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. -[[package]] -name = "argcomplete" -version = "3.1.0" -description = "Bash tab completion for argparse" -optional = false -python-versions = ">=3.6" -files = [ - {file = "argcomplete-3.1.0-py3-none-any.whl", hash = "sha256:faea1cba05479b66628d7b216fcd23f2b9ad0117b13280aad8ae0b30872cd1f5"}, - {file = "argcomplete-3.1.0.tar.gz", hash = "sha256:3a81445fa51c9875c36789ae5386d5e95c5c43d503963a0029a747aea74de1e3"}, -] - -[package.extras] -test = ["coverage", "mypy", "pexpect", "ruff", "wheel"] - [[package]] name = "attrs" version = "23.1.0" @@ -32,173 +18,6 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib- tests = ["attrs[tests-no-zope]", "zope-interface"] tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -[[package]] -name = "black" -version = "23.3.0" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.7" -files = [ - {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, - {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, - {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, - {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, - {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, - {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, - {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, - {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, - {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, - {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, - {file = "black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, - {file = "black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, - {file = "black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, - {file = "black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, - {file = "black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, - {file = "black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, - {file = "black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, - {file = "black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, - {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, - {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, - {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, - {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, - {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, - {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, - {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -packaging = ">=22.0" -pathspec = ">=0.9.0" -platformdirs = ">=2" - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - -[[package]] -name = "certifi" -version = "2023.5.7" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.6" -files = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, -] - -[[package]] -name = "chardet" -version = "5.1.0" -description = "Universal encoding detector for Python 3" -optional = false -python-versions = ">=3.7" -files = [ - {file = "chardet-5.1.0-py3-none-any.whl", hash = "sha256:362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"}, - {file = "chardet-5.1.0.tar.gz", hash = "sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.1.0" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, -] - -[[package]] -name = "click" -version = "8.1.3" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - [[package]] name = "colorama" version = "0.4.6" @@ -211,161 +30,14 @@ files = [ ] [[package]] -name = "datamodel-code-generator" -version = "0.20.0" -description = "Datamodel Code Generator" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "datamodel_code_generator-0.20.0-py3-none-any.whl", hash = "sha256:fdb8dc18fd3a5d2c92e5e3ac473c5a48f3540066941e56fd947d2975cfebd281"}, - {file = "datamodel_code_generator-0.20.0.tar.gz", hash = "sha256:84dc7d6ae64ca67834b414c107adf3510f46a474ac21467a683aca0a4a8f0806"}, -] - -[package.dependencies] -argcomplete = ">=1.10,<4.0" -black = ">=19.10b0" -genson = ">=1.2.1,<2.0" -inflect = ">=4.1.0,<6.0" -isort = ">=4.3.21,<6.0" -jinja2 = ">=2.10.1,<4.0" -openapi-spec-validator = ">=0.2.8,<=0.5.2" -packaging = "*" -prance = ">=0.18.2,<1.0" -pydantic = {version = ">=1.10.0,<2.0.0", extras = ["email"], markers = "python_version >= \"3.11\" and python_version < \"4.0\""} -PySnooper = ">=0.4.1,<2.0.0" -toml = ">=0.10.0,<1.0.0" - -[package.extras] -http = ["httpx"] - -[[package]] -name = "dnspython" -version = "2.3.0" -description = "DNS toolkit" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, - {file = "dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, -] - -[package.extras] -curio = ["curio (>=1.2,<2.0)", "sniffio (>=1.1,<2.0)"] -dnssec = ["cryptography (>=2.6,<40.0)"] -doh = ["h2 (>=4.1.0)", "httpx (>=0.21.1)", "requests (>=2.23.0,<3.0.0)", "requests-toolbelt (>=0.9.1,<0.11.0)"] -doq = ["aioquic (>=0.9.20)"] -idna = ["idna (>=2.1,<4.0)"] -trio = ["trio (>=0.14,<0.23)"] -wmi = ["wmi (>=1.5.1,<2.0.0)"] - -[[package]] -name = "email-validator" -version = "2.0.0.post2" -description = "A robust email address syntax and deliverability validation library." +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" files = [ - {file = "email_validator-2.0.0.post2-py3-none-any.whl", hash = "sha256:2466ba57cda361fb7309fd3d5a225723c788ca4bbad32a0ebd5373b99730285c"}, - {file = "email_validator-2.0.0.post2.tar.gz", hash = "sha256:1ff6e86044200c56ae23595695c54e9614f4a9551e0e393614f764860b3d7900"}, -] - -[package.dependencies] -dnspython = ">=2.0.0" -idna = ">=2.0.0" - -[[package]] -name = "genson" -version = "1.2.2" -description = "GenSON is a powerful, user-friendly JSON Schema generator." -optional = false -python-versions = "*" -files = [ - {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, -] - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.5" -files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] - -[[package]] -name = "inflect" -version = "5.6.2" -description = "Correctly generate plurals, singular nouns, ordinals, indefinite articles; convert numbers to words" -optional = false -python-versions = ">=3.7" -files = [ - {file = "inflect-5.6.2-py3-none-any.whl", hash = "sha256:b45d91a4a28a4e617ff1821117439b06eaa86e2a4573154af0149e9be6687238"}, - {file = "inflect-5.6.2.tar.gz", hash = "sha256:aadc7ed73928f5e014129794bbac03058cca35d0a973a5fc4eb45c7fa26005f9"}, -] - -[package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["pygments", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "isort" -version = "5.12.0" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, - {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, -] - -[package.extras] -colors = ["colorama (>=0.4.3)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "json-ref-dict" -version = "0.7.1" -description = "Python dict-like object which abstracts resolution of JSONSchema references" -optional = false -python-versions = "*" -files = [ - {file = "json-ref-dict-0.7.1.tar.gz", hash = "sha256:b1a0f7acad8a022bdd5e2f1a8943fe4dde6f429ec40966e29c992ee8f6349b72"}, - {file = "json_ref_dict-0.7.1-py3-none-any.whl", hash = "sha256:0cfdd3967594bc24e74a9b37f17cbda597f5a38755c35ace52fef92b7be266cf"}, -] - -[package.dependencies] -jsonpointer = ">=2.0,<3.0" - -[[package]] -name = "jsonpointer" -version = "2.3" -description = "Identify specific nodes in a JSON document (RFC 6901)" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "jsonpointer-2.3-py2.py3-none-any.whl", hash = "sha256:51801e558539b4e9cd268638c078c6c5746c9ac96bc38152d443400e4f3793e9"}, - {file = "jsonpointer-2.3.tar.gz", hash = "sha256:97cba51526c829282218feb99dab1b1e6bdf8efd1c43dc9d57be093c0d69c99a"}, + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] [[package]] @@ -387,176 +59,6 @@ pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] -[[package]] -name = "jsonschema-spec" -version = "0.1.6" -description = "JSONSchema Spec with object-oriented paths" -optional = false -python-versions = ">=3.7.0,<4.0.0" -files = [ - {file = "jsonschema_spec-0.1.6-py3-none-any.whl", hash = "sha256:f2206d18c89d1824c1f775ba14ed039743b41a9167bd2c5bdb774b66b3ca0bbf"}, - {file = "jsonschema_spec-0.1.6.tar.gz", hash = "sha256:90215863b56e212086641956b20127ccbf6d8a3a38343dad01d6a74d19482f76"}, -] - -[package.dependencies] -jsonschema = ">=4.0.0,<4.18.0" -pathable = ">=0.4.1,<0.5.0" -PyYAML = ">=5.1" -requests = ">=2.31.0,<3.0.0" - -[[package]] -name = "lazy-object-proxy" -version = "1.9.0" -description = "A fast and thorough lazy object proxy." -optional = false -python-versions = ">=3.7" -files = [ - {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, -] - -[[package]] -name = "markupsafe" -version = "2.1.3" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, - {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, -] - -[[package]] -name = "mypy-extensions" -version = "1.0.0" -description = "Type system extensions for programs checked with the mypy type checker." -optional = false -python-versions = ">=3.5" -files = [ - {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, - {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, -] - -[[package]] -name = "openapi-schema-validator" -version = "0.4.4" -description = "OpenAPI schema validation for Python" -optional = false -python-versions = ">=3.7.0,<4.0.0" -files = [ - {file = "openapi_schema_validator-0.4.4-py3-none-any.whl", hash = "sha256:79f37f38ef9fd5206b924ed7a6f382cea7b649b3b56383c47f1906082b7b9015"}, - {file = "openapi_schema_validator-0.4.4.tar.gz", hash = "sha256:c573e2be2c783abae56c5a1486ab716ca96e09d1c3eab56020d1dc680aa57bf8"}, -] - -[package.dependencies] -jsonschema = ">=4.0.0,<4.18.0" -rfc3339-validator = "*" - -[package.extras] -docs = ["sphinx (>=5.3.0,<6.0.0)", "sphinx-immaterial (>=0.11.0,<0.12.0)"] - -[[package]] -name = "openapi-spec-validator" -version = "0.5.2" -description = "OpenAPI 2.0 (aka Swagger) and OpenAPI 3 spec validator" -optional = false -python-versions = ">=3.7.0,<4.0.0" -files = [ - {file = "openapi_spec_validator-0.5.2-py3-none-any.whl", hash = "sha256:1f8db08ecbcf4ec8c558d65b65b3b7b428f81da6642f2f163e992ae3e17b229c"}, - {file = "openapi_spec_validator-0.5.2.tar.gz", hash = "sha256:ebed7f1c567780859402ad64b128e17f519d15f605f1b41d1e9a4a7a1690be07"}, -] - -[package.dependencies] -jsonschema = ">=4.0.0,<5.0.0" -jsonschema-spec = ">=0.1.1,<0.2.0" -lazy-object-proxy = ">=1.7.1,<2.0.0" -openapi-schema-validator = ">=0.3.2,<0.5" - -[package.extras] -requests = ["requests"] - [[package]] name = "packaging" version = "23.1" @@ -569,120 +71,19 @@ files = [ ] [[package]] -name = "pathable" -version = "0.4.3" -description = "Object-oriented paths" +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7.0,<4.0.0" +python-versions = ">=3.6" files = [ - {file = "pathable-0.4.3-py3-none-any.whl", hash = "sha256:cdd7b1f9d7d5c8b8d3315dbf5a86b2596053ae845f056f57d97c0eefff84da14"}, - {file = "pathable-0.4.3.tar.gz", hash = "sha256:5c869d315be50776cc8a993f3af43e0c60dc01506b399643f919034ebf4cdcab"}, -] - -[[package]] -name = "pathspec" -version = "0.11.1" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, -] - -[[package]] -name = "platformdirs" -version = "3.5.3" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false -python-versions = ">=3.7" -files = [ - {file = "platformdirs-3.5.3-py3-none-any.whl", hash = "sha256:0ade98a4895e87dc51d47151f7d2ec290365a585151d97b4d8d6312ed6132fed"}, - {file = "platformdirs-3.5.3.tar.gz", hash = "sha256:e48fabd87db8f3a7df7150a4a5ea22c546ee8bc39bc2473244730d4b56d2cc4e"}, + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] [package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] - -[[package]] -name = "prance" -version = "0.22.2.22.0" -description = "Resolving Swagger/OpenAPI 2.0 and 3.0.0 Parser" -optional = false -python-versions = ">=3.8" -files = [ - {file = "prance-0.22.2.22.0-py3-none-any.whl", hash = "sha256:57deeb67b7e93ef27c1c17845bf3ccb4af288ccfb5748c7e01779c01a8507f27"}, - {file = "prance-0.22.2.22.0.tar.gz", hash = "sha256:9a83f8a4f5fe0f2d896d238d4bec6b5788b10b94155414b3d88c21c1579b85bf"}, -] - -[package.dependencies] -chardet = ">=3.0" -packaging = ">=21.3" -requests = ">=2.25" -"ruamel.yaml" = ">=0.17.10" -six = ">=1.15,<2.0" - -[package.extras] -cli = ["click (>=7.0)"] -dev = ["bumpversion (>=0.6)", "pytest (>=6.1)", "pytest-cov (>=2.11)", "sphinx (>=3.4)", "towncrier (>=19.2)", "tox (>=3.4)"] -flex = ["flex (>=6.13,<7.0)"] -icu = ["PyICU (>=2.4,<3.0)"] -osv = ["openapi-spec-validator (>=0.5.1,<0.6.0)"] -ssv = ["swagger-spec-validator (>=2.4,<3.0)"] - -[[package]] -name = "pydantic" -version = "1.10.9" -description = "Data validation and settings management using python type hints" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pydantic-1.10.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e692dec4a40bfb40ca530e07805b1208c1de071a18d26af4a2a0d79015b352ca"}, - {file = "pydantic-1.10.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c52eb595db83e189419bf337b59154bdcca642ee4b2a09e5d7797e41ace783f"}, - {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:939328fd539b8d0edf244327398a667b6b140afd3bf7e347cf9813c736211896"}, - {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b48d3d634bca23b172f47f2335c617d3fcb4b3ba18481c96b7943a4c634f5c8d"}, - {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f0b7628fb8efe60fe66fd4adadd7ad2304014770cdc1f4934db41fe46cc8825f"}, - {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e1aa5c2410769ca28aa9a7841b80d9d9a1c5f223928ca8bec7e7c9a34d26b1d4"}, - {file = "pydantic-1.10.9-cp310-cp310-win_amd64.whl", hash = "sha256:eec39224b2b2e861259d6f3c8b6290d4e0fbdce147adb797484a42278a1a486f"}, - {file = "pydantic-1.10.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d111a21bbbfd85c17248130deac02bbd9b5e20b303338e0dbe0faa78330e37e0"}, - {file = "pydantic-1.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e9aec8627a1a6823fc62fb96480abe3eb10168fd0d859ee3d3b395105ae19a7"}, - {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07293ab08e7b4d3c9d7de4949a0ea571f11e4557d19ea24dd3ae0c524c0c334d"}, - {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee829b86ce984261d99ff2fd6e88f2230068d96c2a582f29583ed602ef3fc2c"}, - {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4b466a23009ff5cdd7076eb56aca537c745ca491293cc38e72bf1e0e00de5b91"}, - {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7847ca62e581e6088d9000f3c497267868ca2fa89432714e21a4fb33a04d52e8"}, - {file = "pydantic-1.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:7845b31959468bc5b78d7b95ec52fe5be32b55d0d09983a877cca6aedc51068f"}, - {file = "pydantic-1.10.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:517a681919bf880ce1dac7e5bc0c3af1e58ba118fd774da2ffcd93c5f96eaece"}, - {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67195274fd27780f15c4c372f4ba9a5c02dad6d50647b917b6a92bf00b3d301a"}, - {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2196c06484da2b3fded1ab6dbe182bdabeb09f6318b7fdc412609ee2b564c49a"}, - {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6257bb45ad78abacda13f15bde5886efd6bf549dd71085e64b8dcf9919c38b60"}, - {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3283b574b01e8dbc982080d8287c968489d25329a463b29a90d4157de4f2baaf"}, - {file = "pydantic-1.10.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5f8bbaf4013b9a50e8100333cc4e3fa2f81214033e05ac5aa44fa24a98670a29"}, - {file = "pydantic-1.10.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9cd67fb763248cbe38f0593cd8611bfe4b8ad82acb3bdf2b0898c23415a1f82"}, - {file = "pydantic-1.10.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f50e1764ce9353be67267e7fd0da08349397c7db17a562ad036aa7c8f4adfdb6"}, - {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73ef93e5e1d3c8e83f1ff2e7fdd026d9e063c7e089394869a6e2985696693766"}, - {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:128d9453d92e6e81e881dd7e2484e08d8b164da5507f62d06ceecf84bf2e21d3"}, - {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ad428e92ab68798d9326bb3e5515bc927444a3d71a93b4a2ca02a8a5d795c572"}, - {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fab81a92f42d6d525dd47ced310b0c3e10c416bbfae5d59523e63ea22f82b31e"}, - {file = "pydantic-1.10.9-cp38-cp38-win_amd64.whl", hash = "sha256:963671eda0b6ba6926d8fc759e3e10335e1dc1b71ff2a43ed2efd6996634dafb"}, - {file = "pydantic-1.10.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:970b1bdc6243ef663ba5c7e36ac9ab1f2bfecb8ad297c9824b542d41a750b298"}, - {file = "pydantic-1.10.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7e1d5290044f620f80cf1c969c542a5468f3656de47b41aa78100c5baa2b8276"}, - {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83fcff3c7df7adff880622a98022626f4f6dbce6639a88a15a3ce0f96466cb60"}, - {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0da48717dc9495d3a8f215e0d012599db6b8092db02acac5e0d58a65248ec5bc"}, - {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0a2aabdc73c2a5960e87c3ffebca6ccde88665616d1fd6d3db3178ef427b267a"}, - {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9863b9420d99dfa9c064042304868e8ba08e89081428a1c471858aa2af6f57c4"}, - {file = "pydantic-1.10.9-cp39-cp39-win_amd64.whl", hash = "sha256:e7c9900b43ac14110efa977be3da28931ffc74c27e96ee89fbcaaf0b0fe338e1"}, - {file = "pydantic-1.10.9-py3-none-any.whl", hash = "sha256:6cafde02f6699ce4ff643417d1a9223716ec25e228ddc3b436fe7e2d25a1f305"}, - {file = "pydantic-1.10.9.tar.gz", hash = "sha256:95c70da2cd3b6ddf3b9645ecaa8d98f3d80c606624b6d245558d202cd23ea3be"}, -] - -[package.dependencies] -email-validator = {version = ">=1.0.3", optional = true, markers = "extra == \"email\""} -typing-extensions = ">=4.2.0" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "pyrsistent" @@ -721,239 +122,44 @@ files = [ ] [[package]] -name = "pysnooper" -version = "1.1.1" -description = "A poor man's debugger for Python." -optional = false -python-versions = "*" -files = [ - {file = "PySnooper-1.1.1-py2.py3-none-any.whl", hash = "sha256:378f13d731a3e04d3d0350e5f295bdd0f1b49fc8a8b8bf2067fe1e5290bd20be"}, - {file = "PySnooper-1.1.1.tar.gz", hash = "sha256:d17dc91cca1593c10230dce45e46b1d3ff0f8910f0c38e941edf6ba1260b3820"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pyyaml" -version = "5.4.1" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, - {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, - {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, - {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, - {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, - {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, - {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, - {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, - {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, -] - -[[package]] -name = "requests" -version = "2.31.0" -description = "Python HTTP for Humans." +name = "pytest" +version = "7.3.2" +description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, + {file = "pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, ] [package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] -name = "rfc3339-validator" -version = "0.1.4" -description = "A pure python RFC3339 validator" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"}, - {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"}, -] - -[package.dependencies] -six = "*" - -[[package]] -name = "ruamel-yaml" -version = "0.17.31" -description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -optional = false -python-versions = ">=3" -files = [ - {file = "ruamel.yaml-0.17.31-py3-none-any.whl", hash = "sha256:3cf153f0047ced526e723097ac615d3009371779432e304dbd5596b6f3a4c777"}, - {file = "ruamel.yaml-0.17.31.tar.gz", hash = "sha256:098ed1eb6d338a684891a72380277c1e6fc4d4ae0e120de9a447275056dda335"}, -] - -[package.dependencies] -"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.12\""} - -[package.extras] -docs = ["ryd"] -jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] - -[[package]] -name = "ruamel-yaml-clib" -version = "0.2.7" -description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -optional = false -python-versions = ">=3.5" -files = [ - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:370445fd795706fd291ab00c9df38a0caed0f17a6fb46b0f607668ecb16ce763"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win32.whl", hash = "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:40d030e2329ce5286d6b231b8726959ebbe0404c92f0a578c0e2482182e38282"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c3ca1fbba4ae962521e5eb66d72998b51f0f4d0f608d3c0347a48e1af262efa7"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3243f48ecd450eddadc2d11b5feb08aca941b5cd98c9b1db14b2fd128be8c697"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win32.whl", hash = "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf9a6bc4a0221538b1a7de3ed7bca4c93c02346853f44e1cd764be0023cd3640"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win32.whl", hash = "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5"}, - {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, -] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - -[[package]] -name = "statham-schema" -version = "0.14.0" -description = "Tools for generating Python models from JSON Schema documents." -optional = false -python-versions = "*" -files = [ - {file = "statham-schema-0.14.0.tar.gz", hash = "sha256:2570642c0c3f0864d86659cafa6293e011a9e113953376e7d79c4c28009c066a"}, - {file = "statham_schema-0.14.0-py3-none-any.whl", hash = "sha256:2e922c21f3ac2ebf0ac79a142df272bd30b0fd16c8235f6e2628240c7fa2bfd9"}, -] - -[package.dependencies] -json-ref-dict = ">=0.6.2,<0.8.0" -jsonpointer = ">=2.0,<3.0" -python-dateutil = ">=2.8,<3.0" -PyYAML = ">=5.4,<6.0" -typing-extensions = ">=3.7.0,<5.0.0" - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] - -[[package]] -name = "typing-extensions" -version = "4.6.3" -description = "Backported and Experimental Type Hints for Python 3.7+" +name = "pytest-asyncio" +version = "0.21.0" +description = "Pytest support for asyncio" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.6.3-py3-none-any.whl", hash = "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26"}, - {file = "typing_extensions-4.6.3.tar.gz", hash = "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5"}, + {file = "pytest-asyncio-0.21.0.tar.gz", hash = "sha256:2b38a496aef56f56b0e87557ec313e11e1ab9276fc3863f6a7be0f1d0e415e1b"}, + {file = "pytest_asyncio-0.21.0-py3-none-any.whl", hash = "sha256:f2b3366b7cd501a4056858bd39349d5af19742aed2d81660b7998b6341c7eb9c"}, ] -[[package]] -name = "urllib3" -version = "2.0.3" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.7" -files = [ - {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, - {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, -] +[package.dependencies] +pytest = ">=7.0.0" [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "f4368567ec647c3c626e34fb7b1115dca196b68a2290bce7f223260a24a9c54c" +content-hash = "03a349f63b3d28e64191b6dd845333914827806332b52f5c52ccbd2863c93b4b" diff --git a/veilid-python/pyproject.toml b/veilid-python/pyproject.toml index c4b15dd9..1eec17f4 100644 --- a/veilid-python/pyproject.toml +++ b/veilid-python/pyproject.toml @@ -6,14 +6,13 @@ authors = ["Christien Rioux "] readme = "README.md" packages = [{include = "veilid_python"}] -[tool.poetry.scripts] -update_bindings = "veilid_python.update_bindings:main" - [tool.poetry.dependencies] python = "^3.11" +jsonschema = "^4.17.3" [tool.poetry.group.dev.dependencies] -datamodel-code-generator = "^0.20.0" +pytest = "^7.3.2" +pytest-asyncio = "^0.21.0" [build-system] requires = ["poetry-core"] diff --git a/veilid-python/tests/test_basic.py b/veilid-python/tests/test_basic.py new file mode 100644 index 00000000..80443727 --- /dev/null +++ b/veilid-python/tests/test_basic.py @@ -0,0 +1,44 @@ +# Basic veilid_python tests + +import veilid_python +import pytest +import os + +pytest_plugins = ('pytest_asyncio',) + +################################################################## +VEILID_SERVER = os.getenv("VEILID_SERVER") +if VEILID_SERVER is not None: + vsparts = VEILID_SERVER.split(":") + VEILID_SERVER = vsparts[0] + if len(vsparts) == 2: + VEILID_SERVER_PORT = int(vsparts[1]) + else: + VEILID_SERVER_PORT = 5959 +else: + VEILID_SERVER = "localhost" + VEILID_SERVER_PORT = 5959 + +################################################################## + +async def _simple_update_callback(update): + print("VeilidUpdate: {}".format(update)) + +@pytest.mark.asyncio +async def test_connect(): + async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, _simple_update_callback) as api: + pass + +@pytest.mark.asyncio +async def test_fail_connect(): + with pytest.raises(Exception): + async with await veilid_python.json_api_connect("fuahwelifuh32luhwafluehawea", 1, _simple_update_callback) as api: + pass + +@pytest.mark.asyncio +async def test_version(): + async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, _simple_update_callback) as api: + v = await api.veilid_version() + print("veilid_version: {}".format(v.__dict__)) + vstr = await api.veilid_version_string() + print("veilid_version_string: {}".format(vstr)) diff --git a/veilid-python/update_schema.sh b/veilid-python/update_schema.sh index b00e14e8..ad2f59df 100755 --- a/veilid-python/update_schema.sh +++ b/veilid-python/update_schema.sh @@ -11,7 +11,7 @@ if [ ! -f "$VEILID_SERVER" ]; then fi # Produce schema from veilid-server -$VEILID_SERVER --emit-schema Request > $SCRIPTDIR/schema/Request.json -$VEILID_SERVER --emit-schema RecvMessage > $SCRIPTDIR/schema/RecvMessage.json +$VEILID_SERVER --emit-schema Request > $SCRIPTDIR/veilid_python/schema/Request.json +$VEILID_SERVER --emit-schema RecvMessage > $SCRIPTDIR/veilid_python/schema/RecvMessage.json diff --git a/veilid-python/veilid_python/__init__.py b/veilid-python/veilid_python/__init__.py index e69de29b..be213c07 100644 --- a/veilid-python/veilid_python/__init__.py +++ b/veilid-python/veilid_python/__init__.py @@ -0,0 +1,6 @@ +from .api import * +from .config import * +from .error import * +from .json_api import * +from .error import * +from .types import * diff --git a/veilid-python/veilid_python/api.py b/veilid-python/veilid_python/api.py index 356866d8..07134d43 100644 --- a/veilid-python/veilid_python/api.py +++ b/veilid-python/veilid_python/api.py @@ -44,11 +44,11 @@ class RoutingContext(ABC): async def watch_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)], expiration: Timestamp, count: int) -> Timestamp: pass @abstractmethod - async def cancel_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)]) -> bool: + async def cancel_dht_watch(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)]) -> bool: pass -class TableDBTransaction(ABC): +class TableDbTransaction(ABC): @abstractmethod async def commit(self): pass @@ -62,15 +62,15 @@ class TableDBTransaction(ABC): async def delete(self, col: int, key: bytes): pass -class TableDB(ABC): +class TableDb(ABC): @abstractmethod async def get_column_count(self) -> int: pass @abstractmethod - async def get_keys(self, col: int) -> list[str]: + async def get_keys(self, col: int) -> list[bytes]: pass @abstractmethod - async def transact(self) -> TableDBTransaction: + async def transact(self) -> TableDbTransaction: pass @abstractmethod async def store(self, col: int, key: bytes, value: bytes): @@ -177,7 +177,7 @@ class VeilidAPI(ABC): async def new_routing_context(self) -> RoutingContext: pass @abstractmethod - async def open_table_db(self, name: str, column_count: int) -> TableDB: + async def open_table_db(self, name: str, column_count: int) -> TableDb: pass @abstractmethod async def delete_table_db(self, name: str): diff --git a/veilid-python/veilid_python/json_api.py b/veilid-python/veilid_python/json_api.py index b68d3a76..4bffc670 100644 --- a/veilid-python/veilid_python/json_api.py +++ b/veilid-python/veilid_python/json_api.py @@ -1,160 +1,74 @@ -import json; -import asyncio; +import json +import asyncio +from jsonschema import validators, exceptions + from typing import Callable, Awaitable -from .api import *; +from .api import * from .state import * from .config import * from .error import * from .types import * from .operations import * -class _JsonRoutingContext(RoutingContext): - api: VeilidAPI - rc_id: int - - def __init__(self, api: VeilidAPI, rc_id: int): - self.api = api - self.rc_id = rc_id +############################################################## - async def with_privacy(self) -> Self: - new_rc_id = raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, - rc_id = self.rc_id, - rc_op = RoutingContextOperation.WITH_PRIVACY)) - return _JsonRoutingContext(self.api, new_rc_id) - - async def with_custom_privacy(self, stability: Stability) -> Self: - new_rc_id = raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, - rc_id = self.rc_id, - rc_op = RoutingContextOperation.WITH_CUSTOM_PRIVACY, - stability = stability)) - return _JsonRoutingContext(self.api, new_rc_id) - async def with_sequencing(self, sequencing: Sequencing) -> Self: - new_rc_id = raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, - rc_id = self.rc_id, - rc_op = RoutingContextOperation.WITH_SEQUENCING, - sequencing = sequencing)) - return _JsonRoutingContext(self.api, new_rc_id) - async def app_call(self, target: TypedKey | RouteId, request: bytes) -> bytes: - return urlsafe_b64decode_no_pad(raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, - rc_id = self.rc_id, - rc_op = RoutingContextOperation.APP_CALL, - target = target, - request = request))) - async def app_message(self, target: TypedKey | RouteId, message: bytes): - raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, - rc_id = self.rc_id, - rc_op = RoutingContextOperation.APP_MESSAGE, - target = target, - message = message)) - async def create_dht_record(self, kind: CryptoKind, schema: DHTSchema) -> DHTRecordDescriptor: - return DHTRecordDescriptor.from_json(raise_api_result(await self.send_ndjson_request(Operation.ROUTING_CONTEXT, - rc_id = self.rc_id, - rc_op = RoutingContextOperation.CREATE_DHT_RECORD, - kind = kind, - schema = schema))) - async def open_dht_record(self, key: TypedKey, writer: Optional[KeyPair]) -> DHTRecordDescriptor: - pass - async def close_dht_record(self, key: TypedKey): - pass - async def delete_dht_record(self, key: TypedKey): - pass - async def get_dht_value(self, key: TypedKey, subkey: ValueSubkey, force_refresh: bool) -> Optional[ValueData]: - pass - async def set_dht_value(self, key: TypedKey, subkey: ValueSubkey, data: bytes) -> Optional[ValueData]: - pass - async def watch_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)], expiration: Timestamp, count: int) -> Timestamp: - pass - async def cancel_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)]) -> bool: - pass - +import importlib.resources as importlib_resources +from . import schema -class _JsonTableDBTransaction(TableDBTransaction): - async def commit(self): - pass - async def rollback(self): - pass - async def store(self, col: int, key: bytes, value: bytes): - pass - async def delete(self, col: int, key: bytes): - pass +def _get_schema_validator(schema): + cls = validators.validator_for(schema) + cls.check_schema(schema) + validator = cls(schema) + return validator -class _JsonTableDB(TableDB): - async def get_column_count(self) -> int: - pass - async def get_keys(self, col: int) -> list[str]: - pass - async def transact(self) -> TableDBTransaction: - pass - async def store(self, col: int, key: bytes, value: bytes): - pass - async def load(self, col: int, key: bytes) -> Optional[bytes]: - pass - async def delete(self, col: int, key: bytes) -> Optional[bytes]: - pass +def _schema_validate(validator, instance): + error = exceptions.best_match(validator.iter_errors(instance)) + if error is not None: + raise error -class _JsonCryptoSystem(CryptoSystem): - async def cached_dh(self, key: PublicKey, secret: SecretKey) -> SharedSecret: - pass - async def compute_dh(self, key: PublicKey, secret: SecretKey) -> SharedSecret: - pass - async def random_bytes(self, len: int) -> bytes: - pass - async def default_salt_length(self) -> int: - pass - async def hash_password(self, password: bytes, salt: bytes) -> str: - pass - async def verify_password(self, password: bytes, password_hash: str) -> bool: - pass - async def derive_shared_secret(self, password: bytes, salt: bytes) -> SharedSecret: - pass - async def random_nonce(self) -> Nonce: - pass - async def random_shared_secret(self) -> SharedSecret: - pass - async def generate_key_pair(self) -> KeyPair: - pass - async def generate_hash(self, data: bytes) -> HashDigest: - pass - async def validate_key_pair(self, key: PublicKey, secret: SecretKey) -> bool: - pass - async def validate_hash(self, data: bytes, hash_digest: HashDigest) -> bool: - pass - async def distance(self, key1: CryptoKey, key2: CryptoKey) -> CryptoKeyDistance: - pass - async def sign(self, key: PublicKey, secret: SecretKey, data: bytes) -> Signature: - pass - async def verify(self, key: PublicKey, data: bytes, signature: Signature): - pass - async def aead_overhead(self) -> int: - pass - async def decrypt_aead(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret, associated_data: Optional[bytes]) -> bytes: - pass - async def encrypt_aead(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret, associated_data: Optional[bytes]) -> bytes: - pass - async def crypt_no_auth(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret) -> bytes: - pass +_VALIDATOR_REQUEST = _get_schema_validator(json.loads((importlib_resources.files(schema) / 'Request.json').read_text())) +_VALIDATOR_RECV_MESSAGE = _get_schema_validator(json.loads((importlib_resources.files(schema) / 'RecvMessage.json').read_text())) +############################################################## + class _JsonVeilidAPI(VeilidAPI): reader: asyncio.StreamReader writer: asyncio.StreamWriter update_callback: Callable[[VeilidUpdate], Awaitable] handle_recv_messages_task: Optional[asyncio.Task] + validate_schemas: bool # Shared Mutable State lock: asyncio.Lock next_id: int in_flight_requests: dict - def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, update_callback: Callable[[VeilidUpdate], Awaitable]): + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, update_callback: Callable[[VeilidUpdate], Awaitable], validate_schema: bool = True): self.reader = reader self.writer = writer self.update_callback = update_callback + self.validate_schema = validate_schema self.handle_recv_messages_task = None self.lock = asyncio.Lock() self.next_id = 1 self.in_flight_requests = dict() + async def __aenter__(self): + return self + + async def __aexit__(self, *excinfo): + await self.close() + + async def close(self): + if self.handle_recv_messages_task is not None: + self.handle_recv_messages_task.cancel() + try: + await self.handle_recv_messages_task + except asyncio.CancelledError: + pass + + @staticmethod async def connect(host: str, port: int, update_callback: Callable[[VeilidUpdate], Awaitable]) -> Self: reader, writer = await asyncio.open_connection(host, port) @@ -184,6 +98,9 @@ class _JsonVeilidAPI(VeilidAPI): # Parse line as ndjson j = json.loads(linebytes.strip()) + if self.validate_schema: + _schema_validate(_VALIDATOR_RECV_MESSAGE, j) + # Process the message if j['type'] == "Response": await self.handle_recv_message_response(j) @@ -196,6 +113,7 @@ class _JsonVeilidAPI(VeilidAPI): self.writer.close() await self.writer.wait_closed() self.writer = None + self.handle_recv_messages_task = None async def allocate_request_future(self, id: int) -> asyncio.Future: reqfuture = asyncio.get_running_loop().create_future() @@ -216,7 +134,22 @@ class _JsonVeilidAPI(VeilidAPI): finally: self.lock.release() - async def send_ndjson_request(self, op: Operation, **kwargs) -> dict: + def send_one_way_ndjson_request(self, op: Operation, **kwargs): + # Make NDJSON string for request + # Always use id 0 because no reply will be received for one-way requests + req = { "id": 0, "op": op } + for k, v in kwargs.items(): + req[k] = v + reqstr = VeilidJSONEncoder.dumps(req) + "\n" + reqbytes = reqstr.encode() + + if self.validate_schema: + _schema_validate(_VALIDATOR_REQUEST, json.loads(reqbytes)) + + # Send to socket without waitings + self.writer.write(reqbytes) + + async def send_ndjson_request(self, op: Operation, validate: Optional[Callable[[dict, dict], None]] = None, **kwargs) -> dict: # Get next id await self.lock.acquire() @@ -229,10 +162,13 @@ class _JsonVeilidAPI(VeilidAPI): # Make NDJSON string for request req = { "id": id, "op": op } for k, v in kwargs.items(): - setattr(req, k, v) + req[k] = v reqstr = VeilidJSONEncoder.dumps(req) + "\n" reqbytes = reqstr.encode() - + + if self.validate_schema: + _schema_validate(_VALIDATOR_REQUEST, json.loads(reqbytes)) + # Allocate future for request reqfuture = await self.allocate_request_future(id) @@ -240,13 +176,20 @@ class _JsonVeilidAPI(VeilidAPI): try: self.writer.write(reqbytes) await self.writer.drain() - finally: + except: # Send failed, release future - self.cancel_request_future(id) + await self.cancel_request_future(id) + raise # Wait for response response = await reqfuture + # Validate if we have a validator + if response["op"] != req["op"]: + raise ValueError("Response op does not match request op") + if validate is not None: + validate(req, response) + return response async def control(self, args: list[str]) -> str: @@ -284,31 +227,387 @@ class _JsonVeilidAPI(VeilidAPI): ) async def new_routing_context(self) -> RoutingContext: rc_id = raise_api_result(await self.send_ndjson_request(Operation.NEW_ROUTING_CONTEXT)) - return RoutingContext(self, rc_id) - - async def open_table_db(self, name: str, column_count: int) -> TableDB: - pass + return _JsonRoutingContext(self, rc_id) + async def open_table_db(self, name: str, column_count: int) -> TableDb: + db_id = raise_api_result(await self.send_ndjson_request(Operation.OPEN_TABLE_DB, + name = name, + column_count = column_count)) + return _JsonTableDb(self, db_id) async def delete_table_db(self, name: str): - pass + return raise_api_result(await self.send_ndjson_request(Operation.DELETE_TABLE_DB, + name = name)) async def get_crypto_system(self, kind: CryptoKind) -> CryptoSystem: - pass + cs_id = raise_api_result(await self.send_ndjson_request(Operation.GET_CRYPTO_SYSTEM, + kind = kind)) + return _JsonCryptoSystem(self, cs_id) async def best_crypto_system(self) -> CryptoSystem: - pass + cs_id = raise_api_result(await self.send_ndjson_request(Operation.BEST_CRYPTO_SYSTEM)) + return _JsonCryptoSystem(self, cs_id) async def verify_signatures(self, node_ids: list[TypedKey], data: bytes, signatures: list[TypedSignature]) -> list[TypedKey]: - pass + return map(lambda x: TypedKey(x), raise_api_result(await self.send_ndjson_request(Operation.VERIFY_SIGNATURES, + node_ids = node_ids, + data = data, + signatures = signatures))) async def generate_signatures(self, data: bytes, key_pairs: list[TypedKeyPair]) -> list[TypedSignature]: - pass + return map(lambda x: TypedSignature(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_SIGNATURES, + data = data, + key_pairs = key_pairs))) async def generate_key_pair(self, kind: CryptoKind) -> list[TypedKeyPair]: - pass + return map(lambda x: TypedKeyPair(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_KEY_PAIR, + kind = kind))) async def now(self) -> Timestamp: - pass + return Timestamp(raise_api_result(await self.send_ndjson_request(Operation.NOW))) async def debug(self, command: str) -> str: - pass + return raise_api_result(await self.send_ndjson_request(Operation.DEBUG, + command = command + )) async def veilid_version_string(self) -> str: - pass + return raise_api_result(await self.send_ndjson_request(Operation.VEILID_VERSION_STRING)) async def veilid_version(self) -> VeilidVersion: - pass + v = await self.send_ndjson_request(Operation.VEILID_VERSION) + return VeilidVersion(v['major'], v['minor'], v['patch']) + +###################################################### + +def validate_rc_op(request: dict, response: dict): + if response['rc_op'] != request['rc_op']: + raise ValueError("Response rc_op does not match request rc_op") + +class _JsonRoutingContext(RoutingContext): + api: _JsonVeilidAPI + rc_id: int + + def __init__(self, api: _JsonVeilidAPI, rc_id: int): + self.api = api + self.rc_id = rc_id + + def __del__(self): + self.api.send_one_way_ndjson_request(Operation.ROUTING_CONTEXT, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.RELEASE) + + async def with_privacy(self) -> Self: + new_rc_id = raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.WITH_PRIVACY)) + return _JsonRoutingContext(self.api, new_rc_id) + + async def with_custom_privacy(self, stability: Stability) -> Self: + new_rc_id = raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.WITH_CUSTOM_PRIVACY, + stability = stability)) + return _JsonRoutingContext(self.api, new_rc_id) + async def with_sequencing(self, sequencing: Sequencing) -> Self: + new_rc_id = raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.WITH_SEQUENCING, + sequencing = sequencing)) + return _JsonRoutingContext(self.api, new_rc_id) + async def app_call(self, target: TypedKey | RouteId, request: bytes) -> bytes: + return urlsafe_b64decode_no_pad(raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.APP_CALL, + target = target, + request = request))) + async def app_message(self, target: TypedKey | RouteId, message: bytes): + raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.APP_MESSAGE, + target = target, + message = message)) + async def create_dht_record(self, kind: CryptoKind, schema: DHTSchema) -> DHTRecordDescriptor: + return DHTRecordDescriptor.from_json(raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.CREATE_DHT_RECORD, + kind = kind, + schema = schema))) + async def open_dht_record(self, key: TypedKey, writer: Optional[KeyPair]) -> DHTRecordDescriptor: + return DHTRecordDescriptor.from_json(raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.OPEN_DHT_RECORD, + key = key, + writer = writer))) + async def close_dht_record(self, key: TypedKey): + raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.CLOSE_DHT_RECORD, + key = key)) + async def delete_dht_record(self, key: TypedKey): + raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.DELETE_DHT_RECORD, + key = key)) + async def get_dht_value(self, key: TypedKey, subkey: ValueSubkey, force_refresh: bool) -> Optional[ValueData]: + ret = raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.GET_DHT_VALUE, + key = key, + subkey = subkey, + force_refresh = force_refresh)) + return None if ret is None else ValueData.from_json(ret) + async def set_dht_value(self, key: TypedKey, subkey: ValueSubkey, data: bytes) -> Optional[ValueData]: + ret = raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.SET_DHT_VALUE, + key = key, + subkey = subkey, + data = data)) + return None if ret is None else ValueData.from_json(ret) + async def watch_dht_values(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)], expiration: Timestamp, count: int) -> Timestamp: + return Timestamp(raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.WATCH_DHT_VALUES, + key = key, + subkeys = subkeys, + expiration = expiration, + count = count))) + async def cancel_dht_watch(self, key: TypedKey, subkeys: list[(ValueSubkey, ValueSubkey)]) -> bool: + return raise_api_result(await self.api.send_ndjson_request(Operation.ROUTING_CONTEXT, validate=validate_rc_op, + rc_id = self.rc_id, + rc_op = RoutingContextOperation.CANCEL_DHT_WATCH, + key = key, + subkeys = subkeys)) + +###################################################### + +def validate_tx_op(request: dict, response: dict): + if response['tx_op'] != request['tx_op']: + raise ValueError("Response tx_op does not match request tx_op") + +class _JsonTableDbTransaction(TableDbTransaction): + api: _JsonVeilidAPI + tx_id: int + done: bool + + def __init__(self, api: _JsonVeilidAPI, tx_id: int): + self.api = api + self.tx_id = tx_id + self.done = False + + def __del__(self): + if not self.done: + raise AssertionError("Should have committed or rolled back transaction") + + async def commit(self): + raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB_TRANSACTION, validate=validate_tx_op, + tx_id = self.tx_id, + tx_op = TableDbTransactionOperation.COMMIT)) + self.done = True + async def rollback(self): + raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB_TRANSACTION, validate=validate_tx_op, + tx_id = self.tx_id, + tx_op = TableDbTransactionOperation.ROLLBACK)) + self.done = True + async def store(self, col: int, key: bytes, value: bytes): + await self.api.send_ndjson_request(Operation.TABLE_DB_TRANSACTION, validate=validate_tx_op, + tx_id = self.tx_id, + tx_op = TableDbTransactionOperation.STORE, + col = col, + key = key, + value = value) + async def delete(self, col: int, key: bytes): + await self.api.send_ndjson_request(Operation.TABLE_DB_TRANSACTION, validate=validate_tx_op, + tx_id = self.tx_id, + tx_op = TableDbTransactionOperation.DELETE, + col = col, + key = key) + +###################################################### + +def validate_db_op(request: dict, response: dict): + if response['db_op'] != request['db_op']: + raise ValueError("Response db_op does not match request db_op") + +class _JsonTableDb(TableDb): + api: _JsonVeilidAPI + db_id: int + + def __init__(self, api: _JsonVeilidAPI, db_id: int): + self.api = api + self.db_id = db_id + + def __del__(self): + self.api.send_one_way_ndjson_request(Operation.TABLE_DB, + db_id = self.db_id, + rc_op = TableDbOperation.RELEASE) + + async def get_column_count(self) -> int: + return raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, + db_id = self.db_id, + db_op = TableDbOperation.GET_COLUMN_COUNT)) + async def get_keys(self, col: int) -> list[bytes]: + return map(lambda x: urlsafe_b64decode_no_pad(x), raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, + db_id = self.db_id, + db_op = TableDbOperation.GET_KEYS, + col = col))) + async def transact(self) -> TableDbTransaction: + tx_id = raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, + db_id = self.db_id, + db_op = TableDbOperation.TRANSACT)) + return _JsonTableDbTransaction(self.api, tx_id) + async def store(self, col: int, key: bytes, value: bytes): + return raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, + db_id = self.db_id, + db_op = TableDbOperation.STORE, + col = col, + key = key, + value = value)) + async def load(self, col: int, key: bytes) -> Optional[bytes]: + res = raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, + db_id = self.db_id, + db_op = TableDbOperation.LOAD, + col = col, + key = key)) + return None if res is None else urlsafe_b64decode_no_pad(res) + async def delete(self, col: int, key: bytes) -> Optional[bytes]: + res = raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, + db_id = self.db_id, + db_op = TableDbOperation.DELETE, + col = col, + key = key)) + return None if res is None else urlsafe_b64decode_no_pad(res) + +###################################################### -def json_api_connect(host:str, port:int) -> VeilidAPI: - return _JsonVeilidAPI.connect(host, port) \ No newline at end of file + +def validate_cs_op(request: dict, response: dict): + if response['cs_op'] != request['cs_op']: + raise ValueError("Response cs_op does not match request cs_op") + +class _JsonCryptoSystem(CryptoSystem): + api: _JsonVeilidAPI + cs_id: int + + def __init__(self, api: _JsonVeilidAPI, cs_id: int): + self.api = api + self.cs_id = cs_id + + def __del__(self): + self.api.send_one_way_ndjson_request(Operation.CRYPTO_SYSTEM, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.RELEASE) + + async def cached_dh(self, key: PublicKey, secret: SecretKey) -> SharedSecret: + return SharedSecret(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.CACHED_DH, + key = key, + secret = secret))) + async def compute_dh(self, key: PublicKey, secret: SecretKey) -> SharedSecret: + return SharedSecret(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.COMPUTE_DH, + key = key, + secret = secret))) + async def random_bytes(self, len: int) -> bytes: + return urlsafe_b64decode_no_pad(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.RANDOM_BYTES, + len = len))) + async def default_salt_length(self) -> int: + return raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.DEFAULT_SALT_LENGTH)) + async def hash_password(self, password: bytes, salt: bytes) -> str: + return raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.HASH_PASSWORD, + password = password, + salt = salt)) + async def verify_password(self, password: bytes, password_hash: str) -> bool: + return raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.VERIFY_PASSWORD, + password = password, + password_hash = password_hash)) + async def derive_shared_secret(self, password: bytes, salt: bytes) -> SharedSecret: + return SharedSecret(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.DERIVE_SHARED_SECRET, + password = password, + salt = salt))) + async def random_nonce(self) -> Nonce: + return Nonce(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.RANDOM_NONCE))) + async def random_shared_secret(self) -> SharedSecret: + return SharedSecret(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.RANDOM_SHARED_SECRET))) + async def generate_key_pair(self) -> KeyPair: + return KeyPair(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.GENERATE_KEY_PAIR))) + async def generate_hash(self, data: bytes) -> HashDigest: + return HashDigest(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.GENERATE_HASH, + data = data))) + async def validate_key_pair(self, key: PublicKey, secret: SecretKey) -> bool: + return raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.VALIDATE_KEY_PAIR, + key = key, + secret = secret)) + async def validate_hash(self, data: bytes, hash_digest: HashDigest) -> bool: + return raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.VALIDATE_HASH, + data = data, + hash_digest = hash_digest)) + async def distance(self, key1: CryptoKey, key2: CryptoKey) -> CryptoKeyDistance: + return CryptoKeyDistance(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.DISTANCE, + key1 = key1, + key2 = key2))) + async def sign(self, key: PublicKey, secret: SecretKey, data: bytes) -> Signature: + return Signature(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.SIGN, + key = key, + secret = secret, + data = data))) + async def verify(self, key: PublicKey, data: bytes, signature: Signature): + raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.VERIFY, + key = key, + data = data, + signature = signature)) + async def aead_overhead(self) -> int: + return raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.AEAD_OVERHEAD)) + async def decrypt_aead(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret, associated_data: Optional[bytes]) -> bytes: + return urlsafe_b64decode_no_pad(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.DECRYPT_AEAD, + body = body, + nonce = nonce, + shared_secret = shared_secret, + associated_data = associated_data))) + async def encrypt_aead(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret, associated_data: Optional[bytes]) -> bytes: + return urlsafe_b64decode_no_pad(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.ENCRYPT_AEAD, + body = body, + nonce = nonce, + shared_secret = shared_secret, + associated_data = associated_data))) + async def crypt_no_auth(self, body: bytes, nonce: Nonce, shared_secret: SharedSecret) -> bytes: + return urlsafe_b64decode_no_pad(raise_api_result(await self.api.send_ndjson_request(Operation.CRYPTO_SYSTEM, validate=validate_cs_op, + cs_id = self.cs_id, + cs_op = CryptoSystemOperation.CRYPT_NO_AUTH, + body = body, + nonce = nonce, + shared_secret = shared_secret))) + + +###################################################### + +def json_api_connect(host:str, port:int, update_callback: Callable[[VeilidUpdate], Awaitable]) -> VeilidAPI: + return _JsonVeilidAPI.connect(host, port, update_callback) + diff --git a/veilid-python/veilid_python/operations.py b/veilid-python/veilid_python/operations.py index 20dd11eb..297a0c77 100644 --- a/veilid-python/veilid_python/operations.py +++ b/veilid-python/veilid_python/operations.py @@ -55,7 +55,7 @@ class TableDbOperation(StrEnum): LOAD = "Load" DELETE = "Delete" -class TableDBTransactionOperation(StrEnum): +class TableDbTransactionOperation(StrEnum): INVALID_ID = "InvalidId" COMMIT = "Commit" ROLLBACK = "Rollback" diff --git a/veilid-python/schema/RecvMessage.json b/veilid-python/veilid_python/schema/RecvMessage.json similarity index 100% rename from veilid-python/schema/RecvMessage.json rename to veilid-python/veilid_python/schema/RecvMessage.json diff --git a/veilid-python/schema/Request.json b/veilid-python/veilid_python/schema/Request.json similarity index 100% rename from veilid-python/schema/Request.json rename to veilid-python/veilid_python/schema/Request.json diff --git a/veilid-python/veilid_python/types.py b/veilid-python/veilid_python/types.py index 9b87670c..0f4ced5e 100644 --- a/veilid-python/veilid_python/types.py +++ b/veilid-python/veilid_python/types.py @@ -3,7 +3,9 @@ import json import base64 from enum import StrEnum -from typing import Self, Optional, Any +from typing import Self, Optional, Any, Tuple + +#################################################################### def urlsafe_b64encode_no_pad(b: bytes) -> str: """ @@ -33,6 +35,33 @@ class VeilidJSONEncoder(json.JSONEncoder): def dumps(req: Any, *args, **kwargs) -> str: return json.dumps(req, cls = VeilidJSONEncoder, *args, **kwargs) +#################################################################### + +class VeilidLogLevel(StrEnum): + ERROR = 'Error' + WARN = 'Warn' + INFO = 'Info' + DEBUG = 'Debug' + TRACE = 'Trace' + +class CryptoKind(StrEnum): + CRYPTO_KIND_NONE = "NONE" + CRYPTO_KIND_VLD0 = "VLD0" + +class Stability(StrEnum): + LOW_LATENCY = "LowLatency" + RELIABLE = "Reliable" + +class Sequencing(StrEnum): + NO_PREFERENCE = "NoPreference" + PREFER_ORDERED = "PreferOrdered" + ENSURE_ORDERED = "EnsureOrdered" + +class DHTSchemaKind(StrEnum): + DFLT = "DFLT" + SMPL = "SMPL" + +#################################################################### class Timestamp(int): pass @@ -49,41 +78,98 @@ class OperationId(int): class RouteId(str): pass -class TypedKey(str): - pass +class CryptoKey: + def to_bytes(self) -> bytes: + return urlsafe_b64decode_no_pad(self) -class TypedKeyPair(str): - pass +class CryptoKeyDistance(CryptoKey, str): + @staticmethod + def from_bytes(b: bytes) -> Self: + return CryptoKeyDistance(urlsafe_b64encode_no_pad(b)) -class TypedSignature(str): - pass +class PublicKey(CryptoKey, str): + @staticmethod + def from_bytes(b: bytes) -> Self: + return PublicKey(urlsafe_b64encode_no_pad(b)) -class CryptoKey(str): - pass +class SecretKey(CryptoKey, str): + @staticmethod + def from_bytes(b: bytes) -> Self: + return SecretKey(urlsafe_b64encode_no_pad(b)) -class CryptoKeyDistance(str): - pass +class SharedSecret(CryptoKey, str): + @staticmethod + def from_bytes(b: bytes) -> Self: + return SharedSecret(urlsafe_b64encode_no_pad(b)) -class PublicKey(CryptoKey): - pass - -class SecretKey(CryptoKey): - pass - -class SharedSecret(CryptoKey): - pass +class HashDigest(CryptoKey, str): + @staticmethod + def from_bytes(b: bytes) -> Self: + return HashDigest(urlsafe_b64encode_no_pad(b)) class Signature(str): - pass + @staticmethod + def from_bytes(b: bytes) -> Self: + return Signature(urlsafe_b64encode_no_pad(b)) + def to_bytes(self) -> bytes: + return urlsafe_b64decode_no_pad(self) class Nonce(str): - pass + @staticmethod + def from_bytes(b: bytes) -> Self: + return Signature(urlsafe_b64encode_no_pad(b)) + def to_bytes(self) -> bytes: + return urlsafe_b64decode_no_pad(self) class KeyPair(str): - pass + @staticmethod + def from_parts(key: PublicKey, secret: SecretKey) -> Self: + return KeyPair(key + ":" + secret) + def key(self) -> PublicKey: + return PublicKey(str.split(":", 1)[0]) + def secret(self) -> SecretKey: + return SecretKey(str.split(":", 1)[1]) + def to_parts(self) -> Tuple[PublicKey, SecretKey]: + parts = str.split(":", 1) + return (PublicKey(parts[0]), SecretKey(parts[1])) -class HashDigest(CryptoKey): - pass +class CryptoTyped: + def kind(self) -> CryptoKind: + if self[4] != ':': + raise ValueError("Not CryptoTyped") + return CryptoKind(self[0:4]) + def _value(self) -> str: + if self[4] != ':': + raise ValueError("Not CryptoTyped") + return self[5:] + +class TypedKey(CryptoTyped, str): + @staticmethod + def from_value(kind: CryptoKind, value: PublicKey) -> Self: + return TypedKey(kind + ":" + value) + def value(self) -> PublicKey: + PublicKey(self._value()) + +class TypedSecret(CryptoTyped, str): + @staticmethod + def from_value(kind: CryptoKind, value: SecretKey) -> Self: + return TypedSecret(kind + ":" + value) + def value(self) -> SecretKey: + SecretKey(self._value()) + +class TypedKeyPair(CryptoTyped, str): + @staticmethod + def from_value(kind: CryptoKind, value: KeyPair) -> Self: + return TypedKeyPair(kind + ":" + value) + def value(self) -> KeyPair: + KeyPair(self._value()) + +class TypedSignature(CryptoTyped, str): + @staticmethod + def from_value(kind: CryptoKind, value: Signature) -> Self: + return TypedSignature(kind + ":" + value) + def value(self) -> Signature: + Signature(self._value()) class ValueSubkey(int): pass @@ -91,6 +177,8 @@ class ValueSubkey(int): class ValueSeqNum(int): pass +#################################################################### + class VeilidVersion: _major: int _minor: int @@ -109,13 +197,6 @@ class VeilidVersion: def patch(self): return self._patch -class VeilidLogLevel(StrEnum): - ERROR = 'Error' - WARN = 'Warn' - INFO = 'Info' - DEBUG = 'Debug' - TRACE = 'Trace' - class NewPrivateRouteResult: route_id: RouteId blob: bytes @@ -130,23 +211,6 @@ class NewPrivateRouteResult: RouteId(j['route_id']), urlsafe_b64decode_no_pad(j['blob'])) -class CryptoKind(StrEnum): - CRYPTO_KIND_NONE = "NONE" - CRYPTO_KIND_VLD0 = "VLD0" - -class Stability(StrEnum): - LOW_LATENCY = "LowLatency" - RELIABLE = "Reliable" - -class Sequencing(StrEnum): - NO_PREFERENCE = "NoPreference" - PREFER_ORDERED = "PreferOrdered" - ENSURE_ORDERED = "EnsureOrdered" - -class DHTSchemaKind(StrEnum): - DFLT = "DFLT" - SMPL = "SMPL" - class DHTSchemaSMPLMember: m_key: PublicKey m_cnt: int From 615e0ca1d075c2d8e6b1c91ca8030ae1dd58851d Mon Sep 17 00:00:00 2001 From: John Smith Date: Wed, 14 Jun 2023 21:06:10 -0400 Subject: [PATCH 16/23] more tests --- .vscode/launch.json | 7 ++++ veilid-python/tests/__init__.py | 22 +++++++++++ veilid-python/tests/test_basic.py | 26 ++----------- veilid-python/tests/test_crypto.py | 28 ++++++++++++++ veilid-python/veilid_python/json_api.py | 51 +++++++++++++++++-------- veilid-server/src/client_api.rs | 14 +++---- 6 files changed, 103 insertions(+), 45 deletions(-) create mode 100644 veilid-python/tests/test_crypto.py diff --git a/.vscode/launch.json b/.vscode/launch.json index c3af759b..b0f99ce6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,6 +11,13 @@ } ], "configurations": [ + { + "name": "Python: Attach using Process Id", + "type": "python", + "request": "attach", + "processId": "${command:pickProcess}", + "justMyCode": true + }, { "type": "lldb", "request": "attach", diff --git a/veilid-python/tests/__init__.py b/veilid-python/tests/__init__.py index e69de29b..e26f2d5f 100644 --- a/veilid-python/tests/__init__.py +++ b/veilid-python/tests/__init__.py @@ -0,0 +1,22 @@ +import pytest +pytest_plugins = ('pytest_asyncio',) + +import os + +################################################################## +VEILID_SERVER = os.getenv("VEILID_SERVER") +if VEILID_SERVER is not None: + vsparts = VEILID_SERVER.split(":") + VEILID_SERVER = vsparts[0] + if len(vsparts) == 2: + VEILID_SERVER_PORT = int(vsparts[1]) + else: + VEILID_SERVER_PORT = 5959 +else: + VEILID_SERVER = "localhost" + VEILID_SERVER_PORT = 5959 + +################################################################## + +async def simple_update_callback(update): + print("VeilidUpdate: {}".format(update)) diff --git a/veilid-python/tests/test_basic.py b/veilid-python/tests/test_basic.py index 80443727..d74b25b2 100644 --- a/veilid-python/tests/test_basic.py +++ b/veilid-python/tests/test_basic.py @@ -2,42 +2,24 @@ import veilid_python import pytest -import os - -pytest_plugins = ('pytest_asyncio',) +from . import * ################################################################## -VEILID_SERVER = os.getenv("VEILID_SERVER") -if VEILID_SERVER is not None: - vsparts = VEILID_SERVER.split(":") - VEILID_SERVER = vsparts[0] - if len(vsparts) == 2: - VEILID_SERVER_PORT = int(vsparts[1]) - else: - VEILID_SERVER_PORT = 5959 -else: - VEILID_SERVER = "localhost" - VEILID_SERVER_PORT = 5959 - -################################################################## - -async def _simple_update_callback(update): - print("VeilidUpdate: {}".format(update)) @pytest.mark.asyncio async def test_connect(): - async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, _simple_update_callback) as api: + async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: pass @pytest.mark.asyncio async def test_fail_connect(): with pytest.raises(Exception): - async with await veilid_python.json_api_connect("fuahwelifuh32luhwafluehawea", 1, _simple_update_callback) as api: + async with await veilid_python.json_api_connect("fuahwelifuh32luhwafluehawea", 1, simple_update_callback) as api: pass @pytest.mark.asyncio async def test_version(): - async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, _simple_update_callback) as api: + async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: v = await api.veilid_version() print("veilid_version: {}".format(v.__dict__)) vstr = await api.veilid_version_string() diff --git a/veilid-python/tests/test_crypto.py b/veilid-python/tests/test_crypto.py new file mode 100644 index 00000000..6ceb993f --- /dev/null +++ b/veilid-python/tests/test_crypto.py @@ -0,0 +1,28 @@ +# Crypto veilid_python tests + +import veilid_python +import pytest +from . import * + +################################################################## + +@pytest.mark.asyncio +async def test_best_crypto_system(): + async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: + bcs = await api.best_crypto_system() + # let handle dangle for test + # del bcs + +@pytest.mark.asyncio +async def test_get_crypto_system(): + async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: + cs = await api.get_crypto_system(veilid_python.CryptoKind.CRYPTO_KIND_VLD0) + # clean up handle early + del cs + +@pytest.mark.asyncio +async def test_get_crypto_system_invalid(): + async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: + with pytest.raises(veilid_python.VeilidAPIError): + cs = await api.get_crypto_system(veilid_python.CryptoKind.CRYPTO_KIND_NONE) + diff --git a/veilid-python/veilid_python/json_api.py b/veilid-python/veilid_python/json_api.py index 4bffc670..7ec5fff8 100644 --- a/veilid-python/veilid_python/json_api.py +++ b/veilid-python/veilid_python/json_api.py @@ -59,16 +59,34 @@ class _JsonVeilidAPI(VeilidAPI): async def __aexit__(self, *excinfo): await self.close() - - async def close(self): - if self.handle_recv_messages_task is not None: - self.handle_recv_messages_task.cancel() - try: - await self.handle_recv_messages_task - except asyncio.CancelledError: - pass - + async def _cleanup_close(self): + await self.lock.acquire() + try: + self.reader = None + self.writer.close() + await self.writer.wait_closed() + self.writer = None + finally: + self.lock.release() + + async def close(self): + # Take the task + await self.lock.acquire() + try: + if self.handle_recv_messages_task is None: + return + handle_recv_messages_task = self.handle_recv_messages_task + self.handle_recv_messages_task = None + finally: + self.lock.release() + # Cancel it + handle_recv_messages_task.cancel() + try: + await handle_recv_messages_task + except asyncio.CancelledError: + pass + @staticmethod async def connect(host: str, port: int, update_callback: Callable[[VeilidUpdate], Awaitable]) -> Self: reader, writer = await asyncio.open_connection(host, port) @@ -109,11 +127,7 @@ class _JsonVeilidAPI(VeilidAPI): except: pass finally: - self.reader = None - self.writer.close() - await self.writer.wait_closed() - self.writer = None - self.handle_recv_messages_task = None + await self._cleanup_close() async def allocate_request_future(self, id: int) -> asyncio.Future: reqfuture = asyncio.get_running_loop().create_future() @@ -135,6 +149,10 @@ class _JsonVeilidAPI(VeilidAPI): self.lock.release() def send_one_way_ndjson_request(self, op: Operation, **kwargs): + + if self.writer is None: + return + # Make NDJSON string for request # Always use id 0 because no reply will be received for one-way requests req = { "id": 0, "op": op } @@ -156,6 +174,7 @@ class _JsonVeilidAPI(VeilidAPI): try: id = self.next_id self.next_id += 1 + writer = self.writer finally: self.lock.release() @@ -174,8 +193,8 @@ class _JsonVeilidAPI(VeilidAPI): # Send to socket try: - self.writer.write(reqbytes) - await self.writer.drain() + writer.write(reqbytes) + await writer.drain() except: # Send failed, release future await self.cancel_request_future(id) diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index 821d06ef..6b2f9d55 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -102,7 +102,7 @@ impl ClientApi { }; trace!("ClientApi::stop: waiting for stop"); if let Err(err) = jh.await { - error!("{}", err); + eprintln!("{}", err); } trace!("ClientApi::stop: stopped"); } @@ -225,7 +225,7 @@ impl ClientApi { // Marshal json + newline => NDJSON let response_string = serialize_json(json_api::RecvMessage::Response(response)) + "\n"; if let Err(e) = responses_tx.send_async(response_string).await { - warn!("response not sent: {}", e) + eprintln!("response not sent: {}", e) } VeilidAPIResult::Ok(None) } @@ -272,7 +272,7 @@ impl ClientApi { responses_tx: responses_tx.clone(), }; if let Err(e) = requests_tx.send_async(Some(request_line)).await { - error!("failed to enqueue request: {}", e); + eprintln!("failed to enqueue request: {}", e); break; } } @@ -291,7 +291,7 @@ impl ClientApi { ) -> VeilidAPIResult> { while let Ok(resp) = responses_rx.recv_async().await { if let Err(e) = writer.write_all(resp.as_bytes()).await { - error!("failed to write response: {}", e) + eprintln!("failed to write response: {}", e) } } VeilidAPIResult::Ok(None) @@ -302,7 +302,7 @@ impl ClientApi { let peer_addr = match stream.peer_addr() { Ok(v) => v, Err(e) => { - error!("can't get peer address: {}", e); + eprintln!("can't get peer address: {}", e); return; } }; @@ -310,7 +310,7 @@ impl ClientApi { let local_addr = match stream.local_addr() { Ok(v) => v, Err(e) => { - error!("can't get local address: {}", e); + eprintln!("can't get local address: {}", e); return; } }; @@ -387,7 +387,7 @@ impl ClientApi { } Err(e) => { // Connection processing failure, abort - error!("Connection processing failure: {}", e); + eprintln!("Connection processing failure: {}", e); break; } }; From d6f442d431b28032d621a0ca93871617ae6083f3 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 15 Jun 2023 20:22:54 -0400 Subject: [PATCH 17/23] better error handling --- veilid-core/src/network_manager/mod.rs | 39 ++++--- veilid-core/src/routing_table/mod.rs | 21 ++-- veilid-core/src/routing_table/node_ref.rs | 33 +++--- veilid-core/src/routing_table/privacy.rs | 18 ++- .../route_spec_store/route_spec_store.rs | 25 +++-- .../route_spec_store_content.rs | 2 +- .../src/routing_table/routing_table_inner.rs | 105 +++++++++--------- .../src/routing_table/tasks/bootstrap.rs | 104 +++++++++-------- .../routing_table/tasks/relay_management.rs | 13 ++- veilid-core/src/rpc_processor/destination.rs | 7 +- veilid-core/src/rpc_processor/mod.rs | 19 +++- veilid-core/src/veilid_api/debug.rs | 4 +- veilid-python/pyproject.toml | 2 +- veilid-python/tests/__init__.py | 12 +- veilid-python/tests/test_basic.py | 13 ++- veilid-python/tests/test_crypto.py | 34 ++++-- veilid-python/tests/test_routing_context.py | 47 ++++++++ veilid-python/update_schema.sh | 4 +- .../{veilid_python => veilid}/__init__.py | 0 .../{veilid_python => veilid}/api.py | 0 .../{veilid_python => veilid}/config.py | 12 +- .../{veilid_python => veilid}/error.py | 0 .../{veilid_python => veilid}/json_api.py | 36 +++--- .../{veilid_python => veilid}/operations.py | 0 .../schema/RecvMessage.json | 0 .../schema/Request.json | 0 .../{veilid_python => veilid}/state.py | 15 +-- .../{veilid_python => veilid}/types.py | 7 +- veilid-server/src/client_api.rs | 30 +++-- 29 files changed, 368 insertions(+), 234 deletions(-) create mode 100644 veilid-python/tests/test_routing_context.py rename veilid-python/{veilid_python => veilid}/__init__.py (100%) rename veilid-python/{veilid_python => veilid}/api.py (100%) rename veilid-python/{veilid_python => veilid}/config.py (98%) rename veilid-python/{veilid_python => veilid}/error.py (100%) rename veilid-python/{veilid_python => veilid}/json_api.py (95%) rename veilid-python/{veilid_python => veilid}/operations.py (100%) rename veilid-python/{veilid_python => veilid}/schema/RecvMessage.json (100%) rename veilid-python/{veilid_python => veilid}/schema/Request.json (100%) rename veilid-python/{veilid_python => veilid}/state.py (96%) rename veilid-python/{veilid_python => veilid}/types.py (97%) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index d11f9b73..e7abd622 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -685,12 +685,12 @@ impl NetworkManager { peer_info, false, ) { - None => { + Ok(nr) => nr, + Err(e) => { return Ok(NetworkResult::invalid_message( - "unable to register reverse connect peerinfo", - )) + format!("unable to register reverse connect peerinfo: {}", e) + )); } - Some(nr) => nr, }; // Make a reverse connection to the peer and send the receipt to it @@ -708,13 +708,12 @@ impl NetworkManager { peer_info, false, ) { - None => { + Ok(nr) => nr, + Err(e) => { return Ok(NetworkResult::invalid_message( - //sender_id, - "unable to register hole punch connect peerinfo", + format!("unable to register hole punch connect peerinfo: {}", e) )); } - Some(nr) => nr, }; // Get the udp direct dialinfo for the hole punch @@ -1103,7 +1102,7 @@ impl NetworkManager { ContactMethod::Direct(di) => NodeContactMethod::Direct(di), ContactMethod::SignalReverse(relay_key, target_key) => { let relay_nr = routing_table - .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter)? .ok_or_else(|| eyre!("couldn't look up relay"))?; if !target_node_ref.node_ids().contains(&target_key) { bail!("target noderef didn't match target key"); @@ -1112,7 +1111,7 @@ impl NetworkManager { } ContactMethod::SignalHolePunch(relay_key, target_key) => { let relay_nr = routing_table - .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter)? .ok_or_else(|| eyre!("couldn't look up relay"))?; if target_node_ref.node_ids().contains(&target_key) { bail!("target noderef didn't match target key"); @@ -1121,13 +1120,13 @@ impl NetworkManager { } ContactMethod::InboundRelay(relay_key) => { let relay_nr = routing_table - .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter)? .ok_or_else(|| eyre!("couldn't look up relay"))?; NodeContactMethod::InboundRelay(relay_nr) } ContactMethod::OutboundRelay(relay_key) => { let relay_nr = routing_table - .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter) + .lookup_and_filter_noderef(relay_key, routing_domain.into(), dial_info_filter)? .ok_or_else(|| eyre!("couldn't look up relay"))?; NodeContactMethod::OutboundRelay(relay_nr) } @@ -1430,7 +1429,13 @@ impl NetworkManager { // We should, because relays are chosen by nodes that have established connectivity and // should be mutually in each others routing tables. The node needing the relay will be // pinging this node regularly to keep itself in the routing table - routing_table.lookup_node_ref(recipient_id) + match routing_table.lookup_node_ref(recipient_id) { + Ok(v) => v, + Err(e) => { + log_net!(debug "failed to look up recipient node for relay, dropping outbound relayed packet: {}" ,e); + return Ok(false); + } + } }; if let Some(relay_nr) = some_relay_nr { @@ -1472,12 +1477,12 @@ impl NetworkManager { connection_descriptor, ts, ) { - None => { + Ok(v) => v, + Err(e) => { // If the node couldn't be registered just skip this envelope, - // the error will have already been logged + log_net!(debug "failed to register node with existing connection: {}", e); return Ok(false); } - Some(v) => v, }; source_noderef.add_envelope_version(envelope.get_version()); @@ -1574,7 +1579,7 @@ impl NetworkManager { peers: { let mut out = Vec::new(); for (k, v) in routing_table.get_recent_peers() { - if let Some(nr) = routing_table.lookup_node_ref(k) { + if let Ok(Some(nr)) = routing_table.lookup_node_ref(k) { let peer_stats = nr.peer_stats(); let peer = PeerTableData { node_ids: nr.node_ids().iter().copied().collect(), diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 1d63fcb5..2223d189 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -628,14 +628,14 @@ impl RoutingTable { } /// Resolve an existing routing table entry using any crypto kind and return a reference to it - pub fn lookup_any_node_ref(&self, node_id_key: PublicKey) -> Option { + pub fn lookup_any_node_ref(&self, node_id_key: PublicKey) -> EyreResult> { self.inner .read() .lookup_any_node_ref(self.clone(), node_id_key) } /// Resolve an existing routing table entry and return a reference to it - pub fn lookup_node_ref(&self, node_id: TypedKey) -> Option { + pub fn lookup_node_ref(&self, node_id: TypedKey) -> EyreResult> { self.inner.read().lookup_node_ref(self.clone(), node_id) } @@ -645,7 +645,7 @@ impl RoutingTable { node_id: TypedKey, routing_domain_set: RoutingDomainSet, dial_info_filter: DialInfoFilter, - ) -> Option { + ) -> EyreResult> { self.inner.read().lookup_and_filter_noderef( self.clone(), node_id, @@ -662,7 +662,7 @@ impl RoutingTable { routing_domain: RoutingDomain, peer_info: PeerInfo, allow_invalid: bool, - ) -> Option { + ) -> EyreResult { self.inner.write().register_node_with_peer_info( self.clone(), routing_domain, @@ -678,7 +678,7 @@ impl RoutingTable { node_id: TypedKey, descriptor: ConnectionDescriptor, timestamp: Timestamp, - ) -> Option { + ) -> EyreResult { self.inner.write().register_node_with_existing_connection( self.clone(), node_id, @@ -711,7 +711,7 @@ impl RoutingTable { // (uses same logic as send_data, ensuring last_connection works for UDP) for e in &recent_peers { let mut dead = true; - if let Some(nr) = self.lookup_node_ref(*e) { + if let Ok(Some(nr)) = self.lookup_node_ref(*e) { if let Some(last_connection) = nr.last_connection() { out.push((*e, RecentPeersEntry { last_connection })); dead = false; @@ -1017,10 +1017,11 @@ impl RoutingTable { } // Register the node if it's new - if let Some(nr) = - self.register_node_with_peer_info(RoutingDomain::PublicInternet, p, false) - { - out.push(nr); + match self.register_node_with_peer_info(RoutingDomain::PublicInternet, p, false) { + Ok(nr) => out.push(nr), + Err(e) => { + log_rtab!(debug "failed to register node with peer info from find node answer: {}", e); + } } } out diff --git a/veilid-core/src/routing_table/node_ref.rs b/veilid-core/src/routing_table/node_ref.rs index 1f637801..5cb53c64 100644 --- a/veilid-core/src/routing_table/node_ref.rs +++ b/veilid-core/src/routing_table/node_ref.rs @@ -192,25 +192,24 @@ pub trait NodeRefBase: Sized { } dif } - fn relay(&self, routing_domain: RoutingDomain) -> Option { + fn relay(&self, routing_domain: RoutingDomain) -> EyreResult> { self.operate_mut(|rti, e| { - e.signed_node_info(routing_domain) - .and_then(|n| n.relay_peer_info()) - .and_then(|rpi| { - // If relay is ourselves, then return None, because we can't relay through ourselves - // and to contact this node we should have had an existing inbound connection - if rti.unlocked_inner.matches_own_node_id(rpi.node_ids()) { - return None; - } + let Some(sni) = e.signed_node_info(routing_domain) else { + return Ok(None); + }; + let Some(rpi) = sni.relay_peer_info() else { + return Ok(None); + }; + // If relay is ourselves, then return None, because we can't relay through ourselves + // and to contact this node we should have had an existing inbound connection + if rti.unlocked_inner.matches_own_node_id(rpi.node_ids()) { + bail!("Can't relay though ourselves"); + } - // Register relay node and return noderef - rti.register_node_with_peer_info( - self.routing_table(), - routing_domain, - rpi, - false, - ) - }) + // Register relay node and return noderef + let nr = + rti.register_node_with_peer_info(self.routing_table(), routing_domain, rpi, false)?; + Ok(Some(nr)) }) } diff --git a/veilid-core/src/routing_table/privacy.rs b/veilid-core/src/routing_table/privacy.rs index fc670375..66f08c57 100644 --- a/veilid-core/src/routing_table/privacy.rs +++ b/veilid-core/src/routing_table/privacy.rs @@ -37,15 +37,27 @@ impl RouteNode { match self { RouteNode::NodeId(id) => { // - routing_table.lookup_node_ref(TypedKey::new(crypto_kind, *id)) + match routing_table.lookup_node_ref(TypedKey::new(crypto_kind, *id)) { + Ok(nr) => nr, + Err(e) => { + log_rtab!(debug "failed to look up route node: {}", e); + return None; + } + } } RouteNode::PeerInfo(pi) => { // - routing_table.register_node_with_peer_info( + match routing_table.register_node_with_peer_info( RoutingDomain::PublicInternet, pi.clone(), false, - ) + ) { + Ok(nr) => Some(nr), + Err(e) => { + log_rtab!(debug "failed to register route node: {}", e); + return None; + } + } } } } diff --git a/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs index 3f29ba0d..043cd1e3 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs @@ -378,7 +378,14 @@ impl RouteSpecStore { // Already seen this node, should not be in the route twice return None; } - if let Some(relay) = node.locked_mut(rti).relay(RoutingDomain::PublicInternet) { + let opt_relay = match node.locked_mut(rti).relay(RoutingDomain::PublicInternet) { + Ok(r) => r, + Err(e) => { + log_rtab!(error "failed to get relay for route node: {}", e); + return None; + } + }; + if let Some(relay) = opt_relay { let relay_id = relay.locked(rti).best_node_id(); if !seen_nodes.insert(relay_id) { // Already seen this node, should not be in the route twice @@ -869,13 +876,15 @@ impl RouteSpecStore { }; let opt_first_hop = match pr_first_hop_node { - RouteNode::NodeId(id) => rti.lookup_node_ref(routing_table.clone(), TypedKey::new(crypto_kind, id)), - RouteNode::PeerInfo(pi) => rti.register_node_with_peer_info( - routing_table.clone(), - RoutingDomain::PublicInternet, - pi, - false, - ), + RouteNode::NodeId(id) => rti.lookup_node_ref(routing_table.clone(), TypedKey::new(crypto_kind, id))?, + RouteNode::PeerInfo(pi) => { + Some(rti.register_node_with_peer_info( + routing_table.clone(), + RoutingDomain::PublicInternet, + pi, + false, + )?) + } }; if opt_first_hop.is_none() { // Can't reach this private route any more diff --git a/veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs b/veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs index b193c398..9c5fbafe 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs @@ -40,7 +40,7 @@ impl RouteSpecStoreContent { // Go through best route and resolve noderefs let mut hop_node_refs = Vec::with_capacity(rsd.hops.len()); for h in &rsd.hops { - let Some(nr) = routing_table.lookup_node_ref(TypedKey::new(rsd.crypto_kind, *h)) else { + let Ok(Some(nr)) = routing_table.lookup_node_ref(TypedKey::new(rsd.crypto_kind, *h)) else { dead_ids.push(rsid.clone()); break; }; diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index eea23a83..e7b8cd05 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -651,14 +651,13 @@ impl RoutingTableInner { outer_self: RoutingTable, node_ids: &TypedKeySet, update_func: F, - ) -> Option + ) -> EyreResult where F: FnOnce(&mut RoutingTableInner, &mut BucketEntryInner), { // Ensure someone isn't trying register this node itself if self.unlocked_inner.matches_own_node_id(node_ids) { - log_rtab!(debug "can't register own node"); - return None; + bail!("can't register own node"); } // Look up all bucket entries and make sure we only have zero or one @@ -688,8 +687,7 @@ impl RoutingTableInner { if let Some(best_entry) = best_entry { // Update the entry with all of the node ids if let Err(e) = self.update_bucket_entries(best_entry.clone(), node_ids) { - log_rtab!(debug "Not registering new ids for existing node: {}", e); - return None; + bail!("Not registering new ids for existing node: {}", e); } // Make a noderef to return @@ -699,7 +697,7 @@ impl RoutingTableInner { best_entry.with_mut_inner(|e| update_func(self, e)); // Return the noderef - return Some(nr); + return Ok(nr); } // If no entry exists yet, add the first entry to a bucket, possibly evicting a bucket member @@ -712,8 +710,7 @@ impl RoutingTableInner { // Update the other bucket entries with the remaining node ids if let Err(e) = self.update_bucket_entries(new_entry.clone(), node_ids) { - log_rtab!(debug "Not registering new node: {}", e); - return None; + bail!("Not registering new node: {}", e); } // Make node ref to return @@ -725,7 +722,7 @@ impl RoutingTableInner { // Kick the bucket log_rtab!(debug "Routing table now has {} nodes, {} live", self.bucket_entry_count(), self.get_entry_count(RoutingDomainSet::all(), BucketEntryState::Unreliable, &VALID_CRYPTO_KINDS)); - Some(nr) + Ok(nr) } /// Resolve an existing routing table entry using any crypto kind and return a reference to it @@ -733,28 +730,35 @@ impl RoutingTableInner { &self, outer_self: RoutingTable, node_id_key: PublicKey, - ) -> Option { - VALID_CRYPTO_KINDS.iter().find_map(|ck| { - self.lookup_node_ref(outer_self.clone(), TypedKey::new(*ck, node_id_key)) - }) + ) -> EyreResult> { + for ck in VALID_CRYPTO_KINDS { + if let Some(nr) = + self.lookup_node_ref(outer_self.clone(), TypedKey::new(ck, node_id_key))? + { + return Ok(Some(nr)); + } + } + Ok(None) } /// Resolve an existing routing table entry and return a reference to it - pub fn lookup_node_ref(&self, outer_self: RoutingTable, node_id: TypedKey) -> Option { + pub fn lookup_node_ref( + &self, + outer_self: RoutingTable, + node_id: TypedKey, + ) -> EyreResult> { if self.unlocked_inner.matches_own_node_id(&[node_id]) { - log_rtab!(error "can't look up own node id in routing table"); - return None; + bail!("can't look up own node id in routing table"); } if !VALID_CRYPTO_KINDS.contains(&node_id.kind) { - log_rtab!(error "can't look up node id with invalid crypto kind"); - return None; + bail!("can't look up node id with invalid crypto kind"); } let bucket_index = self.unlocked_inner.calculate_bucket_index(&node_id); let bucket = self.get_bucket(bucket_index); - bucket + Ok(bucket .entry(&node_id.value) - .map(|e| NodeRef::new(outer_self, e, None)) + .map(|e| NodeRef::new(outer_self, e, None))) } /// Resolve an existing routing table entry and return a filtered reference to it @@ -764,15 +768,15 @@ impl RoutingTableInner { node_id: TypedKey, routing_domain_set: RoutingDomainSet, dial_info_filter: DialInfoFilter, - ) -> Option { + ) -> EyreResult> { let nr = self.lookup_node_ref(outer_self, node_id)?; - Some( + Ok(nr.map(|nr| { nr.filtered_clone( NodeRefFilter::new() .with_dial_info_filter(dial_info_filter) .with_routing_domain_set(routing_domain_set), - ), - ) + ) + })) } /// Resolve an existing routing table entry and call a function on its entry without using a noderef @@ -802,50 +806,53 @@ impl RoutingTableInner { routing_domain: RoutingDomain, peer_info: PeerInfo, allow_invalid: bool, - ) -> Option { + ) -> EyreResult { // if our own node is in the list, then ignore it as we don't add ourselves to our own routing table if self .unlocked_inner .matches_own_node_id(peer_info.node_ids()) { - log_rtab!(debug "can't register own node id in routing table"); - return None; + bail!("can't register own node id in routing table"); } // node can not be its own relay let rids = peer_info.signed_node_info().relay_ids(); let nids = peer_info.node_ids(); if nids.contains_any(&rids) { - log_rtab!(debug "node can not be its own relay"); - return None; + bail!("node can not be its own relay"); } if !allow_invalid { // verify signature if !peer_info.signed_node_info().has_any_signature() { - log_rtab!(debug "signed node info for {:?} has no valid signature", peer_info.node_ids()); - return None; + bail!( + "signed node info for {:?} has no valid signature", + peer_info.node_ids() + ); } // verify signed node info is valid in this routing domain if !self.signed_node_info_is_valid_in_routing_domain( routing_domain, peer_info.signed_node_info(), ) { - log_rtab!(debug "signed node info for {:?} not valid in the {:?} routing domain", peer_info.node_ids(), routing_domain); - return None; + bail!( + "signed node info for {:?} not valid in the {:?} routing domain", + peer_info.node_ids(), + routing_domain + ); } } let (node_ids, signed_node_info) = peer_info.destructure(); - self.create_node_ref(outer_self, &node_ids, |_rti, e| { + let mut nr = self.create_node_ref(outer_self, &node_ids, |_rti, e| { e.update_signed_node_info(routing_domain, signed_node_info); - }) - .map(|mut nr| { - nr.set_filter(Some( - NodeRefFilter::new().with_routing_domain(routing_domain), - )); - nr - }) + })?; + + nr.set_filter(Some( + NodeRefFilter::new().with_routing_domain(routing_domain), + )); + + Ok(nr) } /// Shortcut function to add a node to our routing table if it doesn't exist @@ -856,17 +863,15 @@ impl RoutingTableInner { node_id: TypedKey, descriptor: ConnectionDescriptor, timestamp: Timestamp, - ) -> Option { - let out = self.create_node_ref(outer_self, &TypedKeySet::from(node_id), |_rti, e| { + ) -> EyreResult { + let nr = self.create_node_ref(outer_self, &TypedKeySet::from(node_id), |_rti, e| { // this node is live because it literally just connected to us e.touch_last_seen(timestamp); - }); - if let Some(nr) = &out { - // set the most recent node address for connection finding and udp replies - nr.locked_mut(self) - .set_last_connection(descriptor, timestamp); - } - out + })?; + // set the most recent node address for connection finding and udp replies + nr.locked_mut(self) + .set_last_connection(descriptor, timestamp); + Ok(nr) } ////////////////////////////////////////////////////////////////////// diff --git a/veilid-core/src/routing_table/tasks/bootstrap.rs b/veilid-core/src/routing_table/tasks/bootstrap.rs index 25a51416..f811e4aa 100644 --- a/veilid-core/src/routing_table/tasks/bootstrap.rs +++ b/veilid-core/src/routing_table/tasks/bootstrap.rs @@ -259,19 +259,27 @@ impl RoutingTable { // Got peer info, let's add it to the routing table for pi in peer_info { // Register the node - if let Some(nr) = - self.register_node_with_peer_info(RoutingDomain::PublicInternet, pi, false) - { - // Add this our futures to process in parallel - for crypto_kind in VALID_CRYPTO_KINDS { - let routing_table = self.clone(); - let nr = nr.clone(); - unord.push( - // lets ask bootstrap to find ourselves now - async move { routing_table.reverse_find_node(crypto_kind, nr, true).await } - .instrument(Span::current()), - ); + let nr = match self.register_node_with_peer_info( + RoutingDomain::PublicInternet, + pi, + false, + ) { + Ok(nr) => nr, + Err(e) => { + log_rtab!(error "failed to register direct bootstrap peer info: {}", e); + continue; } + }; + + // Add this our futures to process in parallel + for crypto_kind in VALID_CRYPTO_KINDS { + let routing_table = self.clone(); + let nr = nr.clone(); + unord.push( + // lets ask bootstrap to find ourselves now + async move { routing_table.reverse_find_node(crypto_kind, nr, true).await } + .instrument(Span::current()), + ); } } } @@ -341,44 +349,46 @@ impl RoutingTable { let pi = PeerInfo::new(bsrec.node_ids, sni); - if let Some(nr) = - self.register_node_with_peer_info(RoutingDomain::PublicInternet, pi, true) - { - // Add this our futures to process in parallel - for crypto_kind in VALID_CRYPTO_KINDS { - // Do we need to bootstrap this crypto kind? - let eckey = (RoutingDomain::PublicInternet, crypto_kind); - let cnt = entry_count.get(&eckey).copied().unwrap_or_default(); - if cnt != 0 { + let nr = + match self.register_node_with_peer_info(RoutingDomain::PublicInternet, pi, true) { + Ok(nr) => nr, + Err(e) => { + log_rtab!(error "failed to register bootstrap peer info: {}", e); continue; } - - // Bootstrap this crypto kind - let nr = nr.clone(); - let routing_table = self.clone(); - unord.push( - async move { - // Need VALID signed peer info, so ask bootstrap to find_node of itself - // which will ensure it has the bootstrap's signed peer info as part of the response - let _ = routing_table.find_target(crypto_kind, nr.clone()).await; - - // Ensure we got the signed peer info - if !nr - .signed_node_info_has_valid_signature(RoutingDomain::PublicInternet) - { - log_rtab!(warn - "bootstrap at {:?} did not return valid signed node info", - nr - ); - // If this node info is invalid, it will time out after being unpingable - } else { - // otherwise this bootstrap is valid, lets ask it to find ourselves now - routing_table.reverse_find_node(crypto_kind, nr, true).await - } - } - .instrument(Span::current()), - ); + }; + // Add this our futures to process in parallel + for crypto_kind in VALID_CRYPTO_KINDS { + // Do we need to bootstrap this crypto kind? + let eckey = (RoutingDomain::PublicInternet, crypto_kind); + let cnt = entry_count.get(&eckey).copied().unwrap_or_default(); + if cnt != 0 { + continue; } + + // Bootstrap this crypto kind + let nr = nr.clone(); + let routing_table = self.clone(); + unord.push( + async move { + // Need VALID signed peer info, so ask bootstrap to find_node of itself + // which will ensure it has the bootstrap's signed peer info as part of the response + let _ = routing_table.find_target(crypto_kind, nr.clone()).await; + + // Ensure we got the signed peer info + if !nr.signed_node_info_has_valid_signature(RoutingDomain::PublicInternet) { + log_rtab!(warn + "bootstrap at {:?} did not return valid signed node info", + nr + ); + // If this node info is invalid, it will time out after being unpingable + } else { + // otherwise this bootstrap is valid, lets ask it to find ourselves now + routing_table.reverse_find_node(crypto_kind, nr, true).await + } + } + .instrument(Span::current()), + ); } } diff --git a/veilid-core/src/routing_table/tasks/relay_management.rs b/veilid-core/src/routing_table/tasks/relay_management.rs index 3bc93145..8d980c04 100644 --- a/veilid-core/src/routing_table/tasks/relay_management.rs +++ b/veilid-core/src/routing_table/tasks/relay_management.rs @@ -51,14 +51,19 @@ impl RoutingTable { // The outbound relay is the host of the PWA if let Some(outbound_relay_peerinfo) = intf::get_outbound_relay_peer().await { // Register new outbound relay - if let Some(nr) = self.register_node_with_peer_info( + match self.register_node_with_peer_info( RoutingDomain::PublicInternet, outbound_relay_peerinfo, false, ) { - info!("Outbound relay node selected: {}", nr); - editor.set_relay_node(nr); - got_outbound_relay = true; + Ok(nr) => { + log_rtab!("Outbound relay node selected: {}", nr); + editor.set_relay_node(nr); + got_outbound_relay = true; + } + Err(e) => { + log_rtab!(error "failed to register node with peer info: {}", e); + } } } } diff --git a/veilid-core/src/rpc_processor/destination.rs b/veilid-core/src/rpc_processor/destination.rs index ae19a4b0..e30052c8 100644 --- a/veilid-core/src/rpc_processor/destination.rs +++ b/veilid-core/src/rpc_processor/destination.rs @@ -312,7 +312,12 @@ impl RPCProcessor { NetworkResult::value(Destination::direct(peer_noderef)) } else { // Look up the sender node, we should have added it via senderNodeInfo before getting here. - if let Some(sender_noderef) = self.routing_table.lookup_node_ref(sender_node_id) { + let res = match self.routing_table.lookup_node_ref(sender_node_id) { + Ok(v) => v, + Err(e) => return NetworkResult::invalid_message( + format!("failed to look up node info for respond to: {}", e) + )}; + if let Some(sender_noderef) = res { NetworkResult::value(Destination::relay(peer_noderef, sender_noderef)) } else { return NetworkResult::invalid_message( diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index daf8ff80..805b62e4 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -480,7 +480,10 @@ impl RPCProcessor { let routing_table = this.routing_table(); // First see if we have the node in our routing table already - if let Some(nr) = routing_table.lookup_node_ref(node_id) { + if let Some(nr) = routing_table + .lookup_node_ref(node_id) + .map_err(RPCError::internal)? + { // ensure we have some dial info for the entry already, // if not, we should do the find_node anyway if nr.has_any_dial_info() { @@ -1346,20 +1349,26 @@ impl RPCProcessor { // Sender PeerInfo was specified, update our routing table with it if !self.filter_node_info(routing_domain, sender_peer_info.signed_node_info()) { - return Err(RPCError::invalid_format( + return Ok(NetworkResult::invalid_message( "sender peerinfo has invalid peer scope", )); } - opt_sender_nr = self.routing_table().register_node_with_peer_info( + opt_sender_nr = match self.routing_table().register_node_with_peer_info( routing_domain, sender_peer_info.clone(), false, - ); + ) { + Ok(v) => Some(v), + Err(e) => return Ok(NetworkResult::invalid_message(e)), + } } // look up sender node, in case it's different than our peer due to relaying if opt_sender_nr.is_none() { - opt_sender_nr = self.routing_table().lookup_node_ref(sender_node_id) + opt_sender_nr = match self.routing_table().lookup_node_ref(sender_node_id) { + Ok(v) => v, + Err(e) => return Ok(NetworkResult::invalid_message(e)), + } } // Update the 'seen our node info' timestamp to determine if this node needs a diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 7151b680..7c95e0c9 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -217,9 +217,9 @@ fn get_node_ref(routing_table: RoutingTable) -> impl FnOnce(&str) -> Option"] readme = "README.md" -packages = [{include = "veilid_python"}] +packages = [{include = "veilid"}] [tool.poetry.dependencies] python = "^3.11" diff --git a/veilid-python/tests/__init__.py b/veilid-python/tests/__init__.py index e26f2d5f..7f5c42bb 100644 --- a/veilid-python/tests/__init__.py +++ b/veilid-python/tests/__init__.py @@ -1,7 +1,10 @@ +from typing import Callable, Awaitable +import os import pytest pytest_plugins = ('pytest_asyncio',) -import os +import veilid + ################################################################## VEILID_SERVER = os.getenv("VEILID_SERVER") @@ -18,5 +21,10 @@ else: ################################################################## -async def simple_update_callback(update): +async def simple_connect_and_run(func: Callable[[veilid.VeilidAPI], Awaitable]): + api = await veilid.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) + async with api: + await func(api) + +async def simple_update_callback(update: veilid.VeilidUpdate): print("VeilidUpdate: {}".format(update)) diff --git a/veilid-python/tests/test_basic.py b/veilid-python/tests/test_basic.py index d74b25b2..6b083390 100644 --- a/veilid-python/tests/test_basic.py +++ b/veilid-python/tests/test_basic.py @@ -1,6 +1,6 @@ -# Basic veilid_python tests +# Basic veilid tests -import veilid_python +import veilid import pytest from . import * @@ -8,19 +8,22 @@ from . import * @pytest.mark.asyncio async def test_connect(): - async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: + async def func(api: veilid.VeilidAPI): pass + await simple_connect_and_run(func) @pytest.mark.asyncio async def test_fail_connect(): with pytest.raises(Exception): - async with await veilid_python.json_api_connect("fuahwelifuh32luhwafluehawea", 1, simple_update_callback) as api: + api = await veilid.json_api_connect("fuahwelifuh32luhwafluehawea", 1, simple_update_callback) + async with api: pass @pytest.mark.asyncio async def test_version(): - async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: + async def func(api: veilid.VeilidAPI): v = await api.veilid_version() print("veilid_version: {}".format(v.__dict__)) vstr = await api.veilid_version_string() print("veilid_version_string: {}".format(vstr)) + await simple_connect_and_run(func) diff --git a/veilid-python/tests/test_crypto.py b/veilid-python/tests/test_crypto.py index 6ceb993f..489beb20 100644 --- a/veilid-python/tests/test_crypto.py +++ b/veilid-python/tests/test_crypto.py @@ -1,6 +1,6 @@ -# Crypto veilid_python tests +# Crypto veilid tests -import veilid_python +import veilid import pytest from . import * @@ -8,21 +8,35 @@ from . import * @pytest.mark.asyncio async def test_best_crypto_system(): - async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: + async def func(api: veilid.VeilidAPI): bcs = await api.best_crypto_system() - # let handle dangle for test - # del bcs + await simple_connect_and_run(func) @pytest.mark.asyncio async def test_get_crypto_system(): - async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: - cs = await api.get_crypto_system(veilid_python.CryptoKind.CRYPTO_KIND_VLD0) + async def func(api: veilid.VeilidAPI): + cs = await api.get_crypto_system(veilid.CryptoKind.CRYPTO_KIND_VLD0) # clean up handle early del cs + await simple_connect_and_run(func) @pytest.mark.asyncio async def test_get_crypto_system_invalid(): - async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api: - with pytest.raises(veilid_python.VeilidAPIError): - cs = await api.get_crypto_system(veilid_python.CryptoKind.CRYPTO_KIND_NONE) + async def func(api: veilid.VeilidAPI): + with pytest.raises(veilid.VeilidAPIError): + cs = await api.get_crypto_system(veilid.CryptoKind.CRYPTO_KIND_NONE) + await simple_connect_and_run(func) +@pytest.mark.asyncio +async def test_hash_and_verify_password(): + async def func(api: veilid.VeilidAPI): + bcs = await api.best_crypto_system() + nonce = await bcs.random_nonce() + salt = nonce.to_bytes() + # Password match + phash = await bcs.hash_password(b"abc123", salt) + assert await bcs.verify_password(b"abc123", phash) + # Password mismatch + phash2 = await bcs.hash_password(b"abc1234", salt) + assert not await bcs.verify_password(b"abc12345", phash) + await simple_connect_and_run(func) diff --git a/veilid-python/tests/test_routing_context.py b/veilid-python/tests/test_routing_context.py new file mode 100644 index 00000000..4b782ecb --- /dev/null +++ b/veilid-python/tests/test_routing_context.py @@ -0,0 +1,47 @@ +# Routing context veilid tests + +import veilid +import pytest +import asyncio +import json +from . import * + +################################################################## + +@pytest.mark.asyncio +async def test_routing_contexts(): + async def func(api: veilid.VeilidAPI): + rc = await api.new_routing_context() + rcp = await rc.with_privacy() + rcps = await rcp.with_sequencing(veilid.Sequencing.ENSURE_ORDERED) + rcpsr = await rcps.with_custom_privacy(veilid.Stability.RELIABLE) + await simple_connect_and_run(func) + +@pytest.mark.asyncio +async def test_routing_context_app_message_loopback(): + + app_message_queue = asyncio.Queue() + + async def app_message_queue_update_callback(update: veilid.VeilidUpdate): + if update.kind == veilid.VeilidUpdateKind.APP_MESSAGE: + await app_message_queue.put(update) + + api = await veilid.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, app_message_queue_update_callback) + async with api: + + # make a routing context that uses a safety route + rc = await (await api.new_routing_context()).with_privacy() + + # get our own node id + state = await api.get_state() + node_id = state.config.config.network.routing_table.node_id.pop() + + # send an app message to our node id + message = b"abcd1234" + await rc.app_message(node_id, message) + + # we should get the same message back + #update: veilid.VeilidUpdate = await asyncio.wait_for(app_message_queue.get(), timeout=10) + #appmsg: veilid.VeilidAppMessage = update.detail + #assert appmsg.message == message + diff --git a/veilid-python/update_schema.sh b/veilid-python/update_schema.sh index ad2f59df..41f2366c 100755 --- a/veilid-python/update_schema.sh +++ b/veilid-python/update_schema.sh @@ -11,7 +11,7 @@ if [ ! -f "$VEILID_SERVER" ]; then fi # Produce schema from veilid-server -$VEILID_SERVER --emit-schema Request > $SCRIPTDIR/veilid_python/schema/Request.json -$VEILID_SERVER --emit-schema RecvMessage > $SCRIPTDIR/veilid_python/schema/RecvMessage.json +$VEILID_SERVER --emit-schema Request > $SCRIPTDIR/veilid/schema/Request.json +$VEILID_SERVER --emit-schema RecvMessage > $SCRIPTDIR/veilid/schema/RecvMessage.json diff --git a/veilid-python/veilid_python/__init__.py b/veilid-python/veilid/__init__.py similarity index 100% rename from veilid-python/veilid_python/__init__.py rename to veilid-python/veilid/__init__.py diff --git a/veilid-python/veilid_python/api.py b/veilid-python/veilid/api.py similarity index 100% rename from veilid-python/veilid_python/api.py rename to veilid-python/veilid/api.py diff --git a/veilid-python/veilid_python/config.py b/veilid-python/veilid/config.py similarity index 98% rename from veilid-python/veilid_python/config.py rename to veilid-python/veilid/config.py index d44bc11a..d9394c4e 100644 --- a/veilid-python/veilid_python/config.py +++ b/veilid-python/veilid/config.py @@ -2,6 +2,8 @@ from typing import Self, Optional from enum import StrEnum from json import dumps +from .types import * + class VeilidConfigLogLevel(StrEnum): OFF = 'Off' ERROR = 'Error' @@ -96,8 +98,8 @@ class VeilidConfigBlockStore: return self.__dict__ class VeilidConfigRoutingTable: - node_id: list[str] - node_id_secret: list[str] + node_id: list[TypedKey] + node_id_secret: list[TypedSecret] bootstrap: list[str] limit_over_attached: int limit_fully_attached: int @@ -105,7 +107,7 @@ class VeilidConfigRoutingTable: limit_attached_good: int limit_attached_weak: int - def __init__(self, node_id: list[str], node_id_secret: list[str], bootstrap: list[str], limit_over_attached: int, + def __init__(self, node_id: list[TypedKey], node_id_secret: list[TypedSecret], bootstrap: list[str], limit_over_attached: int, limit_fully_attached: int, limit_attached_strong: int, limit_attached_good: int, limit_attached_weak: int): self.node_id = node_id @@ -120,8 +122,8 @@ class VeilidConfigRoutingTable: @staticmethod def from_json(j: dict) -> Self: return VeilidConfigRoutingTable( - j['node_id'], - j['node_id_secret'], + list(map(lambda x: TypedKey(x), j['node_id'])), + list(map(lambda x: TypedSecret(x), j['node_id_secret'])), j['bootstrap'], j['limit_over_attached'], j['limit_fully_attached'], diff --git a/veilid-python/veilid_python/error.py b/veilid-python/veilid/error.py similarity index 100% rename from veilid-python/veilid_python/error.py rename to veilid-python/veilid/error.py diff --git a/veilid-python/veilid_python/json_api.py b/veilid-python/veilid/json_api.py similarity index 95% rename from veilid-python/veilid_python/json_api.py rename to veilid-python/veilid/json_api.py index 7ec5fff8..5b0d0fc3 100644 --- a/veilid-python/veilid_python/json_api.py +++ b/veilid-python/veilid/json_api.py @@ -2,7 +2,7 @@ import json import asyncio from jsonschema import validators, exceptions -from typing import Callable, Awaitable +from typing import Callable, Awaitable, Mapping from .api import * from .state import * @@ -42,7 +42,7 @@ class _JsonVeilidAPI(VeilidAPI): # Shared Mutable State lock: asyncio.Lock next_id: int - in_flight_requests: dict + in_flight_requests: Mapping[str, asyncio.Future] def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, update_callback: Callable[[VeilidUpdate], Awaitable], validate_schema: bool = True): self.reader = reader @@ -54,7 +54,7 @@ class _JsonVeilidAPI(VeilidAPI): self.next_id = 1 self.in_flight_requests = dict() - async def __aenter__(self): + async def __aenter__(self) -> Self: return self async def __aexit__(self, *excinfo): @@ -67,6 +67,10 @@ class _JsonVeilidAPI(VeilidAPI): self.writer.close() await self.writer.wait_closed() self.writer = None + + for (reqid, reqfuture) in self.in_flight_requests.items(): + reqfuture.cancel() + finally: self.lock.release() @@ -103,7 +107,9 @@ class _JsonVeilidAPI(VeilidAPI): finally: self.lock.release() # Resolve the request's future to the response json - reqfuture.set_result(j) + if reqfuture is not None: + reqfuture.set_result(j) + async def handle_recv_messages(self): # Read lines until we're done @@ -124,8 +130,6 @@ class _JsonVeilidAPI(VeilidAPI): await self.handle_recv_message_response(j) elif j['type'] == "Update": await self.update_callback(VeilidUpdate.from_json(j)) - except: - pass finally: await self._cleanup_close() @@ -263,17 +267,17 @@ class _JsonVeilidAPI(VeilidAPI): cs_id = raise_api_result(await self.send_ndjson_request(Operation.BEST_CRYPTO_SYSTEM)) return _JsonCryptoSystem(self, cs_id) async def verify_signatures(self, node_ids: list[TypedKey], data: bytes, signatures: list[TypedSignature]) -> list[TypedKey]: - return map(lambda x: TypedKey(x), raise_api_result(await self.send_ndjson_request(Operation.VERIFY_SIGNATURES, + return list(map(lambda x: TypedKey(x), raise_api_result(await self.send_ndjson_request(Operation.VERIFY_SIGNATURES, node_ids = node_ids, data = data, - signatures = signatures))) + signatures = signatures)))) async def generate_signatures(self, data: bytes, key_pairs: list[TypedKeyPair]) -> list[TypedSignature]: - return map(lambda x: TypedSignature(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_SIGNATURES, + return list(map(lambda x: TypedSignature(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_SIGNATURES, data = data, - key_pairs = key_pairs))) + key_pairs = key_pairs)))) async def generate_key_pair(self, kind: CryptoKind) -> list[TypedKeyPair]: - return map(lambda x: TypedKeyPair(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_KEY_PAIR, - kind = kind))) + return list(map(lambda x: TypedKeyPair(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_KEY_PAIR, + kind = kind)))) async def now(self) -> Timestamp: return Timestamp(raise_api_result(await self.send_ndjson_request(Operation.NOW))) async def debug(self, command: str) -> str: @@ -456,10 +460,10 @@ class _JsonTableDb(TableDb): db_id = self.db_id, db_op = TableDbOperation.GET_COLUMN_COUNT)) async def get_keys(self, col: int) -> list[bytes]: - return map(lambda x: urlsafe_b64decode_no_pad(x), raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, + return list(map(lambda x: urlsafe_b64decode_no_pad(x), raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, db_id = self.db_id, db_op = TableDbOperation.GET_KEYS, - col = col))) + col = col)))) async def transact(self) -> TableDbTransaction: tx_id = raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op, db_id = self.db_id, @@ -627,6 +631,6 @@ class _JsonCryptoSystem(CryptoSystem): ###################################################### -def json_api_connect(host:str, port:int, update_callback: Callable[[VeilidUpdate], Awaitable]) -> VeilidAPI: - return _JsonVeilidAPI.connect(host, port, update_callback) +async def json_api_connect(host:str, port:int, update_callback: Callable[[VeilidUpdate], Awaitable]) -> VeilidAPI: + return await _JsonVeilidAPI.connect(host, port, update_callback) diff --git a/veilid-python/veilid_python/operations.py b/veilid-python/veilid/operations.py similarity index 100% rename from veilid-python/veilid_python/operations.py rename to veilid-python/veilid/operations.py diff --git a/veilid-python/veilid_python/schema/RecvMessage.json b/veilid-python/veilid/schema/RecvMessage.json similarity index 100% rename from veilid-python/veilid_python/schema/RecvMessage.json rename to veilid-python/veilid/schema/RecvMessage.json diff --git a/veilid-python/veilid_python/schema/Request.json b/veilid-python/veilid/schema/Request.json similarity index 100% rename from veilid-python/veilid_python/schema/Request.json rename to veilid-python/veilid/schema/Request.json diff --git a/veilid-python/veilid_python/state.py b/veilid-python/veilid/state.py similarity index 96% rename from veilid-python/veilid_python/state.py rename to veilid-python/veilid/state.py index a5fd9cd8..13404754 100644 --- a/veilid-python/veilid_python/state.py +++ b/veilid-python/veilid/state.py @@ -181,7 +181,7 @@ class VeilidStateNetwork: j['started'], ByteCount(j['bps_down']), ByteCount(j['bps_up']), - map(lambda x: PeerTableData.from_json(x), j['peers'])) + list(map(lambda x: PeerTableData.from_json(x), j['peers']))) class VeilidStateConfig: config: VeilidConfig @@ -193,7 +193,8 @@ class VeilidStateConfig: def from_json(j: dict) -> Self: '''JSON object hook''' return VeilidStateConfig( - j['config']) + VeilidConfig.from_json(j['config']) + ) class VeilidState: attachment: VeilidStateAttachment @@ -227,7 +228,7 @@ class VeilidLog: def from_json(j: dict) -> Self: '''JSON object hook''' return VeilidLog( - VeilidLogLevel(j['attachment']), + VeilidLogLevel(j['log_level']), j['message'], j['backtrace']) @@ -276,8 +277,8 @@ class VeilidRouteChange: def from_json(j: dict) -> Self: '''JSON object hook''' return VeilidRouteChange( - map(lambda x: RouteId(x), j['dead_routes']), - map(lambda x: RouteId(x), j['dead_remote_routes'])) + list(map(lambda x: RouteId(x), j['dead_routes'])), + list(map(lambda x: RouteId(x), j['dead_remote_routes']))) class VeilidValueChange: key: TypedKey @@ -296,7 +297,7 @@ class VeilidValueChange: '''JSON object hook''' return VeilidValueChange( TypedKey(j['key']), - map(lambda x: ValueSubkey(x), j['subkeys']), + list(map(lambda x: ValueSubkey(x), j['subkeys'])), j['count'], ValueData.from_json(j['value'])) @@ -346,4 +347,4 @@ class VeilidUpdate: detail = None case _: raise ValueError("Unknown VeilidUpdateKind") - + return VeilidUpdate(kind, detail) diff --git a/veilid-python/veilid_python/types.py b/veilid-python/veilid/types.py similarity index 97% rename from veilid-python/veilid_python/types.py rename to veilid-python/veilid/types.py index 0f4ced5e..b3388531 100644 --- a/veilid-python/veilid_python/types.py +++ b/veilid-python/veilid/types.py @@ -11,8 +11,7 @@ def urlsafe_b64encode_no_pad(b: bytes) -> str: """ Removes any `=` used as padding from the encoded string. """ - encoded = str(base64.urlsafe_b64encode(b)) - return encoded.rstrip("=") + return base64.urlsafe_b64encode(b).decode().rstrip("=") def urlsafe_b64decode_no_pad(s: str) -> bytes: @@ -20,7 +19,7 @@ def urlsafe_b64decode_no_pad(s: str) -> bytes: Adds back in the required padding before decoding. """ padding = 4 - (len(s) % 4) - string = string + ("=" * padding) + s = s + ("=" * padding) return base64.urlsafe_b64decode(s) class VeilidJSONEncoder(json.JSONEncoder): @@ -248,7 +247,7 @@ class DHTSchema: if DHTSchemaKind(j['kind']) == DHTSchemaKind.SMPL: return DHTSchema.smpl( j['o_cnt'], - map(lambda x: DHTSchemaSMPLMember.from_json(x), j['members'])) + list(map(lambda x: DHTSchemaSMPLMember.from_json(x), j['members']))) raise Exception("Unknown DHTSchema kind", j['kind']) def to_json(self) -> dict: diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index 6b2f9d55..d13b2a7d 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -238,18 +238,10 @@ impl ClientApi { async fn receive_requests( self, - conn_tuple: (SocketAddr, SocketAddr), mut reader: R, requests_tx: flume::Sender>, responses_tx: flume::Sender, ) -> VeilidAPIResult> { - // responses_tx becomes owned by recv_requests_future - // Start sending updates - self.inner - .lock() - .update_channels - .insert(conn_tuple, responses_tx.clone()); - let mut linebuf = String::new(); while let Ok(size) = reader.read_line(&mut linebuf).await { // Eof? @@ -277,10 +269,6 @@ impl ClientApi { } } - // Stop sending updates - // Will cause send_responses_future to stop because we drop the responses_tx - self.inner.lock().update_channels.remove(&conn_tuple); - VeilidAPIResult::Ok(None) } @@ -290,8 +278,8 @@ impl ClientApi { mut writer: W, ) -> VeilidAPIResult> { while let Ok(resp) = responses_rx.recv_async().await { - if let Err(e) = writer.write_all(resp.as_bytes()).await { - eprintln!("failed to write response: {}", e) + if let Err(_) = writer.write_all(resp.as_bytes()).await { + break; } } VeilidAPIResult::Ok(None) @@ -350,11 +338,16 @@ impl ClientApi { let (requests_tx, requests_rx) = flume::unbounded(); let (responses_tx, responses_rx) = flume::unbounded(); + // Start sending updates + self.inner + .lock() + .update_channels + .insert(conn_tuple, responses_tx.clone()); + // Request receive processor future // Receives from socket and enqueues RequestLines // Completes when the connection is closed or there is a failure unord.push(system_boxed(self.clone().receive_requests( - conn_tuple, reader, requests_tx, responses_tx, @@ -398,6 +391,9 @@ impl ClientApi { )); } + // Stop sending updates + self.inner.lock().update_channels.remove(&conn_tuple); + debug!( "Closed Client API Connection: {:?} -> {:?}", peer_addr, local_addr @@ -414,8 +410,8 @@ impl ClientApi { // Pass other updates to clients let inner = self.inner.lock(); for ch in inner.update_channels.values() { - if let Err(e) = ch.send(veilid_update.clone()) { - eprintln!("failed to send update: {}", e); + if let Err(_) = ch.send(veilid_update.clone()) { + // eprintln!("failed to send update: {}", e); } } } From d114ea3b725c9cadfb03c3e9b3f3047e21c2611e Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 15 Jun 2023 20:29:12 -0400 Subject: [PATCH 18/23] crufy --- veilid-core/src/core_context.rs | 1 - veilid-core/src/routing_table/tests/test_serialize.rs | 1 - veilid-core/src/storage_manager/mod.rs | 5 ----- 3 files changed, 7 deletions(-) diff --git a/veilid-core/src/core_context.rs b/veilid-core/src/core_context.rs index 630f7375..ba128687 100644 --- a/veilid-core/src/core_context.rs +++ b/veilid-core/src/core_context.rs @@ -123,7 +123,6 @@ impl ServicesContext { let storage_manager = StorageManager::new( self.config.clone(), self.crypto.clone().unwrap(), - self.protected_store.clone().unwrap(), self.table_store.clone().unwrap(), #[cfg(feature = "unstable-blockstore")] self.block_store.clone().unwrap(), diff --git a/veilid-core/src/routing_table/tests/test_serialize.rs b/veilid-core/src/routing_table/tests/test_serialize.rs index b3609745..5001770c 100644 --- a/veilid-core/src/routing_table/tests/test_serialize.rs +++ b/veilid-core/src/routing_table/tests/test_serialize.rs @@ -10,7 +10,6 @@ fn fake_routing_table() -> routing_table::RoutingTable { let storage_manager = storage_manager::StorageManager::new( veilid_config.clone(), crypto.clone(), - protected_store.clone(), table_store.clone(), #[cfg(feature = "unstable-blockstore")] block_store.clone(), diff --git a/veilid-core/src/storage_manager/mod.rs b/veilid-core/src/storage_manager/mod.rs index 42f30d94..64a42530 100644 --- a/veilid-core/src/storage_manager/mod.rs +++ b/veilid-core/src/storage_manager/mod.rs @@ -28,7 +28,6 @@ const FLUSH_RECORD_STORES_INTERVAL_SECS: u32 = 1; struct StorageManagerUnlockedInner { config: VeilidConfig, crypto: Crypto, - protected_store: ProtectedStore, table_store: TableStore, #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, @@ -47,14 +46,12 @@ impl StorageManager { fn new_unlocked_inner( config: VeilidConfig, crypto: Crypto, - protected_store: ProtectedStore, table_store: TableStore, #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, ) -> StorageManagerUnlockedInner { StorageManagerUnlockedInner { config, crypto, - protected_store, table_store, #[cfg(feature = "unstable-blockstore")] block_store, @@ -68,14 +65,12 @@ impl StorageManager { pub fn new( config: VeilidConfig, crypto: Crypto, - protected_store: ProtectedStore, table_store: TableStore, #[cfg(feature = "unstable-blockstore")] block_store: BlockStore, ) -> StorageManager { let unlocked_inner = Arc::new(Self::new_unlocked_inner( config, crypto, - protected_store, table_store, #[cfg(feature = "unstable-blockstore")] block_store, From 14ba85efdaea942d56f027e81d573daff0a783b1 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 16 Jun 2023 11:57:55 -0400 Subject: [PATCH 19/23] fixes --- veilid-cli/src/command_processor.rs | 2 +- .../route_spec_store/route_spec_store.rs | 6 +- veilid-core/src/rpc_processor/rpc_app_call.rs | 8 ++- veilid-core/src/veilid_api/api.rs | 8 ++- veilid-core/src/veilid_api/debug.rs | 66 ++++++++++++------- .../src/veilid_api/types/app_message_call.rs | 8 +-- veilid-flutter/lib/veilid_ffi.dart | 6 +- veilid-flutter/lib/veilid_js.dart | 4 +- veilid-flutter/lib/veilid_state.dart | 12 ++-- veilid-flutter/rust/src/dart_ffi.rs | 10 +-- veilid-python/tests/test_basic.py | 10 +++ veilid-python/tests/test_routing_context.py | 56 +++++++++++++--- veilid-python/veilid/api.py | 4 +- veilid-python/veilid/json_api.py | 8 +-- veilid-python/veilid/schema/RecvMessage.json | 4 +- veilid-python/veilid/state.py | 8 +-- veilid-python/veilid/types.py | 3 + veilid-wasm/src/lib.rs | 10 +-- 18 files changed, 158 insertions(+), 75 deletions(-) diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index 636561d1..dcf0a8cb 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -488,7 +488,7 @@ reply - reply to an AppCall not handled directly by the server format!("#{}", hex::encode(&message)) }; - let id = json_str_u64(&call["id"]); + let id = json_str_u64(&call["call_id"]); self.inner().ui_sender.add_node_event(format!( "AppCall ({:?}) id = {:016x} : {}", diff --git a/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs index 043cd1e3..db490b08 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs @@ -1331,9 +1331,9 @@ impl RouteSpecStore { } // ensure this isn't also an allocated route - if inner.content.get_id_by_key(&private_route.public_key.value).is_some() { - bail!("should not import allocated route"); - } + // if inner.content.get_id_by_key(&private_route.public_key.value).is_some() { + // bail!("should not import allocated route"); + // } } inner.cache.cache_remote_private_route(cur_ts, id, private_routes); diff --git a/veilid-core/src/rpc_processor/rpc_app_call.rs b/veilid-core/src/rpc_processor/rpc_app_call.rs index 80919d4c..55399a0a 100644 --- a/veilid-core/src/rpc_processor/rpc_app_call.rs +++ b/veilid-core/src/rpc_processor/rpc_app_call.rs @@ -99,10 +99,14 @@ impl RPCProcessor { } /// Exposed to API for apps to return app call answers - pub async fn app_call_reply(&self, id: OperationId, message: Vec) -> Result<(), RPCError> { + pub async fn app_call_reply( + &self, + call_id: OperationId, + message: Vec, + ) -> Result<(), RPCError> { self.unlocked_inner .waiting_app_call_table - .complete_op_waiter(id, message) + .complete_op_waiter(call_id, message) .await } } diff --git a/veilid-core/src/veilid_api/api.rs b/veilid-core/src/veilid_api/api.rs index 33d4ed91..e694f111 100644 --- a/veilid-core/src/veilid_api/api.rs +++ b/veilid-core/src/veilid_api/api.rs @@ -258,10 +258,14 @@ impl VeilidAPI { // App Calls #[instrument(level = "debug", skip(self))] - pub async fn app_call_reply(&self, id: OperationId, message: Vec) -> VeilidAPIResult<()> { + pub async fn app_call_reply( + &self, + call_id: OperationId, + message: Vec, + ) -> VeilidAPIResult<()> { let rpc_processor = self.rpc_processor()?; rpc_processor - .app_call_reply(id, message) + .app_call_reply(call_id, message) .await .map_err(|e| e.into()) } diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 7c95e0c9..bdc269ee 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -31,16 +31,22 @@ fn get_string(text: &str) -> Option { Some(text.to_owned()) } -fn get_route_id(rss: RouteSpecStore, allow_remote: bool) -> impl Fn(&str) -> Option { +fn get_route_id( + rss: RouteSpecStore, + allow_allocated: bool, + allow_remote: bool, +) -> impl Fn(&str) -> Option { return move |text: &str| { if text.is_empty() { return None; } match RouteId::from_str(text).ok() { Some(key) => { - let routes = rss.list_allocated_routes(|k, _| Some(*k)); - if routes.contains(&key) { - return Some(key); + if allow_allocated { + let routes = rss.list_allocated_routes(|k, _| Some(*k)); + if routes.contains(&key) { + return Some(key); + } } if allow_remote { let rroutes = rss.list_remote_routes(|k, _| Some(*k)); @@ -50,11 +56,13 @@ fn get_route_id(rss: RouteSpecStore, allow_remote: bool) -> impl Fn(&str) -> Opt } } None => { - let routes = rss.list_allocated_routes(|k, _| Some(*k)); - for r in routes { - let rkey = r.encode(); - if rkey.starts_with(text) { - return Some(r); + if allow_allocated { + let routes = rss.list_allocated_routes(|k, _| Some(*k)); + for r in routes { + let rkey = r.encode(); + if rkey.starts_with(text) { + return Some(r); + } } } if allow_remote { @@ -90,7 +98,7 @@ fn get_safety_selection(text: &str, routing_table: RoutingTable) -> Option impl FnOnce(&str) -> Option 2 { @@ -747,7 +765,7 @@ impl VeilidAPI { 1, "debug_route", "route_id", - get_route_id(rss.clone(), false), + get_route_id(rss.clone(), true, false), )?; // Unpublish route @@ -769,7 +787,7 @@ impl VeilidAPI { 1, "debug_route", "route_id", - get_route_id(rss.clone(), true), + get_route_id(rss.clone(), true, true), )?; match rss.debug_route(&route_id) { @@ -831,7 +849,7 @@ impl VeilidAPI { 1, "debug_route", "route_id", - get_route_id(rss.clone(), true), + get_route_id(rss.clone(), true, true), )?; let success = rss diff --git a/veilid-core/src/veilid_api/types/app_message_call.rs b/veilid-core/src/veilid_api/types/app_message_call.rs index fb33ae50..41f3bdff 100644 --- a/veilid-core/src/veilid_api/types/app_message_call.rs +++ b/veilid-core/src/veilid_api/types/app_message_call.rs @@ -67,15 +67,15 @@ pub struct VeilidAppCall { /// The id to reply to #[serde(with = "json_as_string")] #[schemars(with = "String")] - id: OperationId, + call_id: OperationId, } impl VeilidAppCall { - pub fn new(sender: Option, message: Vec, id: OperationId) -> Self { + pub fn new(sender: Option, message: Vec, call_id: OperationId) -> Self { Self { sender, message, - id, + call_id, } } @@ -86,6 +86,6 @@ impl VeilidAppCall { &self.message } pub fn id(&self) -> OperationId { - self.id + self.call_id } } diff --git a/veilid-flutter/lib/veilid_ffi.dart b/veilid-flutter/lib/veilid_ffi.dart index a0a9bdcd..f39773e8 100644 --- a/veilid-flutter/lib/veilid_ffi.dart +++ b/veilid-flutter/lib/veilid_ffi.dart @@ -1560,12 +1560,12 @@ class VeilidFFI implements Veilid { } @override - Future appCallReply(String id, Uint8List message) { - final nativeId = id.toNativeUtf8(); + Future appCallReply(String call_id, Uint8List message) { + final nativeCallId = call_id.toNativeUtf8(); final nativeEncodedMessage = base64UrlNoPadEncode(message).toNativeUtf8(); final recvPort = ReceivePort("app_call_reply"); final sendPort = recvPort.sendPort; - _appCallReply(sendPort.nativePort, nativeId, nativeEncodedMessage); + _appCallReply(sendPort.nativePort, nativeCallId, nativeEncodedMessage); return processFutureVoid(recvPort.first); } diff --git a/veilid-flutter/lib/veilid_js.dart b/veilid-flutter/lib/veilid_js.dart index 139c10a5..f1c5f84b 100644 --- a/veilid-flutter/lib/veilid_js.dart +++ b/veilid-flutter/lib/veilid_js.dart @@ -580,10 +580,10 @@ class VeilidJS implements Veilid { } @override - Future appCallReply(String id, Uint8List message) { + Future appCallReply(String callId, Uint8List message) { var encodedMessage = base64UrlNoPadEncode(message); return _wrapApiPromise( - js_util.callMethod(wasm, "app_call_reply", [id, encodedMessage])); + js_util.callMethod(wasm, "app_call_reply", [callId, encodedMessage])); } @override diff --git a/veilid-flutter/lib/veilid_state.dart b/veilid-flutter/lib/veilid_state.dart index 40c263c5..8d90dbff 100644 --- a/veilid-flutter/lib/veilid_state.dart +++ b/veilid-flutter/lib/veilid_state.dart @@ -262,7 +262,9 @@ abstract class VeilidUpdate { case "AppCall": { return VeilidAppCall( - sender: json["sender"], message: json["message"], id: json["id"]); + sender: json["sender"], + message: json["message"], + callId: json["call_id"]); } case "Attachment": { @@ -348,22 +350,22 @@ class VeilidAppMessage implements VeilidUpdate { class VeilidAppCall implements VeilidUpdate { final String? sender; final Uint8List message; - final String id; + final String callId; // VeilidAppCall({ required this.sender, required this.message, - required this.id, + required this.callId, }); @override Map toJson() { return { - 'kind': "AppMessage", + 'kind': "AppCall", 'sender': sender, 'message': base64UrlNoPadEncode(message), - 'id': id, + 'call_id': callId, }; } } diff --git a/veilid-flutter/rust/src/dart_ffi.rs b/veilid-flutter/rust/src/dart_ffi.rs index 238f1805..2a9b65a7 100644 --- a/veilid-flutter/rust/src/dart_ffi.rs +++ b/veilid-flutter/rust/src/dart_ffi.rs @@ -707,21 +707,21 @@ pub extern "C" fn release_private_route(port: i64, route_id: FfiStr) { } #[no_mangle] -pub extern "C" fn app_call_reply(port: i64, id: FfiStr, message: FfiStr) { - let id = id.into_opt_string().unwrap_or_default(); +pub extern "C" fn app_call_reply(port: i64, call_id: FfiStr, message: FfiStr) { + let call_id = call_id.into_opt_string().unwrap_or_default(); let message = message.into_opt_string().unwrap_or_default(); DartIsolateWrapper::new(port).spawn_result(async move { - let id = match id.parse() { + let call_id = match call_id.parse() { Ok(v) => v, Err(e) => { - return APIResult::Err(veilid_core::VeilidAPIError::invalid_argument(e, "id", id)) + return APIResult::Err(veilid_core::VeilidAPIError::invalid_argument(e, "call_id", call_id)) } }; let message = data_encoding::BASE64URL_NOPAD .decode(message.as_bytes()) .map_err(|e| veilid_core::VeilidAPIError::invalid_argument(e, "message", message))?; let veilid_api = get_veilid_api().await?; - veilid_api.app_call_reply(id, message).await?; + veilid_api.app_call_reply(call_id, message).await?; APIRESULT_VOID }); } diff --git a/veilid-python/tests/test_basic.py b/veilid-python/tests/test_basic.py index 6b083390..b2c44b23 100644 --- a/veilid-python/tests/test_basic.py +++ b/veilid-python/tests/test_basic.py @@ -12,6 +12,15 @@ async def test_connect(): pass await simple_connect_and_run(func) + +@pytest.mark.asyncio +async def test_get_node_id(): + async def func(api: veilid.VeilidAPI): + # get our own node id + state = await api.get_state() + node_id = state.config.config.network.routing_table.node_id.pop() + await simple_connect_and_run(func) + @pytest.mark.asyncio async def test_fail_connect(): with pytest.raises(Exception): @@ -27,3 +36,4 @@ async def test_version(): vstr = await api.veilid_version_string() print("veilid_version_string: {}".format(vstr)) await simple_connect_and_run(func) + diff --git a/veilid-python/tests/test_routing_context.py b/veilid-python/tests/test_routing_context.py index 4b782ecb..2ab8a579 100644 --- a/veilid-python/tests/test_routing_context.py +++ b/veilid-python/tests/test_routing_context.py @@ -32,16 +32,56 @@ async def test_routing_context_app_message_loopback(): # make a routing context that uses a safety route rc = await (await api.new_routing_context()).with_privacy() - # get our own node id - state = await api.get_state() - node_id = state.config.config.network.routing_table.node_id.pop() + # make a new local private route + prl, blob = await api.new_private_route() - # send an app message to our node id + # import it as a remote route as well so we can send to it + prr = await api.import_remote_private_route(blob) + + # send an app message to our own private route message = b"abcd1234" - await rc.app_message(node_id, message) + await rc.app_message(prr, message) # we should get the same message back - #update: veilid.VeilidUpdate = await asyncio.wait_for(app_message_queue.get(), timeout=10) - #appmsg: veilid.VeilidAppMessage = update.detail - #assert appmsg.message == message + update: veilid.VeilidUpdate = await asyncio.wait_for(app_message_queue.get(), timeout=10) + appmsg: veilid.VeilidAppMessage = update.detail + assert appmsg.message == message + +@pytest.mark.asyncio +async def test_routing_context_app_call_loopback(): + + app_call_queue = asyncio.Queue() + + async def app_call_queue_update_callback(update: veilid.VeilidUpdate): + if update.kind == veilid.VeilidUpdateKind.APP_CALL: + await app_call_queue.put(update) + + api = await veilid.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, app_call_queue_update_callback) + async with api: + + # make a routing context that uses a safety route + rc = await (await api.new_routing_context()).with_privacy() + + # make a new local private route + prl, blob = await api.new_private_route() + + # import it as a remote route as well so we can send to it + prr = await api.import_remote_private_route(blob) + + # send an app message to our own private route + request = b"abcd1234" + app_call_task = asyncio.create_task(rc.app_call(prr, request), name = "app call task") + + # we should get the same request back + update: veilid.VeilidUpdate = await asyncio.wait_for(app_call_queue.get(), timeout=10) + appcall: veilid.VeilidAppCall = update.detail + assert appcall.message == request + + # now we reply to the request + reply = b"qwer5678" + await api.app_call_reply(appcall.call_id, reply) + + # now we should get the reply from the call + result = await app_call_task + assert result == reply diff --git a/veilid-python/veilid/api.py b/veilid-python/veilid/api.py index 07134d43..f201a324 100644 --- a/veilid-python/veilid/api.py +++ b/veilid-python/veilid/api.py @@ -159,10 +159,10 @@ class VeilidAPI(ABC): async def detach(self): pass @abstractmethod - async def new_private_route(self) -> NewPrivateRouteResult: + async def new_private_route(self) -> Tuple[RouteId, bytes]: pass @abstractmethod - async def new_custom_private_route(self, kinds: list[CryptoKind], stability: Stability, sequencing: Sequencing) -> NewPrivateRouteResult: + async def new_custom_private_route(self, kinds: list[CryptoKind], stability: Stability, sequencing: Sequencing) -> Tuple[RouteId, bytes]: pass @abstractmethod async def import_remote_private_route(self, blob: bytes) -> RouteId: diff --git a/veilid-python/veilid/json_api.py b/veilid-python/veilid/json_api.py index 5b0d0fc3..eb388fc3 100644 --- a/veilid-python/veilid/json_api.py +++ b/veilid-python/veilid/json_api.py @@ -223,15 +223,15 @@ class _JsonVeilidAPI(VeilidAPI): raise_api_result(await self.send_ndjson_request(Operation.ATTACH)) async def detach(self): raise_api_result(await self.send_ndjson_request(Operation.DETACH)) - async def new_private_route(self) -> NewPrivateRouteResult: - return NewPrivateRouteResult.from_json(raise_api_result(await self.send_ndjson_request(Operation.NEW_PRIVATE_ROUTE))) - async def new_custom_private_route(self, kinds: list[CryptoKind], stability: Stability, sequencing: Sequencing) -> NewPrivateRouteResult: + async def new_private_route(self) -> Tuple[RouteId, bytes]: + return NewPrivateRouteResult.from_json(raise_api_result(await self.send_ndjson_request(Operation.NEW_PRIVATE_ROUTE))).to_tuple() + async def new_custom_private_route(self, kinds: list[CryptoKind], stability: Stability, sequencing: Sequencing) -> Tuple[RouteId, bytes]: return NewPrivateRouteResult.from_json(raise_api_result( await self.send_ndjson_request(Operation.NEW_CUSTOM_PRIVATE_ROUTE, kinds = kinds, stability = stability, sequencing = sequencing) - )) + )).to_tuple() async def import_remote_private_route(self, blob: bytes) -> RouteId: return RouteId(raise_api_result( await self.send_ndjson_request(Operation.IMPORT_REMOTE_PRIVATE_ROUTE, diff --git a/veilid-python/veilid/schema/RecvMessage.json b/veilid-python/veilid/schema/RecvMessage.json index b84dc8e4..15b37df1 100644 --- a/veilid-python/veilid/schema/RecvMessage.json +++ b/veilid-python/veilid/schema/RecvMessage.json @@ -2347,12 +2347,12 @@ "description": "Direct question blob passed to hosting application for processing to send an eventual AppReply", "type": "object", "required": [ - "id", + "call_id", "kind", "message" ], "properties": { - "id": { + "call_id": { "description": "The id to reply to", "type": "string" }, diff --git a/veilid-python/veilid/state.py b/veilid-python/veilid/state.py index 13404754..acb5c37c 100644 --- a/veilid-python/veilid/state.py +++ b/veilid-python/veilid/state.py @@ -250,12 +250,12 @@ class VeilidAppMessage: class VeilidAppCall: sender: Optional[TypedKey] message: bytes - operation_id: str + call_id: str - def __init__(self, sender: Optional[TypedKey], message: bytes, operation_id: str): + def __init__(self, sender: Optional[TypedKey], message: bytes, call_id: str): self.sender = sender self.message = message - self.operation_id = operation_id + self.call_id = call_id @staticmethod def from_json(j: dict) -> Self: @@ -263,7 +263,7 @@ class VeilidAppCall: return VeilidAppCall( None if j['sender'] is None else TypedKey(j['sender']), urlsafe_b64decode_no_pad(j['message']), - j['operation_id']) + j['call_id']) class VeilidRouteChange: dead_routes: list[RouteId] diff --git a/veilid-python/veilid/types.py b/veilid-python/veilid/types.py index b3388531..684f3b17 100644 --- a/veilid-python/veilid/types.py +++ b/veilid-python/veilid/types.py @@ -204,6 +204,9 @@ class NewPrivateRouteResult: self.route_id = route_id self.blob = blob + def to_tuple(self) -> Tuple[RouteId, bytes]: + return (self.route_id, self.blob) + @staticmethod def from_json(j: dict) -> Self: return NewPrivateRouteResult( diff --git a/veilid-wasm/src/lib.rs b/veilid-wasm/src/lib.rs index a43f7ce5..72b2c102 100644 --- a/veilid-wasm/src/lib.rs +++ b/veilid-wasm/src/lib.rs @@ -641,19 +641,21 @@ pub fn release_private_route(route_id: String) -> Promise { } #[wasm_bindgen()] -pub fn app_call_reply(id: String, message: String) -> Promise { +pub fn app_call_reply(call_id: String, message: String) -> Promise { let message: Vec = data_encoding::BASE64URL_NOPAD .decode(message.as_bytes()) .unwrap(); wrap_api_future_void(async move { - let id = match id.parse() { + let call_id = match call_id.parse() { Ok(v) => v, Err(e) => { - return APIResult::Err(veilid_core::VeilidAPIError::invalid_argument(e, "id", id)) + return APIResult::Err(veilid_core::VeilidAPIError::invalid_argument( + e, "call_id", call_id, + )) } }; let veilid_api = get_veilid_api()?; - veilid_api.app_call_reply(id, message).await?; + veilid_api.app_call_reply(call_id, message).await?; APIRESULT_UNDEFINED }) } From c6e2836b85a3c667cb947b4eb643af765152389d Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 16 Jun 2023 13:14:34 -0400 Subject: [PATCH 20/23] xfer --- veilid-core/src/veilid_api/debug.rs | 2 +- veilid-python/tests/__init__.py | 4 ++++ veilid-python/tests/test_routing_context.py | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index bdc269ee..436eec3b 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -934,7 +934,7 @@ impl VeilidAPI { entry nodeinfo config [key [new value]] - purge + purge attach detach restart network diff --git a/veilid-python/tests/__init__.py b/veilid-python/tests/__init__.py index 7f5c42bb..428f1e4c 100644 --- a/veilid-python/tests/__init__.py +++ b/veilid-python/tests/__init__.py @@ -24,6 +24,10 @@ else: async def simple_connect_and_run(func: Callable[[veilid.VeilidAPI], Awaitable]): api = await veilid.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) async with api: + + # purge routes to ensure we start fresh + await api.debug("purge routes") + await func(api) async def simple_update_callback(update: veilid.VeilidUpdate): diff --git a/veilid-python/tests/test_routing_context.py b/veilid-python/tests/test_routing_context.py index 2ab8a579..d1842907 100644 --- a/veilid-python/tests/test_routing_context.py +++ b/veilid-python/tests/test_routing_context.py @@ -29,6 +29,9 @@ async def test_routing_context_app_message_loopback(): api = await veilid.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, app_message_queue_update_callback) async with api: + # purge routes to ensure we start fresh + await api.debug("purge routes") + # make a routing context that uses a safety route rc = await (await api.new_routing_context()).with_privacy() @@ -60,6 +63,9 @@ async def test_routing_context_app_call_loopback(): api = await veilid.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, app_call_queue_update_callback) async with api: + # purge routes to ensure we start fresh + await api.debug("purge routes") + # make a routing context that uses a safety route rc = await (await api.new_routing_context()).with_privacy() From f3a3d5322cdfcba5a7fd796db4b9a76d81b36841 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 16 Jun 2023 13:29:25 -0400 Subject: [PATCH 21/23] fixes for tests --- Cargo.lock | 1 + veilid-core/src/veilid_api/tests/test_serialize_json.rs | 5 +++++ veilid-core/src/veilid_api/tests/test_types.rs | 8 +++++++- veilid-core/src/veilid_api/tests/test_types_dht_schema.rs | 1 - 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eddd7050..bf108ff7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6236,6 +6236,7 @@ dependencies = [ "owo-colors", "paranoid-android", "parking_lot 0.12.1", + "paste", "range-set-blaze", "rkyv", "rtnetlink", diff --git a/veilid-core/src/veilid_api/tests/test_serialize_json.rs b/veilid-core/src/veilid_api/tests/test_serialize_json.rs index 78df9607..bf508779 100644 --- a/veilid-core/src/veilid_api/tests/test_serialize_json.rs +++ b/veilid-core/src/veilid_api/tests/test_serialize_json.rs @@ -17,10 +17,15 @@ pub async fn test_all() { test_transferstatsdownup().await; test_rpcstats().await; test_peerstats().await; + #[cfg(feature = "unstable-tunnels")] test_tunnelmode().await; + #[cfg(feature = "unstable-tunnels")] test_tunnelerror().await; + #[cfg(feature = "unstable-tunnels")] test_tunnelendpoint().await; + #[cfg(feature = "unstable-tunnels")] test_fulltunnel().await; + #[cfg(feature = "unstable-tunnels")] test_partialtunnel().await; test_veilidloglevel().await; test_veilidlog().await; diff --git a/veilid-core/src/veilid_api/tests/test_types.rs b/veilid-core/src/veilid_api/tests/test_types.rs index b56cc024..524f9f7a 100644 --- a/veilid-core/src/veilid_api/tests/test_types.rs +++ b/veilid-core/src/veilid_api/tests/test_types.rs @@ -26,7 +26,7 @@ pub async fn test_veilidappcall() { let orig = VeilidAppCall { sender: Some(fix_typedkey()), message: b"Well, hello!".to_vec(), - id: AlignedU64::from(123), + call_id: AlignedU64::from(123), }; let copy = deserialize_json(&serialize_json(&orig)).unwrap(); @@ -116,12 +116,15 @@ pub async fn test_peerstats() { // tunnel +#[cfg(feature = "unstable-tunnels")] pub async fn test_tunnelmode() { let orig = TunnelMode::Raw; let copy = deserialize_json(&serialize_json(&orig)).unwrap(); assert_eq!(orig, copy); } + +#[cfg(feature = "unstable-tunnels")] pub async fn test_tunnelerror() { let orig = TunnelError::NoCapacity; let copy = deserialize_json(&serialize_json(&orig)).unwrap(); @@ -129,6 +132,7 @@ pub async fn test_tunnelerror() { assert_eq!(orig, copy); } +#[cfg(feature = "unstable-tunnels")] pub async fn test_tunnelendpoint() { let orig = TunnelEndpoint { mode: TunnelMode::Raw, @@ -139,6 +143,7 @@ pub async fn test_tunnelendpoint() { assert_eq!(orig, copy); } +#[cfg(feature = "unstable-tunnels")] pub async fn test_fulltunnel() { let orig = FullTunnel { id: AlignedU64::from(42), @@ -157,6 +162,7 @@ pub async fn test_fulltunnel() { assert_eq!(orig, copy); } +#[cfg(feature = "unstable-tunnels")] pub async fn test_partialtunnel() { let orig = PartialTunnel { id: AlignedU64::from(42), diff --git a/veilid-core/src/veilid_api/tests/test_types_dht_schema.rs b/veilid-core/src/veilid_api/tests/test_types_dht_schema.rs index a7a8379c..582d4040 100644 --- a/veilid-core/src/veilid_api/tests/test_types_dht_schema.rs +++ b/veilid-core/src/veilid_api/tests/test_types_dht_schema.rs @@ -1,6 +1,5 @@ use super::fixtures::*; use crate::*; -use range_set_blaze::*; // dlft From 9ffcc0da876505b2a799621ffc8d54a95cf88926 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 16 Jun 2023 14:18:34 -0400 Subject: [PATCH 22/23] fix tests --- .../tests/test_serialize_routing_table.rs | 29 ++++++++++++++----- .../src/routing_table/types/node_info.rs | 11 ++++++- .../src/routing_table/types/peer_info.rs | 4 ++- .../types/signed_direct_node_info.rs | 4 ++- .../routing_table/types/signed_node_info.rs | 4 ++- .../types/signed_relayed_node_info.rs | 4 ++- veilid-core/src/veilid_api/tests/fixtures.rs | 6 +--- .../src/veilid_api/tests/test_types.rs | 15 ++++------ .../src/veilid_api/tests/test_types_dht.rs | 22 +++++--------- .../src/veilid_api/types/app_message_call.rs | 6 ++-- .../types/dht/dht_record_descriptor.rs | 8 ++--- .../src/veilid_api/types/dht/value_data.rs | 6 ++-- .../types/dht/value_subkey_range_set.rs | 5 +++- 13 files changed, 73 insertions(+), 51 deletions(-) diff --git a/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs b/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs index c2a7b530..5bf624aa 100644 --- a/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs +++ b/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs @@ -1,6 +1,5 @@ use crate::*; - -const SERIALIZED_PEERINFO: &str = r###"{"node_ids":["FAKE:eFOfgm_FNZBsTRi7KAESNwYFAUGgX2uDrTRWAL8ucjM"],"signed_node_info":{"Direct":{"node_info":{"network_class":"InboundCapable","outbound_protocols":1,"address_types":3,"envelope_support":[0],"crypto_support":[[86,76,68,48]],"dial_info_detail_list":[{"class":"Direct","dial_info":{"kind":"UDP","socket_address":{"address":{"IPV4":"1.2.3.4"},"port":5150}}},{"class":"Direct","dial_info":{"kind":"UDP","socket_address":{"address":{"IPV6":"bad:cafe::1"},"port":5150}}},{"class":"Direct","dial_info":{"kind":"TCP","socket_address":{"address":{"IPV4":"5.6.7.8"},"port":5150}}},{"class":"Direct","dial_info":{"kind":"TCP","socket_address":{"address":{"IPV6":"bad:cafe::1"},"port":5150}}},{"class":"Direct","dial_info":{"kind":"WS","socket_address":{"address":{"IPV4":"9.10.11.12"},"port":5150},"request":"bootstrap-1.dev.veilid.net:5150/ws"}},{"class":"Direct","dial_info":{"kind":"WS","socket_address":{"address":{"IPV6":"bad:cafe::1"},"port":5150},"request":"bootstrap-1.dev.veilid.net:5150/ws"}}]},"timestamp":1685058646770389,"signatures":[]}}}"###; +use routing_table::*; fn fake_routing_table() -> routing_table::RoutingTable { let veilid_config = VeilidConfig::new(); @@ -25,7 +24,7 @@ fn fake_routing_table() -> routing_table::RoutingTable { block_store.clone(), crypto.clone(), ); - routing_table::RoutingTable::new(network_manager) + RoutingTable::new(network_manager) } pub async fn test_routingtable_buckets_round_trip() { @@ -84,11 +83,27 @@ pub async fn test_routingtable_buckets_round_trip() { } pub async fn test_round_trip_peerinfo() { - let pi: routing_table::PeerInfo = deserialize_json(SERIALIZED_PEERINFO).unwrap(); + let pi: PeerInfo = PeerInfo::new( + TypedKeySet::new(), + SignedNodeInfo::Direct(SignedDirectNodeInfo::new( + NodeInfo::new( + NetworkClass::OutboundOnly, + ProtocolTypeSet::new(), + AddressTypeSet::new(), + vec![0], + vec![CRYPTO_KIND_VLD0], + vec![], + ), + Timestamp::new(0), + Vec::new(), + )), + ); + let s = serialize_json(&pi); + let pi2 = deserialize_json(&s).expect("Should deserialize"); + let s2 = serialize_json(&pi2); - let back = serialize_json(pi); - - assert_eq!(SERIALIZED_PEERINFO, back); + assert_eq!(pi, pi2); + assert_eq!(s, s2); } pub async fn test_all() { diff --git a/veilid-core/src/routing_table/types/node_info.rs b/veilid-core/src/routing_table/types/node_info.rs index ca5cbde8..e67f4711 100644 --- a/veilid-core/src/routing_table/types/node_info.rs +++ b/veilid-core/src/routing_table/types/node_info.rs @@ -1,7 +1,16 @@ use super::*; #[derive( - Clone, Default, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, + Clone, + Default, + PartialEq, + Eq, + Debug, + Serialize, + Deserialize, + RkyvArchive, + RkyvSerialize, + RkyvDeserialize, )] #[archive_attr(repr(C), derive(CheckBytes))] pub struct NodeInfo { diff --git a/veilid-core/src/routing_table/types/peer_info.rs b/veilid-core/src/routing_table/types/peer_info.rs index b7037646..e3037459 100644 --- a/veilid-core/src/routing_table/types/peer_info.rs +++ b/veilid-core/src/routing_table/types/peer_info.rs @@ -1,6 +1,8 @@ use super::*; -#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[derive( + Clone, Debug, Serialize, Deserialize, PartialEq, Eq, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] #[archive_attr(repr(C), derive(CheckBytes))] pub struct PeerInfo { node_ids: TypedKeySet, diff --git a/veilid-core/src/routing_table/types/signed_direct_node_info.rs b/veilid-core/src/routing_table/types/signed_direct_node_info.rs index e6fee40d..32efb8de 100644 --- a/veilid-core/src/routing_table/types/signed_direct_node_info.rs +++ b/veilid-core/src/routing_table/types/signed_direct_node_info.rs @@ -1,7 +1,9 @@ use super::*; /// Signed NodeInfo that can be passed around amongst peers and verifiable -#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[derive( + Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] #[archive_attr(repr(C), derive(CheckBytes))] pub struct SignedDirectNodeInfo { node_info: NodeInfo, diff --git a/veilid-core/src/routing_table/types/signed_node_info.rs b/veilid-core/src/routing_table/types/signed_node_info.rs index 5557a76a..60e99c89 100644 --- a/veilid-core/src/routing_table/types/signed_node_info.rs +++ b/veilid-core/src/routing_table/types/signed_node_info.rs @@ -1,6 +1,8 @@ use super::*; -#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[derive( + Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] #[archive_attr(repr(u8), derive(CheckBytes))] pub enum SignedNodeInfo { Direct(SignedDirectNodeInfo), diff --git a/veilid-core/src/routing_table/types/signed_relayed_node_info.rs b/veilid-core/src/routing_table/types/signed_relayed_node_info.rs index 8a429e4c..cdfda702 100644 --- a/veilid-core/src/routing_table/types/signed_relayed_node_info.rs +++ b/veilid-core/src/routing_table/types/signed_relayed_node_info.rs @@ -1,7 +1,9 @@ use super::*; /// Signed NodeInfo with a relay that can be passed around amongst peers and verifiable -#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)] +#[derive( + Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, +)] #[archive_attr(repr(C), derive(CheckBytes))] pub struct SignedRelayedNodeInfo { node_info: NodeInfo, diff --git a/veilid-core/src/veilid_api/tests/fixtures.rs b/veilid-core/src/veilid_api/tests/fixtures.rs index 7f86ac22..bb503260 100644 --- a/veilid-core/src/veilid_api/tests/fixtures.rs +++ b/veilid-core/src/veilid_api/tests/fixtures.rs @@ -212,10 +212,6 @@ pub fn fix_veilidvaluechange() -> VeilidValueChange { key: fix_typedkey(), subkeys: vec![1, 2, 3, 4], count: 5, - value: ValueData { - seq: 23, - data: b"ValueData".to_vec(), - writer: fix_cryptokey(), - }, + value: ValueData::new_with_seq(23, b"ValueData".to_vec(), fix_cryptokey()), } } diff --git a/veilid-core/src/veilid_api/tests/test_types.rs b/veilid-core/src/veilid_api/tests/test_types.rs index 524f9f7a..7cbe27bb 100644 --- a/veilid-core/src/veilid_api/tests/test_types.rs +++ b/veilid-core/src/veilid_api/tests/test_types.rs @@ -13,21 +13,18 @@ pub async fn test_alignedu64() { // app_messsage_call pub async fn test_veilidappmessage() { - let orig = VeilidAppMessage { - sender: Some(fix_typedkey()), - message: b"Hi there!".to_vec(), - }; + let orig = VeilidAppMessage::new(Some(fix_typedkey()), b"Hi there!".to_vec()); let copy = deserialize_json(&serialize_json(&orig)).unwrap(); assert_eq!(orig, copy); } pub async fn test_veilidappcall() { - let orig = VeilidAppCall { - sender: Some(fix_typedkey()), - message: b"Well, hello!".to_vec(), - call_id: AlignedU64::from(123), - }; + let orig = VeilidAppCall::new( + Some(fix_typedkey()), + b"Well, hello!".to_vec(), + AlignedU64::from(123), + ); let copy = deserialize_json(&serialize_json(&orig)).unwrap(); assert_eq!(orig, copy); diff --git a/veilid-core/src/veilid_api/tests/test_types_dht.rs b/veilid-core/src/veilid_api/tests/test_types_dht.rs index f098e06b..73f32eda 100644 --- a/veilid-core/src/veilid_api/tests/test_types_dht.rs +++ b/veilid-core/src/veilid_api/tests/test_types_dht.rs @@ -5,12 +5,12 @@ use range_set_blaze::*; // dht_record_descriptors pub async fn test_dhtrecorddescriptor() { - let orig = DHTRecordDescriptor { - key: fix_typedkey(), - owner: fix_cryptokey(), - owner_secret: Some(fix_cryptokey()), - schema: DHTSchema::DFLT(DHTSchemaDFLT { o_cnt: 4321 }), - }; + let orig = DHTRecordDescriptor::new( + fix_typedkey(), + fix_cryptokey(), + Some(fix_cryptokey()), + DHTSchema::DFLT(DHTSchemaDFLT { o_cnt: 4321 }), + ); let copy = deserialize_json(&serialize_json(&orig)).unwrap(); assert_eq!(orig, copy); @@ -19,11 +19,7 @@ pub async fn test_dhtrecorddescriptor() { // value_data pub async fn test_valuedata() { - let orig = ValueData { - seq: 42, - data: b"Brent Spiner".to_vec(), - writer: fix_cryptokey(), - }; + let orig = ValueData::new_with_seq(42, b"Brent Spiner".to_vec(), fix_cryptokey()); let copy = deserialize_json(&serialize_json(&orig)).unwrap(); assert_eq!(orig, copy); @@ -32,9 +28,7 @@ pub async fn test_valuedata() { // value_subkey_range_set pub async fn test_valuesubkeyrangeset() { - let orig = ValueSubkeyRangeSet { - data: RangeSetBlaze::from_iter([20..=30]), - }; + let orig = ValueSubkeyRangeSet::new_with_data(RangeSetBlaze::from_iter([20..=30])); let copy = deserialize_json(&serialize_json(&orig)).unwrap(); assert_eq!(orig, copy); diff --git a/veilid-core/src/veilid_api/types/app_message_call.rs b/veilid-core/src/veilid_api/types/app_message_call.rs index 3d6a096e..36fa47e1 100644 --- a/veilid-core/src/veilid_api/types/app_message_call.rs +++ b/veilid-core/src/veilid_api/types/app_message_call.rs @@ -57,17 +57,17 @@ pub struct VeilidAppCall { /// Some(sender) if the request was sent directly, None if received via a private/safety route #[serde(with = "opt_json_as_string")] #[schemars(with = "Option")] - pub sender: Option, + sender: Option, /// The content of the request to deliver to the application #[serde(with = "json_as_base64")] #[schemars(with = "String")] - pub message: Vec, + message: Vec, /// The id to reply to #[serde(with = "json_as_string")] #[schemars(with = "String")] - pub call_id: OperationId, + call_id: OperationId, } impl VeilidAppCall { diff --git a/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs b/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs index 03857c1a..6162fd9f 100644 --- a/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs +++ b/veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs @@ -19,16 +19,16 @@ use super::*; pub struct DHTRecordDescriptor { /// DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ] #[schemars(with = "String")] - pub key: TypedKey, + key: TypedKey, /// The public key of the owner #[schemars(with = "String")] - pub owner: PublicKey, + owner: PublicKey, /// If this key is being created: Some(the secret key of the owner) /// If this key is just being opened: None #[schemars(with = "Option")] - pub owner_secret: Option, + owner_secret: Option, /// The schema in use associated with the key - pub schema: DHTSchema, + schema: DHTSchema, } impl DHTRecordDescriptor { diff --git a/veilid-core/src/veilid_api/types/dht/value_data.rs b/veilid-core/src/veilid_api/types/dht/value_data.rs index 737d4a7d..c57fc2e5 100644 --- a/veilid-core/src/veilid_api/types/dht/value_data.rs +++ b/veilid-core/src/veilid_api/types/dht/value_data.rs @@ -18,16 +18,16 @@ use super::*; #[archive_attr(repr(C), derive(CheckBytes))] pub struct ValueData { /// An increasing sequence number to time-order the DHT record changes - pub seq: ValueSeqNum, + seq: ValueSeqNum, /// The contents of a DHT Record #[serde(with = "json_as_base64")] #[schemars(with = "String")] - pub data: Vec, + data: Vec, /// The public identity key of the writer of the data #[schemars(with = "String")] - pub writer: PublicKey, + writer: PublicKey, } impl ValueData { pub const MAX_LEN: usize = 32768; diff --git a/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs b/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs index 8a7196ab..466346d3 100644 --- a/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs +++ b/veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs @@ -23,7 +23,7 @@ pub struct ValueSubkeyRangeSet { #[with(RkyvRangeSetBlaze)] #[serde(with = "serialize_range_set_blaze")] #[schemars(with = "Vec<(u32,u32)>")] - pub data: RangeSetBlaze, + data: RangeSetBlaze, } impl ValueSubkeyRangeSet { @@ -32,6 +32,9 @@ impl ValueSubkeyRangeSet { data: Default::default(), } } + pub fn new_with_data(data: RangeSetBlaze) -> Self { + Self { data } + } pub fn single(value: ValueSubkey) -> Self { let mut data = RangeSetBlaze::new(); data.insert(value); From d308ebc324793acd486de4701b2fa3748e66d6b8 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 16 Jun 2023 14:23:36 -0400 Subject: [PATCH 23/23] fix tests --- .../tests/test_serialize_routing_table.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs b/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs index 5bf624aa..5f06866e 100644 --- a/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs +++ b/veilid-core/src/routing_table/tests/test_serialize_routing_table.rs @@ -83,8 +83,16 @@ pub async fn test_routingtable_buckets_round_trip() { } pub async fn test_round_trip_peerinfo() { + let mut tks = TypedKeySet::new(); + tks.add(TypedKey::new( + CRYPTO_KIND_VLD0, + CryptoKey::new([ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, + ]), + )); let pi: PeerInfo = PeerInfo::new( - TypedKeySet::new(), + tks, SignedNodeInfo::Direct(SignedDirectNodeInfo::new( NodeInfo::new( NetworkClass::OutboundOnly,