liz/veilid
liz/veilid
1
0
Fork 0
Code Pull Requests Packages Releases Activity

remove rkyv

Browse Source
This commit is contained in:
Christien Rioux 2023-07-15 16:18:13 -04:00
parent 4078c00098
commit 80cb23c0c6
86 changed files with 943 additions and 2442 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

772
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

138
veilid-core/Cargo.toml
View File

@ -31,100 +31,126 @@ veilid_core_android_tests = ["dep:paranoid-android"]
veilid_core_ios_tests = ["dep:tracing-oslog"]
network-result-extra = ["veilid-tools/network-result-extra"]
### DEPENDENCIES
[dependencies]
# Tools
veilid-tools = { path = "../veilid-tools", features = [ "tracing" ] }
paste = "1.0.12"
once_cell = "^1"
owning_ref = "^0"
backtrace = { version = "^0" }
num-traits = "0.2.15"
shell-words = "1.1.0"
static_assertions = "^1"
cfg-if = "^1"
hex = "^0"
lazy_static = "^1"
directories = "^4"
# Logging
tracing = { version = "^0", features = ["log", "attributes"] }
tracing-subscriber = "^0"
tracing-error = "^0"
eyre = "^0"
capnp = { version = "^0", default_features = false }
static_assertions = "^1"
cfg-if = "^1"
thiserror = "^1"
hex = "^0"
generic-array = "^0"
secrecy = "^0"
chacha20poly1305 = "^0"
chacha20 = "^0"
hashlink = { path = "../external/hashlink", features = ["serde_impl"] }
serde = { version = "^1", features = ["derive" ] }
serde_json = { version = "^1" }
serde-big-array = "^0"
futures-util = { version = "^0", default_features = false, features = ["alloc"] }
parking_lot = "^0"
lazy_static = "^1"
directories = "^4"
once_cell = "^1"
json = "^0"
owning_ref = "^0"
flume = { version = "^0", features = ["async"] }
enumset = { version= "^1", features = ["serde"] }
backtrace = { version = "^0" }
stop-token = { version = "^0", default-features = false }
num-traits = "0.2.15"
shell-words = "1.1.0"
# Data structures
enumset = { version= "^1", features = ["serde"] }
keyvaluedb = { path = "../external/keyvaluedb/keyvaluedb" }
range-set-blaze = "0.1.5"
weak-table = "0.3.2"
generic-array = "^0"
hashlink = { path = "../external/hashlink", features = ["serde_impl"] }
# System
futures-util = { version = "^0", default_features = false, features = ["alloc"] }
flume = { version = "^0", features = ["async"] }
parking_lot = "^0"
stop-token = { version = "^0", default-features = false }
# Crypto
ed25519-dalek = { version = "^1", default_features = false, features = ["alloc", "u64_backend"] }
x25519-dalek = { version = "^1", default_features = false, features = ["u64_backend"] }
curve25519-dalek = { version = "^3", default_features = false, features = ["alloc", "u64_backend"] }
blake3 = { version = "^1" }
chacha20poly1305 = "^0"
chacha20 = "^0"
argon2 = "0.5.0"
# Network
async-std-resolver = { version = "^0", optional = true }
trust-dns-resolver = { version = "^0", optional = true }
enum-as-inner = "=0.5.1" # temporary fix for trust-dns-resolver v0.22.0
keyvaluedb = { path = "../external/keyvaluedb/keyvaluedb" }
rkyv = { version = "^0", default_features = false, features = ["std", "alloc", "strict", "size_32", "validation"] }
# Serialization
capnp = { version = "^0", default_features = false }
serde = { version = "^1", features = ["derive" ] }
serde_json = { version = "^1" }
serde-big-array = "^0"
json = "^0"
data-encoding = { version = "^2" }
weak-table = "0.3.2"
range-set-blaze = "0.1.5"
argon2 = "0.5.0"
paste = "1.0.12"
schemars = "0.8.12"
# Dependencies for native builds only
# Linux, Windows, Mac, iOS, Android
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
async-std = { version = "^1", features = ["unstable"], optional = true}
tokio = { version = "^1", features = ["full"], optional = true}
tokio-util = { version = "^0", features = ["compat"], optional = true}
tokio-stream = { version = "^0", features = ["net"], optional = true}
async-io = { version = "^1" }
async-tungstenite = { version = "^0", features = ["async-tls"] }
maplit = "^1"
# Tools
config = { version = "^0", features = ["yaml"] }
keyring-manager = { path = "../external/keyring-manager" }
async-tls = "^0.11"
igd = { path = "../external/rust-igd" }
webpki = "^0"
webpki-roots = "^0"
rustls = "^0.19"
rustls-pemfile = "^0.2"
futures-util = { version = "^0", default-features = false, features = ["async-await", "sink", "std", "io"] }
keyvaluedb-sqlite = { path = "../external/keyvaluedb/keyvaluedb-sqlite" }
socket2 = { version = "^0", features = ["all"] }
bugsalot = { git = "https://github.com/crioux/bugsalot.git" }
chrono = "^0"
libc = "^0"
nix = "^0"
# System
async-std = { version = "^1", features = ["unstable"], optional = true}
tokio = { version = "^1", features = ["full"], optional = true}
tokio-util = { version = "^0", features = ["compat"], optional = true}
tokio-stream = { version = "^0", features = ["net"], optional = true}
async-io = { version = "^1" }
futures-util = { version = "^0", default-features = false, features = ["async-await", "sink", "std", "io"] }
# Data structures
keyring-manager = { path = "../external/keyring-manager" }
keyvaluedb-sqlite = { path = "../external/keyvaluedb/keyvaluedb-sqlite" }
# Network
async-tungstenite = { version = "^0", features = ["async-tls"] }
igd = { path = "../external/rust-igd" }
async-tls = "^0.11"
webpki = "^0"
webpki-roots = "^0"
rustls = "^0.19"
rustls-pemfile = "^0.2"
socket2 = { version = "^0", features = ["all"] }
# Dependencies for WASM builds only
[target.'cfg(target_arch = "wasm32")'.dependencies]
# Tools
getrandom = { version = "^0", features = ["js"] }
# System
async_executors = { version = "^0", default-features = false, features = [ "bindgen", "timer" ]}
async-lock = "^2"
wasm-bindgen = "^0"
js-sys = "^0"
wasm-bindgen-futures = "^0"
keyvaluedb-web = { path = "../external/keyvaluedb/keyvaluedb-web" }
getrandom = { version = "^0", features = ["js"] }
ws_stream_wasm = "^0"
async_executors = { version = "^0", default-features = false, features = [ "bindgen", "timer" ]}
async-lock = "^2"
send_wrapper = { version = "^0", features = ["futures"] }
# Network
ws_stream_wasm = "^0"
# Logging
wasm-logger = "^0"
tracing-wasm = "^0"
# Configuration for WASM32 'web-sys' crate
# Data Structures
keyvaluedb-web = { path = "../external/keyvaluedb/keyvaluedb-web" }
### Configuration for WASM32 'web-sys' crate
[target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
version = "^0"
features = [

3
veilid-core/src/crypto/types/byte_array_types.rs
View File

@ -77,8 +77,7 @@ where
macro_rules! byte_array_type {
($name:ident, $size:expr, $encoded_size:expr) => {
#[derive(Clone, Copy, Hash, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes, Hash, Eq, PartialEq, PartialOrd, Ord))]
#[derive(Clone, Copy, Hash)]
pub struct $name {
pub bytes: [u8; $size],
}

19
veilid-core/src/crypto/types/crypto_typed.rs
View File

@ -1,7 +1,6 @@
use super::*;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes, Hash, PartialEq, Eq))]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct CryptoTyped<K>
where
K: Clone
@ -14,9 +13,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
pub kind: CryptoKind,
pub value: K,
@ -34,9 +31,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
pub fn new(kind: CryptoKind, value: K) -> Self {
Self { kind, value }
@ -54,9 +49,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
Some(self.cmp(other))
@ -75,9 +68,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn cmp(&self, other: &Self) -> cmp::Ordering {
let x = compare_crypto_kind(&self.kind, &other.kind);
@ -100,9 +91,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
write!(f, "{}:{}", self.kind, self.value)
@ -120,9 +109,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
type Err = VeilidAPIError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
@ -152,9 +139,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
@ -176,9 +161,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where

36
veilid-core/src/crypto/types/crypto_typed_group.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Debug,
Serialize,
Deserialize,
PartialOrd,
Ord,
PartialEq,
Eq,
Hash,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
Default,
)]
#[archive_attr(repr(C), derive(CheckBytes, Hash, PartialEq, Eq))]
#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash, Default)]
#[serde(from = "Vec<CryptoTyped<K>>", into = "Vec<CryptoTyped<K>>")]
pub struct CryptoTypedGroup<K = PublicKey>
where
@ -29,10 +14,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
<Vec<CryptoTyped<K>> as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
items: Vec<CryptoTyped<K>>,
}
@ -49,9 +31,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
pub fn new() -> Self {
Self { items: Vec::new() }
@ -163,9 +143,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
type Target = [CryptoTyped<K>];
@ -187,9 +165,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
write!(f, "[")?;
@ -217,9 +193,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
type Err = VeilidAPIError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
@ -250,9 +224,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn from(x: CryptoTyped<K>) -> Self {
let mut tks = CryptoTypedGroup::<K>::with_capacity(1);
@ -272,9 +244,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn from(x: Vec<CryptoTyped<K>>) -> Self {
let mut tks = CryptoTypedGroup::<K>::with_capacity(x.len());
@ -294,9 +264,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn from(x: &[CryptoTyped<K>]) -> Self {
let mut tks = CryptoTypedGroup::<K>::with_capacity(x.len());
@ -316,9 +284,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn into(self) -> Vec<CryptoTyped<K>> {
self.items

15
veilid-core/src/crypto/types/keypair.rs
View File

@ -1,19 +1,6 @@
use super::*;
#[derive(
Clone,
Copy,
Default,
PartialOrd,
Ord,
PartialEq,
Eq,
Hash,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes, Hash, PartialEq, Eq))]
#[derive(Clone, Copy, Default, PartialOrd, Ord, PartialEq, Eq, Hash)]
pub struct KeyPair {
pub key: PublicKey,
pub secret: SecretKey,

31
veilid-core/src/intf/native/protected_store.rs
View File

@ -141,16 +141,6 @@ impl ProtectedStore {
}
}
#[instrument(level = "trace", skip(self, value))]
pub async fn save_user_secret_rkyv<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
K: AsRef<str> + fmt::Debug,
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let v = to_rkyv(value)?;
self.save_user_secret(key, &v).await
}
#[instrument(level = "trace", skip(self, value))]
pub async fn save_user_secret_json<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
@ -161,27 +151,6 @@ impl ProtectedStore {
self.save_user_secret(&key, &v).await
}
#[instrument(level = "trace", skip(self))]
pub async fn load_user_secret_rkyv<K, T>(&self, key: K) -> EyreResult<Option<T>>
where
K: AsRef<str> + fmt::Debug,
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = self.load_user_secret(key).await?;
let b = match out {
Some(v) => v,
None => {
return Ok(None);
}
};
let obj = from_rkyv(b)?;
Ok(Some(obj))
}
#[instrument(level = "trace", skip(self))]
pub async fn load_user_secret_json<K, T>(&self, key: K) -> EyreResult<Option<T>>
where

35
veilid-core/src/intf/wasm/protected_store.rs
View File

@ -1,9 +1,5 @@
use super::*;
use data_encoding::BASE64URL_NOPAD;
use rkyv::{
bytecheck::CheckBytes, Archive as RkyvArchive, Deserialize as RkyvDeserialize,
Serialize as RkyvSerialize,
};
use web_sys::*;
@ -126,16 +122,6 @@ impl ProtectedStore {
}
}
#[instrument(level = "trace", skip(self, value))]
pub async fn save_user_secret_rkyv<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
K: AsRef<str> + fmt::Debug,
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let v = to_rkyv(value)?;
self.save_user_secret(key, &v).await
}
#[instrument(level = "trace", skip(self, value))]
pub async fn save_user_secret_json<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
@ -146,27 +132,6 @@ impl ProtectedStore {
self.save_user_secret(key, &v).await
}
#[instrument(level = "trace", skip(self))]
pub async fn load_user_secret_rkyv<K, T>(&self, key: K) -> EyreResult<Option<T>>
where
K: AsRef<str> + fmt::Debug,
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = self.load_user_secret(key).await?;
let b = match out {
Some(v) => v,
None => {
return Ok(None);
}
};
let obj = from_rkyv(b)?;
Ok(Some(obj))
}
#[instrument(level = "trace", skip(self))]
pub async fn load_user_secret_json<K, T>(&self, key: K) -> EyreResult<Option<T>>
where

11
veilid-core/src/lib.rs
View File

@ -88,16 +88,11 @@ pub static DEFAULT_LOG_IGNORE_LIST: [&str; 21] = [
use cfg_if::*;
use enumset::*;
use eyre::{bail, eyre, Report as EyreReport, Result as EyreResult, WrapErr};
use parking_lot::*;
use rkyv::{
bytecheck, bytecheck::CheckBytes, de::deserializers::SharedDeserializeMap, with::Skip,
Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize,
};
use tracing::*;
use veilid_tools::*;
type RkyvDefaultValidator<'t> = rkyv::validation::validators::DefaultValidator<'t>;
use futures_util::stream::FuturesUnordered;
use parking_lot::*;
use schemars::{schema_for, JsonSchema};
use serde::*;
use stop_token::*;
use thiserror::Error as ThisError;
use tracing::*;
use veilid_tools::*;

4
veilid-core/src/network_manager/mod.rs
View File

@ -36,12 +36,16 @@ use hashlink::LruCache;
use intf::*;
#[cfg(not(target_arch = "wasm32"))]
use native::*;
#[cfg(not(target_arch = "wasm32"))]
pub use native::{LOCAL_NETWORK_CAPABILITIES, MAX_CAPABILITIES, PUBLIC_INTERNET_CAPABILITIES};
use receipt_manager::*;
use routing_table::*;
use rpc_processor::*;
use storage_manager::*;
#[cfg(target_arch = "wasm32")]
use wasm::*;
#[cfg(target_arch = "wasm32")]
pub use wasm::{LOCAL_NETWORK_CAPABILITIES, MAX_CAPABILITIES, PUBLIC_INTERNET_CAPABILITIES};
////////////////////////////////////////////////////////////////////////////////////////

37
veilid-core/src/network_manager/native/mod.rs
View File

@ -30,6 +30,43 @@ use std::path::{Path, PathBuf};
pub const PEEK_DETECT_LEN: usize = 64;
cfg_if! {
if #[cfg(all(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 8;
} else if #[cfg(any(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 7;
} else {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 6;
}
}
pub const PUBLIC_INTERNET_CAPABILITIES: [Capability; PUBLIC_INTERNET_CAPABILITIES_LEN] = [
CAP_ROUTE,
#[cfg(feature = "unstable-tunnels")]
CAP_TUNNEL,
CAP_SIGNAL,
CAP_RELAY,
CAP_VALIDATE_DIAL_INFO,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
#[cfg(feature = "unstable-blockstore")]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 4;
#[cfg(not(feature = "unstable-blockstore"))]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 3;
pub const LOCAL_NETWORK_CAPABILITIES: [Capability; LOCAL_NETWORK_CAPABILITIES_LEN] = [
CAP_RELAY,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
pub const MAX_CAPABILITIES: usize = 64;
/////////////////////////////////////////////////////////////////
struct NetworkInner {

17
veilid-core/src/network_manager/types/address.rs
View File

@ -2,22 +2,7 @@ use super::*;
// Ordering here matters, IPV6 is preferred to IPV4 in dial info sorts
// See issue #236 for eventual resolution of this unfortunate implementation
#[derive(
Copy,
Clone,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub enum Address {
IPV6(Ipv6Addr),
IPV4(Ipv4Addr),

19
veilid-core/src/network_manager/types/address_type.rs
View File

@ -1,22 +1,11 @@
#![allow(non_snake_case)]
use super::*;
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
EnumSetType,
)]
#[derive(Debug, PartialOrd, Ord, Hash, Serialize, Deserialize, EnumSetType)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum AddressType {
IPV4,
IPV6,
IPV6 = 0,
IPV4 = 1,
}
pub type AddressTypeSet = EnumSet<AddressType>;

17
veilid-core/src/network_manager/types/connection_descriptor.rs
View File

@ -7,22 +7,7 @@ use super::*;
/// If the medium does not allow local addresses, None should have been used or 'new_no_local'
/// If we are specifying only a port, then the socket's 'local_address()' should have been used, since an
/// established connection is always from a real address to another real address.
#[derive(
Copy,
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ConnectionDescriptor {
remote: PeerAddress,
local: Option<SocketAddress>,

16
veilid-core/src/network_manager/types/dial_info/mod.rs
View File

@ -12,21 +12,7 @@ pub use wss::*;
// Keep member order appropriate for sorting < preference
// Must match ProtocolType order
#[derive(
Clone,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
#[serde(tag = "kind")]
pub enum DialInfo {
UDP(DialInfoUDP),

17
veilid-core/src/network_manager/types/dial_info/tcp.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Default,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoTCP {
pub socket_address: SocketAddress,
}

17
veilid-core/src/network_manager/types/dial_info/udp.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Default,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoUDP {
pub socket_address: SocketAddress,
}

17
veilid-core/src/network_manager/types/dial_info/ws.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Default,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoWS {
pub socket_address: SocketAddress,
pub request: String,

17
veilid-core/src/network_manager/types/dial_info/wss.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Default,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoWSS {
pub socket_address: SocketAddress,
pub request: String,

17
veilid-core/src/network_manager/types/dial_info_class.rs
View File

@ -1,22 +1,7 @@
use super::*;
// Keep member order appropriate for sorting < preference
#[derive(
Copy,
Clone,
Debug,
Eq,
PartialEq,
Ord,
PartialOrd,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub enum DialInfoClass {
Direct = 0, // D = Directly reachable with public IP and no firewall, with statically configured port
Mapped = 1, // M = Directly reachable with via portmap behind any NAT or firewalled with dynamically negotiated port

18
veilid-core/src/network_manager/types/dial_info_filter.rs
View File

@ -1,24 +1,8 @@
use super::*;
#[derive(
Copy,
Clone,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct DialInfoFilter {
#[with(RkyvEnumSet)]
pub protocol_type_set: ProtocolTypeSet,
#[with(RkyvEnumSet)]
pub address_type_set: AddressTypeSet,
}

20
veilid-core/src/network_manager/types/low_level_protocol_type.rs
View File

@ -1,25 +1,15 @@
#![allow(non_snake_case)]
use super::*;
// Keep member order appropriate for sorting < preference
// Must match DialInfo order
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
PartialOrd,
Ord,
Hash,
EnumSetType,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[derive(Debug, PartialOrd, Ord, Hash, EnumSetType, Serialize, Deserialize)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum LowLevelProtocolType {
UDP,
TCP,
UDP = 0,
TCP = 1,
}
impl LowLevelProtocolType {

17
veilid-core/src/network_manager/types/network_class.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Copy,
Clone,
Debug,
Eq,
PartialEq,
Ord,
PartialOrd,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub enum NetworkClass {
InboundCapable = 0, // I = Inbound capable without relay, may require signal
OutboundOnly = 1, // O = Outbound only, inbound relay required except with reverse connect signal

19
veilid-core/src/network_manager/types/peer_address.rs
View File

@ -1,24 +1,9 @@
use super::*;
#[derive(
Copy,
Clone,
Debug,
PartialEq,
PartialOrd,
Eq,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
pub struct PeerAddress {
protocol_type: ProtocolType,
#[serde(with = "json_as_string")]
#[serde(with = "as_human_string")]
socket_address: SocketAddress,
}

23
veilid-core/src/network_manager/types/protocol_type.rs
View File

@ -1,27 +1,16 @@
#![allow(non_snake_case)]
use super::*;
// Keep member order appropriate for sorting < preference
// Must match DialInfo order
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
PartialOrd,
Ord,
Hash,
EnumSetType,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[derive(Debug, PartialOrd, Ord, Hash, EnumSetType, Serialize, Deserialize)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum ProtocolType {
UDP,
TCP,
WS,
WSS,
UDP = 0,
TCP = 1,
WS = 2,
WSS = 3,
}
impl ProtocolType {

3
veilid-core/src/network_manager/types/signal_info.rs
View File

@ -1,8 +1,7 @@
use super::*;
/// Parameter for Signal operation
#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum SignalInfo {
/// UDP Hole Punch Request
HolePunch {

16
veilid-core/src/network_manager/types/socket_address.rs
View File

@ -1,22 +1,8 @@
use super::*;
#[derive(
Copy,
Default,
Clone,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
Copy, Default, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
pub struct SocketAddress {
address: Address,
port: u16,

59
veilid-core/src/network_manager/wasm/mod.rs
View File

@ -10,6 +10,45 @@ use std::io;
/////////////////////////////////////////////////////////////////
cfg_if! {
if #[cfg(all(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 6;
} else if #[cfg(any(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 5;
} else {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 4;
}
}
pub const PUBLIC_INTERNET_CAPABILITIES: [Capability; PUBLIC_INTERNET_CAPABILITIES_LEN] = [
CAP_ROUTE,
#[cfg(feature = "unstable-tunnels")]
CAP_TUNNEL,
CAP_SIGNAL,
//CAP_RELAY,
//CAP_VALIDATE_DIAL_INFO,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
#[cfg(feature = "unstable-blockstore")]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 3;
#[cfg(not(feature = "unstable-blockstore"))]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 2;
pub const LOCAL_NETWORK_CAPABILITIES: [Capability; LOCAL_NETWORK_CAPABILITIES_LEN] = [
//CAP_RELAY,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
pub const MAX_CAPABILITIES: usize = 64;
/////////////////////////////////////////////////////////////////
struct NetworkInner {
network_started: bool,
network_needs_restart: bool,
@ -273,10 +312,10 @@ impl Network {
let inbound = ProtocolTypeSet::new();
let mut outbound = ProtocolTypeSet::new();
if c.network.protocol.ws.connect && c.capabilities.protocol_connect_ws {
if c.network.protocol.ws.connect {
outbound.insert(ProtocolType::WS);
}
if c.network.protocol.wss.connect && c.capabilities.protocol_connect_wss {
if c.network.protocol.wss.connect {
outbound.insert(ProtocolType::WSS);
}
@ -301,15 +340,26 @@ impl Network {
// set up the routing table's network config
// if we have static public dialinfo, upgrade our network class
let public_internet_capabilities = {
let c = self.config.get();
PUBLIC_INTERNET_CAPABILITIES
.iter()
.copied()
.filter(|cap| !c.capabilities.disable.contains(cap))
.collect::<Vec<Capability>>()
};
editor_public_internet.setup_network(
protocol_config.outbound,
protocol_config.inbound,
protocol_config.family_global,
public_internet_capabilities,
);
editor_public_internet.set_network_class(Some(NetworkClass::WebApp));
// commit routing table edits
editor_public_internet.commit().await;
editor_public_internet.commit();
self.inner.lock().network_started = true;
Ok(())
@ -339,8 +389,7 @@ impl Network {
.clear_dial_info_details()
.set_network_class(None)
.clear_relay_node()
.commit()
.await;
.commit();
// Cancels all async background tasks by dropping join handles
*self.inner.lock() = Self::new_inner();

15
veilid-core/src/routing_table/bucket.rs
View File

@ -1,6 +1,5 @@
use super::*;
use core::sync::atomic::Ordering;
use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
/// Routing Table Bucket
/// Stores map of public keys to entries, which may be in multiple routing tables per crypto kind
@ -16,15 +15,13 @@ pub struct Bucket {
pub(super) type EntriesIter<'a> =
alloc::collections::btree_map::Iter<'a, PublicKey, Arc<BucketEntry>>;
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
struct SerializedBucketEntryData {
key: PublicKey,
value: u32, // index into serialized entries list
}
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
struct SerializedBucketData {
entries: Vec<SerializedBucketEntryData>,
}
@ -50,7 +47,7 @@ impl Bucket {
data: Vec<u8>,
all_entries: &[Arc<BucketEntry>],
) -> EyreResult<()> {
let bucket_data: SerializedBucketData = from_rkyv(data)?;
let bucket_data: SerializedBucketData = deserialize_json_bytes(&data)?;
for e in bucket_data.entries {
self.entries
@ -64,7 +61,7 @@ impl Bucket {
&self,
all_entries: &mut Vec<Arc<BucketEntry>>,
entry_map: &mut HashMap<*const BucketEntry, u32>,
) -> EyreResult<Vec<u8>> {
) -> Vec<u8> {
let mut entries = Vec::new();
for (k, v) in &self.entries {
let entry_index = entry_map.entry(Arc::as_ptr(v)).or_insert_with(|| {
@ -78,8 +75,8 @@ impl Bucket {
});
}
let bucket_data = SerializedBucketData { entries };
let out = to_rkyv(&bucket_data)?;
Ok(out)
let out = serialize_json_bytes(&bucket_data);
out
}
/// Create a new entry with a node_id of this crypto kind and return it

19
veilid-core/src/routing_table/bucket_entry.rs
View File

@ -39,8 +39,7 @@ pub enum BucketEntryState {
pub struct LastConnectionKey(ProtocolType, AddressType);
/// Bucket entry information specific to the LocalNetwork RoutingDomain
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
pub struct BucketEntryPublicInternet {
/// The PublicInternet node info
signed_node_info: Option<Box<SignedNodeInfo>>,
@ -51,8 +50,7 @@ pub struct BucketEntryPublicInternet {
}
/// Bucket entry information specific to the LocalNetwork RoutingDomain
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
pub struct BucketEntryLocalNetwork {
/// The LocalNetwork node info
signed_node_info: Option<Box<SignedNodeInfo>>,
@ -63,8 +61,7 @@ pub struct BucketEntryLocalNetwork {
}
/// The data associated with each bucket entry
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
pub struct BucketEntryInner {
/// The node ids matching this bucket entry, with the cryptography versions supported by this node as the 'kind' field
validated_node_ids: TypedKeyGroup,
@ -79,7 +76,7 @@ pub struct BucketEntryInner {
/// unreachable may now be reachable with the same SignedNodeInfo/DialInfo
updated_since_last_network_change: bool,
/// The last connection descriptors used to contact this node, per protocol type
#[with(Skip)]
#[serde(skip)]
last_connections: BTreeMap<LastConnectionKey, (ConnectionDescriptor, Timestamp)>,
/// The node info for this entry on the publicinternet routing domain
public_internet: BucketEntryPublicInternet,
@ -88,18 +85,18 @@ pub struct BucketEntryInner {
/// Statistics gathered for the peer
peer_stats: PeerStats,
/// The accounting for the latency statistics
#[with(Skip)]
#[serde(skip)]
latency_stats_accounting: LatencyStatsAccounting,
/// The accounting for the transfer statistics
#[with(Skip)]
#[serde(skip)]
transfer_stats_accounting: TransferStatsAccounting,
/// Tracking identifier for NodeRef debugging
#[cfg(feature = "tracking")]
#[with(Skip)]
#[serde(skip)]
next_track_id: usize,
/// Backtraces for NodeRef debugging
#[cfg(feature = "tracking")]
#[with(Skip)]
#[serde(skip)]
node_ref_tracks: HashMap<usize, backtrace::Backtrace>,
}

41
veilid-core/src/routing_table/mod.rs
View File

@ -48,6 +48,12 @@ pub const PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS: u32 = 1;
// We should ping them with some frequency and 30 seconds is typical timeout
pub const CONNECTIONLESS_TIMEOUT_SECS: u32 = 29;
// Table store keys
const ALL_ENTRY_BYTES: &[u8] = b"all_entry_bytes";
const ROUTING_TABLE: &str = "routing_table";
const SERIALIZED_BUCKET_MAP: &[u8] = b"serialized_bucket_map";
const CACHE_VALIDITY_KEY: &[u8] = b"cache_validity_key";
pub type LowLevelProtocolPorts = BTreeSet<(LowLevelProtocolType, AddressType, u16)>;
pub type ProtocolToPortMapping = BTreeMap<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>;
#[derive(Clone, Debug)]
@ -295,7 +301,7 @@ impl RoutingTable {
}
/// Serialize the routing table.
fn serialized_buckets(&self) -> EyreResult<(SerializedBucketMap, SerializedBuckets)> {
fn serialized_buckets(&self) -> (SerializedBucketMap, SerializedBuckets) {
// Since entries are shared by multiple buckets per cryptokind
// we need to get the list of all unique entries when serializing
let mut all_entries: Vec<Arc<BucketEntry>> = Vec::new();
@ -309,7 +315,7 @@ impl RoutingTable {
let buckets = inner.buckets.get(&ck).unwrap();
let mut serialized_buckets = Vec::new();
for bucket in buckets.iter() {
serialized_buckets.push(bucket.save_bucket(&mut all_entries, &mut entry_map)?)
serialized_buckets.push(bucket.save_bucket(&mut all_entries, &mut entry_map))
}
serialized_bucket_map.insert(ck, serialized_buckets);
}
@ -319,25 +325,25 @@ impl RoutingTable {
let mut all_entry_bytes = Vec::with_capacity(all_entries.len());
for entry in all_entries {
// Serialize entry
let entry_bytes = entry.with_inner(|e| to_rkyv(e))?;
let entry_bytes = entry.with_inner(|e| serialize_json_bytes(e));
all_entry_bytes.push(entry_bytes);
}
Ok((serialized_bucket_map, all_entry_bytes))
(serialized_bucket_map, all_entry_bytes)
}
/// Write the serialized routing table to the table store.
async fn save_buckets(&self) -> EyreResult<()> {
let (serialized_bucket_map, all_entry_bytes) = self.serialized_buckets()?;
let (serialized_bucket_map, all_entry_bytes) = self.serialized_buckets();
let table_store = self.unlocked_inner.network_manager().table_store();
let tdb = table_store.open("routing_table", 1).await?;
let tdb = table_store.open(ROUTING_TABLE, 1).await?;
let dbx = tdb.transact();
if let Err(e) = dbx.store_rkyv(0, b"serialized_bucket_map", &serialized_bucket_map) {
if let Err(e) = dbx.store_json(0, SERIALIZED_BUCKET_MAP, &serialized_bucket_map) {
dbx.rollback();
return Err(e.into());
}
if let Err(e) = dbx.store_rkyv(0, b"all_entry_bytes", &all_entry_bytes) {
if let Err(e) = dbx.store_json(0, ALL_ENTRY_BYTES, &all_entry_bytes) {
dbx.rollback();
return Err(e.into());
}
@ -362,9 +368,9 @@ impl RoutingTable {
// Deserialize bucket map and all entries from the table store
let table_store = self.unlocked_inner.network_manager().table_store();
let db = table_store.open("routing_table", 1).await?;
let db = table_store.open(ROUTING_TABLE, 1).await?;
let caches_valid = match db.load(0, b"cache_validity_key").await? {
let caches_valid = match db.load(0, CACHE_VALIDITY_KEY).await? {
Some(v) => v == cache_validity_key,
None => false,
};
@ -372,19 +378,18 @@ impl RoutingTable {
// Caches not valid, start over
log_rtab!(debug "cache validity key changed, emptying routing table");
drop(db);
table_store.delete("routing_table").await?;
let db = table_store.open("routing_table", 1).await?;
db.store(0, b"cache_validity_key", &cache_validity_key)
.await?;
table_store.delete(ROUTING_TABLE).await?;
let db = table_store.open(ROUTING_TABLE, 1).await?;
db.store(0, CACHE_VALIDITY_KEY, &cache_validity_key).await?;
return Ok(());
}
// Caches valid, load saved routing table
let Some(serialized_bucket_map): Option<SerializedBucketMap> = db.load_rkyv(0, b"serialized_bucket_map").await? else {
let Some(serialized_bucket_map): Option<SerializedBucketMap> = db.load_json(0, SERIALIZED_BUCKET_MAP).await? else {
log_rtab!(debug "no bucket map in saved routing table");
return Ok(());
};
let Some(all_entry_bytes): Option<SerializedBuckets> = db.load_rkyv(0, b"all_entry_bytes").await? else {
let Some(all_entry_bytes): Option<SerializedBuckets> = db.load_json(0, ALL_ENTRY_BYTES).await? else {
log_rtab!(debug "no all_entry_bytes in saved routing table");
return Ok(());
};
@ -405,8 +410,8 @@ impl RoutingTable {
) -> EyreResult<()> {
let mut all_entries: Vec<Arc<BucketEntry>> = Vec::with_capacity(all_entry_bytes.len());
for entry_bytes in all_entry_bytes {
let entryinner =
from_rkyv(entry_bytes).wrap_err("failed to deserialize bucket entry")?;
let entryinner = deserialize_json_bytes(&entry_bytes)
.wrap_err("failed to deserialize bucket entry")?;
let entry = Arc::new(BucketEntry::new_with_inner(entryinner));
// Keep strong reference in table

11
veilid-core/src/routing_table/route_spec_store/route_set_spec_detail.rs
View File

@ -1,7 +1,6 @@
use super::*;
#[derive(Clone, Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RouteSpecDetail {
/// Crypto kind
pub crypto_kind: CryptoKind,
@ -11,20 +10,18 @@ pub struct RouteSpecDetail {
pub hops: Vec<PublicKey>,
}
#[derive(Clone, Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RouteSetSpecDetail {
/// Route set per crypto kind
route_set: BTreeMap<PublicKey, RouteSpecDetail>,
/// Route noderefs
#[with(Skip)]
#[serde(skip)]
hop_node_refs: Vec<NodeRef>,
/// Published private route, do not reuse for ephemeral routes
/// Not serialized because all routes should be re-published when restarting
#[with(Skip)]
#[serde(skip)]
published: bool,
/// Directions this route is guaranteed to work in
#[with(RkyvEnumSet)]
directions: DirectionSet,
/// Stability preference (prefer reliable nodes over faster)
stability: Stability,

13
veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs
View File

@ -1,8 +1,7 @@
use super::*;
/// The core representation of the RouteSpecStore that can be serialized
#[derive(Debug, Clone, Default, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RouteSpecStoreContent {
/// All of the route sets we have allocated so far indexed by key
id_by_key: HashMap<PublicKey, RouteId>,
@ -23,7 +22,7 @@ impl RouteSpecStoreContent {
let table_store = routing_table.network_manager().table_store();
let rsstdb = table_store.open("RouteSpecStore", 1).await?;
let mut content: RouteSpecStoreContent =
rsstdb.load_rkyv(0, b"content").await?.unwrap_or_default();
rsstdb.load_json(0, b"content").await?.unwrap_or_default();
// Look up all route hop noderefs since we can't serialize those
let mut dead_ids = Vec::new();
@ -63,7 +62,7 @@ impl RouteSpecStoreContent {
// This skips #[with(Skip)] saving the secret keys, we save them in the protected store instead
let table_store = routing_table.network_manager().table_store();
let rsstdb = table_store.open("RouteSpecStore", 1).await?;
rsstdb.store_rkyv(0, b"content", self).await?;
rsstdb.store_json(0, b"content", self).await?;
Ok(())
}
@ -96,9 +95,9 @@ impl RouteSpecStoreContent {
pub fn get_id_by_key(&self, key: &PublicKey) -> Option<RouteId> {
self.id_by_key.get(key).cloned()
}
pub fn iter_ids(&self) -> std::collections::hash_map::Keys<RouteId, RouteSetSpecDetail> {
self.details.keys()
}
// pub fn iter_ids(&self) -> std::collections::hash_map::Keys<RouteId, RouteSetSpecDetail> {
// self.details.keys()
// }
pub fn iter_details(&self) -> std::collections::hash_map::Iter<RouteId, RouteSetSpecDetail> {
self.details.iter()
}

17
veilid-core/src/routing_table/route_spec_store/route_stats.rs
View File

@ -1,34 +1,33 @@
use super::*;
#[derive(Clone, Debug, Default, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct RouteStats {
/// Consecutive failed to send count
#[with(Skip)]
#[serde(skip)]
pub failed_to_send: u32,
/// Questions lost
#[with(Skip)]
#[serde(skip)]
pub questions_lost: u32,
/// Timestamp of when the route was created
pub created_ts: Timestamp,
/// Timestamp of when the route was last checked for validity
#[with(Skip)]
#[serde(skip)]
pub last_tested_ts: Option<Timestamp>,
/// Timestamp of when the route was last sent to
#[with(Skip)]
#[serde(skip)]
pub last_sent_ts: Option<Timestamp>,
/// Timestamp of when the route was last received over
#[with(Skip)]
#[serde(skip)]
pub last_received_ts: Option<Timestamp>,
/// Transfers up and down
pub transfer_stats_down_up: TransferStatsDownUp,
/// Latency stats
pub latency_stats: LatencyStats,
/// Accounting mechanism for this route's RPC latency
#[with(Skip)]
#[serde(skip)]
latency_stats_accounting: LatencyStatsAccounting,
/// Accounting mechanism for the bandwidth across this route
#[with(Skip)]
#[serde(skip)]
transfer_stats_accounting: TransferStatsAccounting,
}

2
veilid-core/src/routing_table/tests/test_serialize_routing_table.rs
View File

@ -35,7 +35,7 @@ pub async fn test_routingtable_buckets_round_trip() {
// Add lots of routes to `original` here to exercise all various types.
let (serialized_bucket_map, all_entry_bytes) = original.serialized_buckets().unwrap();
let (serialized_bucket_map, all_entry_bytes) = original.serialized_buckets();
copy.populate_routing_table(
&mut copy.inner.write(),

16
veilid-core/src/routing_table/types/dial_info_detail.rs
View File

@ -1,21 +1,7 @@
use super::*;
// Keep member order appropriate for sorting < preference
#[derive(
Debug,
Clone,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoDetail {
pub class: DialInfoClass,
pub dial_info: DialInfo,

20
veilid-core/src/routing_table/types/direction.rs
View File

@ -1,22 +1,12 @@
#![allow(non_snake_case)]
use super::*;
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
PartialOrd,
Ord,
Hash,
EnumSetType,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[derive(Debug, PartialOrd, Ord, Hash, EnumSetType, Serialize, Deserialize)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum Direction {
Inbound,
Outbound,
Inbound = 0,
Outbound = 1,
}
pub type DirectionSet = EnumSet<Direction>;

53
veilid-core/src/routing_table/types/node_info.rs
View File

@ -12,61 +12,10 @@ pub const CAP_APPMESSAGE: Capability = FourCC(*b"APPM");
#[cfg(feature = "unstable-blockstore")]
pub const CAP_BLOCKSTORE: Capability = FourCC(*b"BLOC");
cfg_if! {
if #[cfg(all(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 8;
} else if #[cfg(any(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 7;
} else {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 6;
}
}
pub const PUBLIC_INTERNET_CAPABILITIES: [Capability; PUBLIC_INTERNET_CAPABILITIES_LEN] = [
CAP_ROUTE,
#[cfg(feature = "unstable-tunnels")]
CAP_TUNNEL,
CAP_SIGNAL,
CAP_RELAY,
CAP_VALIDATE_DIAL_INFO,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
#[cfg(feature = "unstable-blockstore")]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 4;
#[cfg(not(feature = "unstable-blockstore"))]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 3;
pub const LOCAL_NETWORK_CAPABILITIES: [Capability; LOCAL_NETWORK_CAPABILITIES_LEN] = [
CAP_RELAY,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
pub const MAX_CAPABILITIES: usize = 64;
#[derive(
Clone,
Default,
PartialEq,
Eq,
Debug,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub struct NodeInfo {
network_class: NetworkClass,
#[with(RkyvEnumSet)]
outbound_protocols: ProtocolTypeSet,
#[with(RkyvEnumSet)]
address_types: AddressTypeSet,
envelope_support: Vec<u8>,
crypto_support: Vec<CryptoKind>,

3
veilid-core/src/routing_table/types/node_status.rs
View File

@ -2,8 +2,7 @@ use super::*;
/// Non-nodeinfo status for each node is returned by the StatusA call
#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct NodeStatus {
// Reserved for expansion
}

5
veilid-core/src/routing_table/types/peer_info.rs
View File

@ -1,9 +1,6 @@
use super::*;
#[derive(
Clone, Debug, Serialize, Deserialize, PartialEq, Eq, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct PeerInfo {
node_ids: TypedKeyGroup,
signed_node_info: SignedNodeInfo,

14
veilid-core/src/routing_table/types/routing_domain.rs
View File

@ -4,20 +4,8 @@ use super::*;
// Routing domain here is listed in order of preference, keep in order
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
Ord,
PartialOrd,
Hash,
EnumSetType,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[derive(Debug, Ord, PartialOrd, Hash, EnumSetType, Serialize, Deserialize)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum RoutingDomain {
LocalNetwork = 0,
PublicInternet = 1,

5
veilid-core/src/routing_table/types/signed_direct_node_info.rs
View File

@ -1,10 +1,7 @@
use super::*;
/// Signed NodeInfo that can be passed around amongst peers and verifiable
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SignedDirectNodeInfo {
node_info: NodeInfo,
timestamp: Timestamp,

5
veilid-core/src/routing_table/types/signed_node_info.rs
View File

@ -1,9 +1,6 @@
use super::*;
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum SignedNodeInfo {
Direct(SignedDirectNodeInfo),
Relayed(SignedRelayedNodeInfo),

5
veilid-core/src/routing_table/types/signed_relayed_node_info.rs
View File

@ -1,10 +1,7 @@
use super::*;
/// Signed NodeInfo with a relay that can be passed around amongst peers and verifiable
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SignedRelayedNodeInfo {
node_info: NodeInfo,
relay_ids: TypedKeyGroup,

22
veilid-core/src/storage_manager/record_store.rs
View File

@ -9,9 +9,7 @@ use hashlink::LruCache;
pub struct RecordStore<D>
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize + for<'d> Deserialize<'d>,
{
table_store: TableStore,
name: String,
@ -41,9 +39,7 @@ pub struct SubkeyResult {
impl<D> RecordStore<D>
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize + for<'d> Deserialize<'d>,
{
pub fn new(table_store: TableStore, name: &str, limits: RecordStoreLimits) -> Self {
let subkey_cache_size = limits.subkey_cache_size as usize;
@ -93,7 +89,7 @@ where
let mut record_index_saved: Vec<(RecordTableKey, Record<D>)> =
Vec::with_capacity(record_table_keys.len());
for rtk in record_table_keys {
if let Some(vr) = record_table.load_rkyv::<Record<D>>(0, &rtk).await? {
if let Some(vr) = record_table.load_json::<Record<D>>(0, &rtk).await? {
let rik = RecordTableKey::try_from(rtk.as_ref())?;
record_index_saved.push((rik, vr));
}
@ -265,7 +261,7 @@ where
for rtk in changed_records {
// Get the changed record and save it to the table
if let Some(r) = self.record_index.peek(&rtk) {
if let Err(e) = rt_xact.store_rkyv(0, &rtk.bytes(), r) {
if let Err(e) = rt_xact.store_json(0, &rtk.bytes(), r) {
log_stor!(error "failed to save record: {}", e);
}
}
@ -303,7 +299,7 @@ where
// Save to record table
record_table
.store_rkyv(0, &rtk.bytes(), &record)
.store_json(0, &rtk.bytes(), &record)
.await
.map_err(VeilidAPIError::internal)?;
@ -451,7 +447,7 @@ where
}
// If not in cache, try to pull from table store if it is in our stored subkey set
let Some(record_data) = subkey_table
.load_rkyv::<RecordData>(0, &stk.bytes())
.load_json::<RecordData>(0, &stk.bytes())
.await
.map_err(VeilidAPIError::internal)? else {
apibail_internal!("failed to get subkey that was stored");
@ -517,7 +513,7 @@ where
}
// If not in cache, try to pull from table store if it is in our stored subkey set
let Some(record_data) = subkey_table
.load_rkyv::<RecordData>(0, &stk.bytes())
.load_json::<RecordData>(0, &stk.bytes())
.await
.map_err(VeilidAPIError::internal)? else {
apibail_internal!("failed to peek subkey that was stored");
@ -575,7 +571,7 @@ where
} else {
// If not in cache, try to pull from table store
if let Some(record_data) = subkey_table
.load_rkyv::<RecordData>(0, &stk_bytes)
.load_json::<RecordData>(0, &stk_bytes)
.await
.map_err(VeilidAPIError::internal)?
{
@ -606,7 +602,7 @@ where
// Write subkey
subkey_table
.store_rkyv(0, &stk_bytes, &record_data)
.store_json(0, &stk_bytes, &record_data)
.await
.map_err(VeilidAPIError::internal)?;

15
veilid-core/src/storage_manager/storage_manager_inner.rs
View File

@ -1,5 +1,8 @@
use super::*;
const STORAGE_MANAGER_METADATA: &str = "storage_manager_metadata";
const OFFLINE_SUBKEY_WRITES: &[u8] = b"offline_subkey_writes";
/// Locked structure for storage manager
pub(super) struct StorageManagerInner {
unlocked_inner: Arc<StorageManagerUnlockedInner>,
@ -68,7 +71,7 @@ impl StorageManagerInner {
let metadata_db = self.unlocked_inner
.table_store
.open(&format!("storage_manager_metadata"), 1)
.open(STORAGE_MANAGER_METADATA, 1)
.await?;
let local_limits = local_limits_from_config(self.unlocked_inner.config.clone());
@ -146,7 +149,7 @@ impl StorageManagerInner {
async fn save_metadata(&mut self) -> EyreResult<()>{
if let Some(metadata_db) = &self.metadata_db {
let tx = metadata_db.transact();
tx.store_rkyv(0, b"offline_subkey_writes", &self.offline_subkey_writes)?;
tx.store_json(0, OFFLINE_SUBKEY_WRITES, &self.offline_subkey_writes)?;
tx.commit().await.wrap_err("failed to commit")?
}
Ok(())
@ -154,10 +157,10 @@ impl StorageManagerInner {
async fn load_metadata(&mut self) -> EyreResult<()> {
if let Some(metadata_db) = &self.metadata_db {
self.offline_subkey_writes = match metadata_db.load_rkyv(0, b"offline_subkey_writes").await {
self.offline_subkey_writes = match metadata_db.load_json(0, OFFLINE_SUBKEY_WRITES).await {
Ok(v) => v.unwrap_or_default(),
Err(_) => {
if let Err(e) = metadata_db.delete(0,b"offline_subkey_writes").await {
if let Err(e) = metadata_db.delete(0, OFFLINE_SUBKEY_WRITES).await {
debug!("offline_subkey_writes format changed, clearing: {}", e);
}
Default::default()
@ -483,9 +486,7 @@ impl StorageManagerInner {
/// # DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ]
fn get_key<D>(vcrypto: CryptoSystemVersion, record: &Record<D>) -> TypedKey
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize,
{
let compiled = record.descriptor().schema_data();
let mut hash_data = Vec::<u8>::with_capacity(PUBLIC_KEY_LENGTH + 4 + compiled.len());

5
veilid-core/src/storage_manager/types/local_record_detail.rs
View File

@ -1,10 +1,7 @@
use super::*;
/// Information required to handle locally opened records
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalRecordDetail {
/// The last 'safety selection' used when creating/opening this record.
/// Even when closed, this safety selection applies to re-publication attempts by the system.

19
veilid-core/src/storage_manager/types/record.rs
View File

@ -1,14 +1,9 @@
use super::*;
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Record<D>
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize,
{
descriptor: SignedValueDescriptor,
subkey_count: usize,
@ -20,9 +15,7 @@ where
impl<D> Record<D>
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize,
{
pub fn new(
cur_ts: Timestamp,
@ -84,9 +77,9 @@ where
mem::size_of::<Record<D>>() + self.descriptor.total_size() + self.record_data_size
}
pub fn detail(&self) -> &D {
&self.detail
}
// pub fn detail(&self) -> &D {
// &self.detail
// }
pub fn detail_mut(&mut self) -> &mut D {
&mut self.detail
}

15
veilid-core/src/storage_manager/types/record_data.rs
View File

@ -1,19 +1,6 @@
use super::*;
#[derive(
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub struct RecordData {
signed_value_data: SignedValueData,
}

5
veilid-core/src/storage_manager/types/remote_record_detail.rs
View File

@ -1,7 +1,4 @@
use super::*;
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RemoteRecordDetail {}

15
veilid-core/src/storage_manager/types/signed_value_data.rs
View File

@ -3,20 +3,7 @@ use super::*;
/////////////////////////////////////////////////////////////////////////////////////////////////////
///
#[derive(
Clone,
Debug,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)]
pub struct SignedValueData {
value_data: ValueData,
signature: Signature,

14
veilid-core/src/storage_manager/types/signed_value_descriptor.rs
View File

@ -3,19 +3,7 @@ use super::*;
/////////////////////////////////////////////////////////////////////////////////////////////////////
///
#[derive(
Clone,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)]
pub struct SignedValueDescriptor {
owner: PublicKey,
schema_data: Vec<u8>,

50
veilid-core/src/table_store/table_db.rs
View File

@ -195,15 +195,6 @@ impl TableDB {
db.write(dbt).await.map_err(VeilidAPIError::generic)
}
/// Store a key in rkyv format with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let value = to_rkyv(value)?;
self.store(col, key, &value).await
}
/// Store a key in json format with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
@ -230,21 +221,6 @@ impl TableDB {
.map(|v| self.maybe_decrypt(&v)))
}
/// Read an rkyv key from a column in the TableDB immediately
pub async fn load_rkyv<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = match self.load(col, key).await? {
Some(v) => Some(from_rkyv(v)?),
None => None,
};
Ok(out)
}
/// Read an serde-json key from a column in the TableDB immediately
pub async fn load_json<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
@ -276,21 +252,6 @@ impl TableDB {
Ok(old_value)
}
/// Delete rkyv key with from a column in the TableDB
pub async fn delete_rkyv<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let old_value = match self.delete(col, key).await? {
Some(v) => Some(from_rkyv(v)?),
None => None,
};
Ok(old_value)
}
/// Delete serde-json key with from a column in the TableDB
pub async fn delete_json<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
@ -377,16 +338,7 @@ impl TableDBTransaction {
Ok(())
}
/// Store a key in rkyv format with a value in a column in the TableDB
pub fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let value = to_rkyv(value)?;
self.store(col, key, &value)
}
/// Store a key in rkyv format with a value in a column in the TableDB
/// Store a key in json format with a value in a column in the TableDB
pub fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: serde::Serialize,

11
veilid-core/src/table_store/table_store.rs
View File

@ -1,6 +1,8 @@
use super::*;
use keyvaluedb::*;
const ALL_TABLE_NAMES: &[u8] = b"all_table_names";
struct TableStoreInner {
opened: BTreeMap<String, Weak<TableDBUnlockedInner>>,
encryption_key: Option<TypedSharedSecret>,
@ -52,12 +54,11 @@ impl TableStore {
async fn flush(&self) {
let (all_table_names_value, all_tables_db) = {
let inner = self.inner.lock();
let all_table_names_value =
to_rkyv(&inner.all_table_names).expect("failed to archive all_table_names");
let all_table_names_value = serialize_json_bytes(&inner.all_table_names);
(all_table_names_value, inner.all_tables_db.clone().unwrap())
};
let mut dbt = DBTransaction::new();
dbt.put(0, b"all_table_names", &all_table_names_value);
dbt.put(0, ALL_TABLE_NAMES, &all_table_names_value);
if let Err(e) = all_tables_db.write(dbt).await {
error!("failed to write all tables db: {}", e);
}
@ -373,8 +374,8 @@ impl TableStore {
.open("__veilid_all_tables", 1)
.await
.wrap_err("failed to create all tables table")?;
match all_tables_db.get(0, b"all_table_names").await {
Ok(Some(v)) => match from_rkyv::<HashMap<String, String>>(v) {
match all_tables_db.get(0, ALL_TABLE_NAMES).await {
Ok(Some(v)) => match deserialize_json_bytes::<HashMap<String, String>>(&v) {
Ok(all_table_names) => {
let mut inner = self.inner.lock();
inner.all_table_names = all_table_names;

44
veilid-core/src/table_store/tests/test_table_store.rs
View File

@ -145,7 +145,6 @@ pub async fn test_transaction(ts: TableStore) {
let tx = db.transact();
assert!(tx.store(0, b"aaa", b"a-value").is_ok());
assert!(tx.store_json(1, b"bbb", &"b-value".to_owned()).is_ok());
assert!(tx.store_rkyv(2, b"ccc", &"c-value".to_owned()).is_ok());
assert!(tx.store(3, b"ddd", b"d-value").is_err());
assert!(tx.store(0, b"ddd", b"d-value").is_ok());
assert!(tx.delete(0, b"ddd").is_ok());
@ -160,51 +159,9 @@ pub async fn test_transaction(ts: TableStore) {
db.load_json::<String>(1, b"bbb").await,
Ok(Some("b-value".to_owned()))
);
assert_eq!(
db.load_rkyv::<String>(2, b"ccc").await,
Ok(Some("c-value".to_owned()))
);
assert_eq!(db.load(0, b"ddd").await, Ok(None));
}
pub async fn test_rkyv(vcrypto: CryptoSystemVersion, ts: TableStore) {
trace!("test_rkyv");
let _ = ts.delete("test");
let db = ts.open("test", 3).await.expect("should have opened");
let keypair = vcrypto.generate_keypair();
assert!(db.store_rkyv(0, b"asdf", &keypair).await.is_ok());
assert_eq!(db.load_rkyv::<KeyPair>(0, b"qwer").await.unwrap(), None);
let d = match db.load_rkyv::<KeyPair>(0, b"asdf").await {
Ok(x) => x,
Err(e) => {
panic!("couldn't decode: {}", e);
}
};
assert_eq!(d, Some(keypair), "keys should be equal");
let d = match db.delete_rkyv::<KeyPair>(0, b"asdf").await {
Ok(x) => x,
Err(e) => {
panic!("couldn't decode: {}", e);
}
};
assert_eq!(d, Some(keypair), "keys should be equal");
assert!(
db.store(1, b"foo", b"1234567890").await.is_ok(),
"should store new key"
);
assert!(
db.load_rkyv::<TypedKey>(1, b"foo").await.is_err(),
"should fail to unfreeze"
);
}
pub async fn test_json(vcrypto: CryptoSystemVersion, ts: TableStore) {
trace!("test_json");
@ -304,7 +261,6 @@ pub async fn test_all() {
test_delete_open_delete(ts.clone()).await;
test_store_delete_load(ts.clone()).await;
test_transaction(ts.clone()).await;
test_rkyv(vcrypto.clone(), ts.clone()).await;
test_json(vcrypto, ts.clone()).await;
let _ = ts.delete("test").await;
}

4
veilid-core/src/tests/native/mod.rs
View File

@ -35,8 +35,6 @@ pub async fn run_all_tests() {
test_envelope_receipt::test_all().await;
info!("TEST: veilid_api::tests::test_serialize_json");
veilid_api::tests::test_serialize_json::test_all().await;
info!("TEST: veilid_api::tests::test_serialize_rkyv");
veilid_api::tests::test_serialize_rkyv::test_all().await;
info!("TEST: routing_table::test_serialize_routing_table");
routing_table::tests::test_serialize_routing_table::test_all().await;
@ -138,8 +136,6 @@ cfg_if! {
run_test!(veilid_api, test_serialize_json);
run_test!(veilid_api, test_serialize_rkyv);
run_test!(routing_table, test_serialize_routing_table);
}
}

19
veilid-core/src/veilid_api/error.rs
View File

@ -105,21 +105,8 @@ macro_rules! apibail_already_initialized {
}
#[derive(
ThisError,
Clone,
Debug,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
ThisError, Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[serde(tag = "kind")]
pub enum VeilidAPIError {
#[error("Not initialized")]
@ -138,8 +125,8 @@ pub enum VeilidAPIError {
NoConnection { message: String },
#[error("Key not found: {key}")]
KeyNotFound {
#[schemars(with="String")]
key: TypedKey
#[schemars(with = "String")]
key: TypedKey,
},
#[error("Internal: {message}")]
Internal { message: String },

32
veilid-core/src/veilid_api/json_api/crypto_system.rs
View File

@ -35,24 +35,24 @@ pub enum CryptoSystemRequestOp {
},
DefaultSaltLength,
HashPassword {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
password: Vec<u8>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
salt: Vec<u8>,
},
VerifyPassword {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
password: Vec<u8>,
password_hash: String,
},
DeriveSharedSecret {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
password: Vec<u8>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
salt: Vec<u8>,
},
@ -60,7 +60,7 @@ pub enum CryptoSystemRequestOp {
RandomSharedSecret,
GenerateKeyPair,
GenerateHash {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
},
@ -71,7 +71,7 @@ pub enum CryptoSystemRequestOp {
secret: SecretKey,
},
ValidateHash {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
#[schemars(with = "String")]
@ -88,14 +88,14 @@ pub enum CryptoSystemRequestOp {
key: PublicKey,
#[schemars(with = "String")]
secret: SecretKey,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
},
Verify {
#[schemars(with = "String")]
key: PublicKey,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
#[schemars(with = "String")]
@ -103,31 +103,31 @@ pub enum CryptoSystemRequestOp {
},
AeadOverhead,
DecryptAead {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
body: Vec<u8>,
#[schemars(with = "String")]
nonce: Nonce,
#[schemars(with = "String")]
shared_secret: SharedSecret,
#[serde(with = "opt_json_as_base64")]
#[serde(with = "as_human_opt_base64")]
#[schemars(with = "Option<String>")]
associated_data: Option<Vec<u8>>,
},
EncryptAead {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
body: Vec<u8>,
#[schemars(with = "String")]
nonce: Nonce,
#[schemars(with = "String")]
shared_secret: SharedSecret,
#[serde(with = "opt_json_as_base64")]
#[serde(with = "as_human_opt_base64")]
#[schemars(with = "Option<String>")]
associated_data: Option<Vec<u8>>,
},
CryptNoAuth {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
body: Vec<u8>,
#[schemars(with = "String")]
@ -152,7 +152,7 @@ pub enum CryptoSystemResponseOp {
result: ApiResultWithString<SharedSecret>,
},
RandomBytes {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},
@ -221,7 +221,7 @@ pub enum CryptoSystemResponseOp {
result: ApiResultWithVecU8,
},
CryptNoAuth {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},

14
veilid-core/src/veilid_api/json_api/mod.rs
View File

@ -58,7 +58,7 @@ pub enum RequestOp {
sequencing: Sequencing,
},
ImportRemotePrivateRoute {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
blob: Vec<u8>,
},
@ -69,7 +69,7 @@ pub enum RequestOp {
AppCallReply {
#[schemars(with = "String")]
call_id: OperationId,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
message: Vec<u8>,
},
@ -96,14 +96,14 @@ pub enum RequestOp {
VerifySignatures {
#[schemars(with = "Vec<String>")]
node_ids: Vec<TypedKey>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
#[schemars(with = "Vec<String>")]
signatures: Vec<TypedSignature>,
},
GenerateSignatures {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
#[schemars(with = "Vec<String>")]
@ -126,7 +126,7 @@ pub enum RequestOp {
pub struct NewPrivateRouteResult {
#[schemars(with = "String")]
route_id: RouteId,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
blob: Vec<u8>,
}
@ -260,7 +260,7 @@ where
#[serde(untagged)]
pub enum ApiResultWithVecU8 {
Ok {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},
@ -271,7 +271,7 @@ pub enum ApiResultWithVecU8 {
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
#[serde(transparent)]
pub struct VecU8 {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
}

6
veilid-core/src/veilid_api/json_api/routing_context.rs
View File

@ -27,13 +27,13 @@ pub enum RoutingContextRequestOp {
},
AppCall {
target: String,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
message: Vec<u8>,
},
AppMessage {
target: String,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
message: Vec<u8>,
},
@ -66,7 +66,7 @@ pub enum RoutingContextRequestOp {
#[schemars(with = "String")]
key: TypedKey,
subkey: ValueSubkey,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
},

14
veilid-core/src/veilid_api/json_api/table_db.rs
View File

@ -25,22 +25,22 @@ pub enum TableDbRequestOp {
Transact,
Store {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},
Load {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
},
Delete {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
},
@ -101,16 +101,16 @@ pub enum TableDbTransactionRequestOp {
Rollback,
Store {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},
Delete {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
},

13
veilid-core/src/veilid_api/serialize_helpers/mod.rs
View File

@ -1,14 +1,7 @@
mod rkyv_enum_set;
mod rkyv_range_set_blaze;
pub mod serialize_arc;
mod serialize_json;
pub mod serialize_range_set_blaze;
mod veilid_rkyv;
use super::*;
use core::fmt::Debug;
pub use rkyv_enum_set::*;
pub use rkyv_range_set_blaze::*;
pub mod serialize_arc;
mod serialize_json;
pub mod serialize_range_set_blaze;
pub use serialize_json::*;
pub use veilid_rkyv::*;

53
veilid-core/src/veilid_api/serialize_helpers/rkyv_enum_set.rs
View File

@ -1,53 +0,0 @@
use super::*;
pub struct RkyvEnumSet;
impl<T> rkyv::with::ArchiveWith<EnumSet<T>> for RkyvEnumSet
where
T: EnumSetType + EnumSetTypeWithRepr,
<T as EnumSetTypeWithRepr>::Repr: rkyv::Archive,
{
type Archived = rkyv::Archived<<T as EnumSetTypeWithRepr>::Repr>;
type Resolver = rkyv::Resolver<<T as EnumSetTypeWithRepr>::Repr>;
#[inline]
unsafe fn resolve_with(
field: &EnumSet<T>,
pos: usize,
resolver: Self::Resolver,
out: *mut Self::Archived,
) {
let r = field.as_repr();
r.resolve(pos, resolver, out);
}
}
impl<T, S> rkyv::with::SerializeWith<EnumSet<T>, S> for RkyvEnumSet
where
S: rkyv::Fallible + ?Sized,
T: EnumSetType + EnumSetTypeWithRepr,
<T as EnumSetTypeWithRepr>::Repr: rkyv::Serialize<S>,
{
fn serialize_with(field: &EnumSet<T>, serializer: &mut S) -> Result<Self::Resolver, S::Error> {
let r = field.as_repr();
r.serialize(serializer)
}
}
impl<T, D>
rkyv::with::DeserializeWith<rkyv::Archived<<T as EnumSetTypeWithRepr>::Repr>, EnumSet<T>, D>
for RkyvEnumSet
where
D: rkyv::Fallible + ?Sized,
T: EnumSetType + EnumSetTypeWithRepr,
<T as EnumSetTypeWithRepr>::Repr: rkyv::Archive,
rkyv::Archived<<T as EnumSetTypeWithRepr>::Repr>:
rkyv::Deserialize<<T as EnumSetTypeWithRepr>::Repr, D>,
{
fn deserialize_with(
field: &rkyv::Archived<<T as EnumSetTypeWithRepr>::Repr>,
deserializer: &mut D,
) -> Result<EnumSet<T>, D::Error> {
Ok(EnumSet::<T>::from_repr(field.deserialize(deserializer)?))
}
}

73
veilid-core/src/veilid_api/serialize_helpers/rkyv_range_set_blaze.rs
View File

@ -1,73 +0,0 @@
use super::*;
use range_set_blaze::*;
pub struct RkyvRangeSetBlaze;
impl<T> rkyv::with::ArchiveWith<RangeSetBlaze<T>> for RkyvRangeSetBlaze
where
T: rkyv::Archive + Integer,
{
type Archived = rkyv::Archived<Vec<T>>;
type Resolver = rkyv::Resolver<Vec<T>>;
#[inline]
unsafe fn resolve_with(
field: &RangeSetBlaze<T>,
pos: usize,
resolver: Self::Resolver,
out: *mut Self::Archived,
) {
let mut r = Vec::<T>::with_capacity(field.ranges_len() * 2);
for range in field.ranges() {
r.push(*range.start());
r.push(*range.end());
}
r.resolve(pos, resolver, out);
}
}
impl<T, S> rkyv::with::SerializeWith<RangeSetBlaze<T>, S> for RkyvRangeSetBlaze
where
S: rkyv::Fallible + ?Sized,
Vec<T>: rkyv::Serialize<S>,
T: rkyv::Archive + Integer,
{
fn serialize_with(
field: &RangeSetBlaze<T>,
serializer: &mut S,
) -> Result<Self::Resolver, S::Error> {
let mut r = Vec::<T>::with_capacity(field.ranges_len() * 2);
for range in field.ranges() {
r.push(*range.start());
r.push(*range.end());
}
r.serialize(serializer)
}
}
impl<T, D> rkyv::with::DeserializeWith<rkyv::Archived<Vec<T>>, RangeSetBlaze<T>, D>
for RkyvRangeSetBlaze
where
D: rkyv::Fallible + ?Sized,
T: rkyv::Archive + Integer,
rkyv::Archived<T>: rkyv::Deserialize<T, D>,
D::Error: From<String>,
{
fn deserialize_with(
field: &rkyv::Archived<Vec<T>>,
deserializer: &mut D,
) -> Result<RangeSetBlaze<T>, D::Error> {
let mut out = RangeSetBlaze::<T>::new();
if field.len() % 2 == 1 {
return Err("invalid range set length".to_owned().into());
}
let f = field.as_slice();
for i in 0..field.len() / 2 {
let l: T = f[i * 2].deserialize(deserializer)?;
let u: T = f[i * 2 + 1].deserialize(deserializer)?;
out.ranges_insert(l..=u);
}
Ok(out)
}
}

213
veilid-core/src/veilid_api/serialize_helpers/serialize_json.rs
View File

@ -13,6 +13,18 @@ pub fn deserialize_json<'a, T: de::Deserialize<'a> + Debug>(arg: &'a str) -> Vei
),
})
}
pub fn deserialize_json_bytes<'a, T: de::Deserialize<'a> + Debug>(
arg: &'a [u8],
) -> VeilidAPIResult<T> {
serde_json::from_slice(arg).map_err(|e| VeilidAPIError::ParseError {
message: e.to_string(),
value: format!(
"deserialize_json_bytes:\n---\n{:?}\n---\n to type {}",
arg,
std::any::type_name::<T>()
),
})
}
// #[instrument(level = "trace", ret, err)]
pub fn deserialize_opt_json<T: de::DeserializeOwned + Debug>(
@ -28,6 +40,20 @@ pub fn deserialize_opt_json<T: de::DeserializeOwned + Debug>(
deserialize_json(arg)
}
// #[instrument(level = "trace", ret, err)]
pub fn deserialize_opt_json_bytes<T: de::DeserializeOwned + Debug>(
arg: Option<Vec<u8>>,
) -> VeilidAPIResult<T> {
let arg = arg.as_ref().ok_or_else(|| VeilidAPIError::ParseError {
message: "invalid null string".to_owned(),
value: format!(
"deserialize_json_opt: null to type {}",
std::any::type_name::<T>()
),
})?;
deserialize_json_bytes(arg.as_slice())
}
// #[instrument(level = "trace", ret)]
pub fn serialize_json<T: Serialize + Debug>(val: T) -> String {
match serde_json::to_string(&val) {
@ -37,97 +63,138 @@ pub fn serialize_json<T: Serialize + Debug>(val: T) -> String {
}
}
}
pub fn serialize_json_bytes<T: Serialize + Debug>(val: T) -> Vec<u8> {
match serde_json::to_vec(&val) {
Ok(v) => v,
Err(e) => {
panic!(
"failed to serialize json value to bytes: {}\nval={:?}",
e, val
);
}
}
}
pub mod json_as_base64 {
pub mod as_human_base64 {
use data_encoding::BASE64URL_NOPAD;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<S: Serializer>(v: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
let base64 = BASE64URL_NOPAD.encode(v);
String::serialize(&base64, s)
if s.is_human_readable() {
let base64 = BASE64URL_NOPAD.encode(v);
String::serialize(&base64, s)
} else {
Vec::<u8>::serialize(v, s)
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
let base64 = String::deserialize(d)?;
BASE64URL_NOPAD
.decode(base64.as_bytes())
.map_err(|e| serde::de::Error::custom(e))
if d.is_human_readable() {
let base64 = String::deserialize(d)?;
BASE64URL_NOPAD
.decode(base64.as_bytes())
.map_err(|e| serde::de::Error::custom(e))
} else {
Vec::<u8>::deserialize(d)
}
}
}
pub mod opt_json_as_base64 {
pub mod as_human_opt_base64 {
use data_encoding::BASE64URL_NOPAD;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<S: Serializer>(v: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
let base64 = v.as_ref().map(|x| BASE64URL_NOPAD.encode(&x));
Option::<String>::serialize(&base64, s)
if s.is_human_readable() {
let base64 = v.as_ref().map(|x| BASE64URL_NOPAD.encode(&x));
Option::<String>::serialize(&base64, s)
} else {
Option::<Vec<u8>>::serialize(v, s)
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Vec<u8>>, D::Error> {
let base64 = Option::<String>::deserialize(d)?;
base64
.map(|x| {
BASE64URL_NOPAD
.decode(x.as_bytes())
.map_err(|e| serde::de::Error::custom(e))
})
.transpose()
}
}
pub mod json_as_string {
use std::fmt::Display;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serializer};
pub fn serialize<T, S>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
where
T: Display,
S: Serializer,
{
serializer.collect_str(value)
}
pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
where
T: FromStr,
T::Err: Display,
D: Deserializer<'de>,
{
String::deserialize(deserializer)?
.parse()
.map_err(de::Error::custom)
}
}
pub mod opt_json_as_string {
use std::fmt::Display;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serializer};
pub fn serialize<T, S>(value: &Option<T>, serializer: S) -> Result<S::Ok, S::Error>
where
T: Display,
S: Serializer,
{
match value {
Some(v) => serializer.collect_str(v),
None => serializer.serialize_none(),
}
}
pub fn deserialize<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
where
T: FromStr,
T::Err: Display,
D: Deserializer<'de>,
{
match Option::<String>::deserialize(deserializer)? {
None => Ok(None),
Some(v) => Ok(Some(v.parse::<T>().map_err(de::Error::custom)?)),
if d.is_human_readable() {
let base64 = Option::<String>::deserialize(d)?;
base64
.map(|x| {
BASE64URL_NOPAD
.decode(x.as_bytes())
.map_err(|e| serde::de::Error::custom(e))
})
.transpose()
} else {
Option::<Vec<u8>>::deserialize(d)
}
}
}
pub mod as_human_string {
use std::fmt::Display;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<T, S>(value: &T, s: S) -> Result<S::Ok, S::Error>
where
T: Display + Serialize,
S: Serializer,
{
if s.is_human_readable() {
s.collect_str(value)
} else {
T::serialize(value, s)
}
}
pub fn deserialize<'de, T, D>(d: D) -> Result<T, D::Error>
where
T: FromStr + Deserialize<'de>,
T::Err: Display,
D: Deserializer<'de>,
{
if d.is_human_readable() {
String::deserialize(d)?.parse().map_err(de::Error::custom)
} else {
T::deserialize(d)
}
}
}
pub mod as_human_opt_string {
use std::fmt::Display;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<T, S>(value: &Option<T>, s: S) -> Result<S::Ok, S::Error>
where
T: Display + Serialize,
S: Serializer,
{
if s.is_human_readable() {
match value {
Some(v) => s.collect_str(v),
None => s.serialize_none(),
}
} else {
Option::<T>::serialize(value, s)
}
}
pub fn deserialize<'de, T, D>(d: D) -> Result<Option<T>, D::Error>
where
T: FromStr + Deserialize<'de>,
T::Err: Display,
D: Deserializer<'de>,
{
if d.is_human_readable() {
match Option::<String>::deserialize(d)? {
None => Ok(None),
Some(v) => Ok(Some(v.parse::<T>().map_err(de::Error::custom)?)),
}
} else {
Option::<T>::deserialize(d)
}
}
}

151
veilid-core/src/veilid_api/serialize_helpers/veilid_rkyv.rs
View File

@ -1,151 +0,0 @@
use super::*;
use rkyv::ser::Serializer;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
pub struct VeilidRkyvSerializer<S> {
inner: S,
}
impl<S> VeilidRkyvSerializer<S> {
pub fn into_inner(self) -> S {
self.inner
}
}
impl<S: rkyv::Fallible> rkyv::Fallible for VeilidRkyvSerializer<S> {
type Error = VeilidRkyvError<S::Error>;
}
impl<S: rkyv::ser::ScratchSpace> rkyv::ser::ScratchSpace for VeilidRkyvSerializer<S> {
unsafe fn push_scratch(
&mut self,
layout: core::alloc::Layout,
) -> Result<core::ptr::NonNull<[u8]>, Self::Error> {
self.inner
.push_scratch(layout)
.map_err(VeilidRkyvError::Inner)
}
unsafe fn pop_scratch(
&mut self,
ptr: core::ptr::NonNull<u8>,
layout: core::alloc::Layout,
) -> Result<(), Self::Error> {
self.inner
.pop_scratch(ptr, layout)
.map_err(VeilidRkyvError::Inner)
}
}
impl<S: rkyv::ser::Serializer> rkyv::ser::Serializer for VeilidRkyvSerializer<S> {
#[inline]
fn pos(&self) -> usize {
self.inner.pos()
}
#[inline]
fn write(&mut self, bytes: &[u8]) -> Result<(), Self::Error> {
self.inner.write(bytes).map_err(VeilidRkyvError::Inner)
}
}
impl<S: Default> Default for VeilidRkyvSerializer<S> {
fn default() -> Self {
Self {
inner: S::default(),
}
}
}
pub type DefaultVeilidRkyvSerializer =
VeilidRkyvSerializer<rkyv::ser::serializers::AllocSerializer<1024>>;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#[derive(Debug, Default)]
pub struct VeilidSharedDeserializeMap {
inner: SharedDeserializeMap,
}
impl VeilidSharedDeserializeMap {
#[inline]
pub fn new() -> Self {
Self {
inner: SharedDeserializeMap::new(),
}
}
}
impl rkyv::Fallible for VeilidSharedDeserializeMap {
type Error = VeilidRkyvError<rkyv::de::deserializers::SharedDeserializeMapError>;
}
impl rkyv::de::SharedDeserializeRegistry for VeilidSharedDeserializeMap {
fn get_shared_ptr(&mut self, ptr: *const u8) -> Option<&dyn rkyv::de::SharedPointer> {
self.inner.get_shared_ptr(ptr)
}
fn add_shared_ptr(
&mut self,
ptr: *const u8,
shared: Box<dyn rkyv::de::SharedPointer>,
) -> Result<(), Self::Error> {
self.inner
.add_shared_ptr(ptr, shared)
.map_err(VeilidRkyvError::Inner)
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#[derive(Debug)]
pub enum VeilidRkyvError<E> {
Inner(E),
StringError(String),
}
impl<E: Debug> From<String> for VeilidRkyvError<E> {
fn from(s: String) -> Self {
Self::StringError(s)
}
}
impl<E: Debug + fmt::Display> fmt::Display for VeilidRkyvError<E> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
VeilidRkyvError::Inner(e) => write!(f, "Inner: {}", e),
VeilidRkyvError::StringError(s) => write!(f, "StringError: {}", s),
}
}
}
impl<E: Debug + fmt::Display> std::error::Error for VeilidRkyvError<E> {}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
pub fn to_rkyv<T>(value: &T) -> VeilidAPIResult<Vec<u8>>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let mut serializer = DefaultVeilidRkyvSerializer::default();
serializer
.serialize_value(value)
.map_err(|e| VeilidAPIError::generic(format!("failed to serialize object: {}", e)))?;
Ok(serializer
.into_inner()
.into_serializer()
.into_inner()
.to_vec())
}
pub fn from_rkyv<T>(bytes: Vec<u8>) -> VeilidAPIResult<T>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
rkyv::check_archived_root::<T>(&bytes)
.map_err(|e| VeilidAPIError::generic(format!("checkbytes failed: {}", e)))?
.deserialize(&mut VeilidSharedDeserializeMap::default())
.map_err(|e| VeilidAPIError::generic(format!("failed to deserialize: {}", e)))
}

1
veilid-core/src/veilid_api/tests/mod.rs
View File

@ -1,6 +1,5 @@
mod fixtures;
pub mod test_serialize_json;
pub mod test_serialize_rkyv;
mod test_types;
mod test_types_dht;
mod test_types_dht_schema;

16
veilid-core/src/veilid_api/tests/test_serialize_rkyv.rs
View File

@ -1,16 +0,0 @@
use crate::*;
pub async fn test_simple_string() {
let plain = "basic string".to_string();
let serialized = b"basic string\x0c\x00\x00\x00\xf4\xff\xff\xff".to_vec();
let a = to_rkyv(&plain);
assert_eq!(a.unwrap(), serialized);
let b = from_rkyv::<String>(serialized);
assert_eq!(b.unwrap(), plain);
}
pub async fn test_all() {
test_simple_string().await;
}

21
veilid-core/src/veilid_api/types/aligned_u64.rs
View File

@ -2,29 +2,16 @@ use super::*;
/// Aligned u64
/// Required on 32-bit platforms for serialization because Rust aligns u64 on 4 byte boundaries
/// And zero-copy serialization with Rkyv requires 8-byte alignment
/// Some zero-copy serialization frameworks also want 8-byte alignment
/// Supports serializing to string for JSON as well, since JSON can't handle 64-bit numbers to Javascript
#[derive(
Clone,
Default,
PartialEq,
Eq,
PartialOrd,
Ord,
Copy,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Clone, Default, PartialEq, Eq, PartialOrd, Ord, Copy, Hash, Serialize, Deserialize, JsonSchema,
)]
#[repr(C, align(8))]
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
#[serde(transparent)]
pub struct AlignedU64(
#[serde(with = "json_as_string")]
#[serde(with = "as_human_string")]
#[schemars(with = "String")]
u64,
);

38
veilid-core/src/veilid_api/types/app_message_call.rs
View File

@ -1,27 +1,15 @@
use super::*;
/// Direct statement blob passed to hosting application for processing
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidAppMessage {
/// Some(sender) if the message was sent directly, None if received via a private/safety route
#[serde(with = "opt_json_as_string")]
#[serde(with = "as_human_opt_string")]
#[schemars(with = "Option<String>")]
pub sender: Option<TypedKey>,
/// The content of the message to deliver to the application
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
pub message: Vec<u8>,
}
@ -40,32 +28,20 @@ impl VeilidAppMessage {
}
/// Direct question blob passed to hosting application for processing to send an eventual AppReply
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidAppCall {
/// Some(sender) if the request was sent directly, None if received via a private/safety route
#[serde(with = "opt_json_as_string")]
#[serde(with = "as_human_opt_string")]
#[schemars(with = "Option<String>")]
sender: Option<TypedKey>,
/// The content of the request to deliver to the application
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
message: Vec<u8>,
/// The id to reply to
#[serde(with = "json_as_string")]
#[serde(with = "as_human_string")]
#[schemars(with = "String")]
call_id: OperationId,
}

16
veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs
View File

@ -1,21 +1,7 @@
use super::*;
/// DHT Record Descriptor
#[derive(
Debug,
Clone,
PartialOrd,
Ord,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct DHTRecordDescriptor {
/// DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ]
#[schemars(with = "String")]

16
veilid-core/src/veilid_api/types/dht/schema/dflt.rs
View File

@ -1,21 +1,7 @@
use super::*;
/// Default DHT Schema (DFLT)
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Ord,
PartialOrd,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, JsonSchema)]
pub struct DHTSchemaDFLT {
/// Owner subkey count
pub o_cnt: u16,

16
veilid-core/src/veilid_api/types/dht/schema/mod.rs
View File

@ -7,21 +7,7 @@ pub use dflt::*;
pub use smpl::*;
/// Enum over all the supported DHT Schemas
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Ord,
PartialOrd,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "kind")]
pub enum DHTSchema {
DFLT(DHTSchemaDFLT),

32
veilid-core/src/veilid_api/types/dht/schema/smpl.rs
View File

@ -1,21 +1,7 @@
use super::*;
/// Simple DHT Schema (SMPL) Member
#[derive(
Debug,
Clone,
PartialEq,
Eq,
PartialOrd,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema)]
pub struct DHTSchemaSMPLMember {
/// Member key
#[schemars(with = "String")]
@ -25,21 +11,7 @@ pub struct DHTSchemaSMPLMember {
}
/// Simple DHT Schema (SMPL)
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Ord,
PartialOrd,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, JsonSchema)]
pub struct DHTSchemaSMPL {
/// Owner subkey count
pub o_cnt: u16,

18
veilid-core/src/veilid_api/types/dht/value_data.rs
View File

@ -1,26 +1,12 @@
use super::*;
#[derive(
Clone,
Default,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize, JsonSchema)]
pub struct ValueData {
/// An increasing sequence number to time-order the DHT record changes
seq: ValueSeqNum,
/// The contents of a DHT Record
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,

17
veilid-core/src/veilid_api/types/dht/value_subkey_range_set.rs
View File

@ -2,24 +2,9 @@ use super::*;
use core::ops::{Deref, DerefMut};
use range_set_blaze::*;
#[derive(
Clone,
Default,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize, JsonSchema)]
#[serde(transparent)]
pub struct ValueSubkeyRangeSet {
#[with(RkyvRangeSetBlaze)]
#[serde(with = "serialize_range_set_blaze")]
#[schemars(with = "Vec<(u32,u32)>")]
data: RangeSetBlaze<ValueSubkey>,

16
veilid-core/src/veilid_api/types/fourcc.rs
View File

@ -2,22 +2,8 @@ use super::*;
/// FOURCC code
#[derive(
Copy,
Default,
Clone,
Hash,
PartialOrd,
Ord,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Copy, Default, Clone, Hash, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes, PartialOrd, Ord, PartialEq, Eq, Hash))]
#[serde(try_from = "String")]
#[serde(into = "String")]
pub struct FourCC(pub [u8; 4]);

74
veilid-core/src/veilid_api/types/safety.rs
View File

@ -2,26 +2,12 @@ use super::*;
// Ordering here matters, >= is used to check strength of sequencing requirement
#[derive(
Copy,
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum Sequencing {
NoPreference,
PreferOrdered,
EnsureOrdered,
NoPreference = 0,
PreferOrdered = 1,
EnsureOrdered = 2,
}
impl Default for Sequencing {
@ -32,25 +18,11 @@ impl Default for Sequencing {
// Ordering here matters, >= is used to check strength of stability requirement
#[derive(
Copy,
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum Stability {
LowLatency,
Reliable,
LowLatency = 0,
Reliable = 1,
}
impl Default for Stability {
@ -61,22 +33,8 @@ impl Default for Stability {
/// The choice of safety route to include in compiled routes
#[derive(
Copy,
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum SafetySelection {
/// Don't use a safety route, only specify the sequencing preference
Unsafe(Sequencing),
@ -101,22 +59,8 @@ impl Default for SafetySelection {
/// Options for safety routes (sender privacy)
#[derive(
Copy,
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
pub struct SafetySpec {
/// preferred safety route set id if it still exists
#[schemars(with = "Option<String>")]

75
veilid-core/src/veilid_api/types/stats.rs
View File

@ -1,39 +1,13 @@
use super::*;
#[derive(
Clone,
Debug,
Default,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct LatencyStats {
pub fastest: TimestampDuration, // fastest latency in the ROLLING_LATENCIES_SIZE last latencies
pub average: TimestampDuration, // average latency over the ROLLING_LATENCIES_SIZE last latencies
pub slowest: TimestampDuration, // slowest latency in the ROLLING_LATENCIES_SIZE last latencies
}
#[derive(
Clone,
Debug,
Default,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct TransferStats {
pub total: ByteCount, // total amount transferred ever
pub maximum: ByteCount, // maximum rate over the ROLLING_TRANSFERS_SIZE last amounts
@ -41,39 +15,13 @@ pub struct TransferStats {
pub minimum: ByteCount, // minimum rate over the ROLLING_TRANSFERS_SIZE last amounts
}
#[derive(
Clone,
Debug,
Default,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct TransferStatsDownUp {
pub down: TransferStats,
pub up: TransferStats,
}
#[derive(
Clone,
Debug,
Default,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct RPCStats {
pub messages_sent: u32, // number of rpcs that have been sent in the total_time range
pub messages_rcvd: u32, // number of rpcs that have been received in the total_time range
@ -85,20 +33,7 @@ pub struct RPCStats {
pub failed_to_send: u32, // number of messages that have failed to send since we last successfully sent one
}
#[derive(
Clone,
Debug,
Default,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct PeerStats {
pub time_added: Timestamp, // when the peer was added to the routing table
pub rpc_stats: RPCStats, // information about RPCs

86
veilid-core/src/veilid_api/types/tunnel.rs
View File

@ -7,64 +7,26 @@ pub type TunnelId = AlignedU64;
#[cfg(feature = "unstable-tunnels")]
#[derive(
Copy,
Clone,
Debug,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Copy, Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum TunnelMode {
Raw,
Turn,
Raw = 0,
Turn = 1,
}
#[cfg(feature = "unstable-tunnels")]
#[derive(
Copy,
Clone,
Debug,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Copy, Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum TunnelError {
BadId, // Tunnel ID was rejected
NoEndpoint, // Endpoint was unreachable
RejectedMode, // Endpoint couldn't provide mode
NoCapacity, // Endpoint is full
BadId = 0, // Tunnel ID was rejected
NoEndpoint = 1, // Endpoint was unreachable
RejectedMode = 2, // Endpoint couldn't provide mode
NoCapacity = 3, // Endpoint is full
}
#[cfg(feature = "unstable-tunnels")]
#[derive(
Clone,
Debug,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct TunnelEndpoint {
pub mode: TunnelMode,
pub description: String, // XXX: TODO
@ -81,20 +43,7 @@ impl Default for TunnelEndpoint {
}
#[cfg(feature = "unstable-tunnels")]
#[derive(
Clone,
Debug,
Default,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct FullTunnel {
pub id: TunnelId,
pub timeout: TimestampDuration,
@ -103,20 +52,7 @@ pub struct FullTunnel {
}
#[cfg(feature = "unstable-tunnels")]
#[derive(
Clone,
Debug,
Default,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct PartialTunnel {
pub id: TunnelId,
pub timeout: TimestampDuration,

37
veilid-core/src/veilid_api/types/veilid_log.rs
View File

@ -2,27 +2,14 @@ use super::*;
/// Log level for VeilidCore
#[derive(
Debug,
Clone,
PartialEq,
Eq,
PartialOrd,
Ord,
Copy,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Copy, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum VeilidLogLevel {
Error = 1,
Warn,
Info,
Debug,
Trace,
Warn = 2,
Info = 3,
Debug = 4,
Trace = 5,
}
impl VeilidLogLevel {
@ -92,19 +79,7 @@ impl fmt::Display for VeilidLogLevel {
}
}
/// A VeilidCore log message with optional backtrace
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidLog {
pub log_level: VeilidLogLevel,
pub message: String,

143
veilid-core/src/veilid_api/types/veilid_state.rs
View File

@ -1,29 +1,16 @@
use super::*;
/// Attachment abstraction for network 'signal strength'
#[derive(
Debug,
PartialEq,
Eq,
Clone,
Copy,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, JsonSchema)]
pub enum AttachmentState {
Detached,
Attaching,
AttachedWeak,
AttachedGood,
AttachedStrong,
FullyAttached,
OverAttached,
Detaching,
Detached = 0,
Attaching = 1,
AttachedWeak = 2,
AttachedGood = 3,
AttachedStrong = 4,
FullyAttached = 5,
OverAttached = 6,
Detaching = 7,
}
impl fmt::Display for AttachmentState {
@ -60,38 +47,14 @@ impl TryFrom<String> for AttachmentState {
}
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidStateAttachment {
pub state: AttachmentState,
pub public_internet_ready: bool,
pub local_network_ready: bool,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct PeerTableData {
#[schemars(with = "Vec<String>")]
pub node_ids: Vec<TypedKey>,
@ -99,19 +62,7 @@ pub struct PeerTableData {
pub peer_stats: PeerStats,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidStateNetwork {
pub started: bool,
pub bps_down: ByteCount,
@ -119,19 +70,7 @@ pub struct VeilidStateNetwork {
pub peers: Vec<PeerTableData>,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidRouteChange {
#[schemars(with = "Vec<String>")]
pub dead_routes: Vec<RouteId>,
@ -139,36 +78,12 @@ pub struct VeilidRouteChange {
pub dead_remote_routes: Vec<RouteId>,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidStateConfig {
pub config: VeilidConfigInner,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidValueChange {
#[schemars(with = "String")]
pub key: TypedKey,
@ -177,19 +92,7 @@ pub struct VeilidValueChange {
pub value: ValueData,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "kind")]
pub enum VeilidUpdate {
Log(VeilidLog),
@ -203,19 +106,7 @@ pub enum VeilidUpdate {
Shutdown,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidState {
pub attachment: VeilidStateAttachment,
pub network: VeilidStateNetwork,

269
veilid-core/src/veilid_config.rs
View File

@ -14,19 +14,7 @@ pub type ConfigCallback = Arc<dyn Fn(String) -> ConfigCallbackReturn + Send + Sy
/// url: 'https://localhost:5150'
/// ```
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigHTTPS {
pub enabled: bool,
pub listen_address: String,
@ -44,19 +32,7 @@ pub struct VeilidConfigHTTPS {
/// url: 'https://localhost:5150'
/// ```
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigHTTP {
pub enabled: bool,
pub listen_address: String,
@ -70,19 +46,7 @@ pub struct VeilidConfigHTTP {
///
/// To be implemented...
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigApplication {
pub https: VeilidConfigHTTPS,
pub http: VeilidConfigHTTP,
@ -98,19 +62,7 @@ pub struct VeilidConfigApplication {
/// public_address: ''
/// ```
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigUDP {
pub enabled: bool,
pub socket_pool_size: u32,
@ -128,19 +80,7 @@ pub struct VeilidConfigUDP {
/// listen_address: ':5150'
/// public_address: ''
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigTCP {
pub connect: bool,
pub listen: bool,
@ -160,19 +100,8 @@ pub struct VeilidConfigTCP {
/// path: 'ws'
/// url: 'ws://localhost:5150/ws'
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigWS {
pub connect: bool,
pub listen: bool,
@ -193,19 +122,8 @@ pub struct VeilidConfigWS {
/// path: 'ws'
/// url: ''
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigWSS {
pub connect: bool,
pub listen: bool,
@ -222,19 +140,8 @@ pub struct VeilidConfigWSS {
/// All protocols are available by default, and the Veilid node will
/// sort out which protocol is used for each peer connection.
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigProtocol {
pub udp: VeilidConfigUDP,
pub tcp: VeilidConfigTCP,
@ -250,19 +157,7 @@ pub struct VeilidConfigProtocol {
/// private_key_path: /path/to/private/key
/// connection_initial_timeout_ms: 2000
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigTLS {
pub certificate_path: String,
pub private_key_path: String,
@ -271,19 +166,7 @@ pub struct VeilidConfigTLS {
/// Configure the Distributed Hash Table (DHT)
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigDHT {
pub max_find_node_count: u32,
pub resolve_node_timeout_ms: u32,
@ -308,19 +191,7 @@ pub struct VeilidConfigDHT {
/// Configure RPC
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigRPC {
pub concurrency: u32,
pub queue_size: u32,
@ -333,19 +204,7 @@ pub struct VeilidConfigRPC {
/// Configure the network routing table
///
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigRoutingTable {
#[schemars(with = "Vec<String>")]
pub node_id: TypedKeyGroup,
@ -361,19 +220,7 @@ pub struct VeilidConfigRoutingTable {
// xxx pub enable_local_network: bool,
}
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigNetwork {
pub connection_initial_timeout_ms: u32,
pub connection_inactivity_timeout_ms: u32,
@ -396,55 +243,19 @@ pub struct VeilidConfigNetwork {
pub protocol: VeilidConfigProtocol,
}
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigTableStore {
pub directory: String,
pub delete: bool,
}
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigBlockStore {
pub directory: String,
pub delete: bool,
}
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigProtectedStore {
pub allow_insecure_fallback: bool,
pub always_use_insecure_storage: bool,
@ -454,36 +265,12 @@ pub struct VeilidConfigProtectedStore {
pub new_device_encryption_key_password: Option<String>,
}
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigCapabilities {
pub disable: Vec<FourCC>,
}
#[derive(
Clone,
Copy,
PartialEq,
Eq,
Debug,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, JsonSchema)]
pub enum VeilidConfigLogLevel {
Off,
Error,
@ -570,19 +357,7 @@ impl fmt::Display for VeilidConfigLogLevel {
}
}
#[derive(
Default,
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidConfigInner {
pub program_name: String,
pub namespace: String,

6
veilid-core/tests/web.rs
View File

@ -97,12 +97,6 @@ async fn wasm_test_serialize_json() {
test_serialize_json::test_all().await;
}
#[wasm_bindgen_test]
async fn wasm_test_serialize_rkyv() {
setup();
test_serialize_rkyv::test_all().await;
}
#[wasm_bindgen_test]
async fn wasm_test_serialize_routing_table() {
setup();

2
veilid-flutter/rust/src/dart_ffi.rs
View File

@ -119,7 +119,7 @@ pub struct VeilidFFIConfig {
#[derive(Debug, Deserialize, Serialize)]
pub struct VeilidFFIRouteBlob {
pub route_id: veilid_core::RouteId,
#[serde(with = "veilid_core::json_as_base64")]
#[serde(with = "veilid_core::as_human_base64")]
pub blob: Vec<u8>,
}

1
veilid-tools/Cargo.toml
View File

@ -46,7 +46,6 @@ flume = { version = "^0", features = ["async"] }
async-std = { version = "^1", features = ["unstable"], optional = true}
tokio = { version = "^1", features = ["full"], optional = true}
tokio-util = { version = "^0", features = ["compat"], optional = true}
maplit = "^1"
futures-util = { version = "^0", default-features = false, features = ["async-await", "sink", "std", "io"] }
chrono = "^0"