refactor api to clean up internals

veilid-core/src/veilid_api/debug.rs

@@ -3,6 +3,7 @@
 
 use super::*;
 use data_encoding::BASE64URL_NOPAD;
+use network_manager::*;
 use routing_table::*;
 
 #[derive(Default, Debug)]

veilid-core/src/veilid_api/mod.rs

@@ -1,6 +1,5 @@
 #![allow(dead_code)]
 
-mod aligned_u64;
 mod api;
 mod debug;
 mod error;
@@ -8,7 +7,6 @@ mod routing_context;
 mod serialize_helpers;
 mod types;
 
-pub use aligned_u64::*;
 pub use api::*;
 pub use debug::*;
 pub use error::*;
@@ -31,7 +29,7 @@ use core::fmt;
 use core_context::{api_shutdown, VeilidCoreContext};
 use enumset::*;
 use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
-use routing_table::{RouteSpecStore, RoutingTable};
+use routing_table::{Direction, RouteSpecStore, RoutingTable};
 use rpc_processor::*;
 use serde::*;
 use storage_manager::StorageManager;

veilid-core/src/veilid_api/types/aligned_u64.rs

@@ -120,3 +120,17 @@ impl AlignedU64 {
         Self(self.0.saturating_sub(rhs.0))
     }
 }
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/// Microseconds since epoch
+pub type Timestamp = AlignedU64;
+pub fn get_aligned_timestamp() -> Timestamp {
+    get_timestamp().into()
+}
+/// Microseconds duration
+pub type TimestampDuration = AlignedU64;
+/// Request/Response matching id
+pub type OperationId = AlignedU64;
+/// Number of bytes
+pub type ByteCount = AlignedU64;

veilid-core/src/veilid_api/types/app_message_call.rs

@@ -0,0 +1,32 @@
+use super::*;
+
+/// Direct statement blob passed to hosting application for processing
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidAppMessage {
+    /// Some(sender) if the message was sent directly, None if received via a private/safety route
+    #[serde(with = "opt_json_as_string")]
+    pub sender: Option<PublicKey>,
+    /// The content of the message to deliver to the application
+    #[serde(with = "json_as_base64")]
+    pub message: Vec<u8>,
+}
+
+/// Direct question blob passed to hosting application for processing to send an eventual AppReply
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidAppCall {
+    /// Some(sender) if the request was sent directly, None if received via a private/safety route
+    #[serde(with = "opt_json_as_string")]
+    pub sender: Option<PublicKey>,
+    /// The content of the request to deliver to the application
+    #[serde(with = "json_as_base64")]
+    pub message: Vec<u8>,
+    /// The id to reply to
+    #[serde(with = "json_as_string")]
+    pub id: OperationId,
+}

veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs

@@ -0,0 +1,35 @@
+use super::*;
+
+/// DHT Record Descriptor
+#[derive(
+    Debug,
+    Clone,
+    PartialOrd,
+    Ord,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct DHTRecordDescriptor {
+    owner: PublicKey,
+    schema: DHTSchema,
+}
+
+impl DHTRecordDescriptor {
+    pub fn new(owner: PublicKey, schema: DHTSchema) -> Self {
+        Self { owner, schema }
+    }
+
+    pub fn owner(&self) -> PublicKey {
+        self.owner
+    }
+
+    pub fn schema(&self) -> DHTSchema {
+        self.schema
+    }
+}

veilid-core/src/veilid_api/types/dht/mod.rs

@@ -0,0 +1,16 @@
+mod dht_record_descriptor;
+mod schema;
+mod value_data;
+
+use super::*;
+
+pub use dht_record_descriptor::*;
+pub use schema::*;
+pub use value_data::*;
+
+/// Value subkey
+pub type ValueSubkey = u32;
+/// Value subkey range
+pub type ValueSubkeyRange = (u32, u32);
+/// Value sequence number
+pub type ValueSeqNum = u32;

veilid-core/src/veilid_api/types/dht/schema/dflt.rs

@@ -0,0 +1,61 @@
+use super::*;
+
+/// Default DHT Schema (DFLT)
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    Ord,
+    PartialOrd,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct DHTSchemaDFLT {
+    /// Owner subkey count
+    pub o_cnt: u16,
+}
+
+impl DHTSchemaDFLT {
+    pub const FCC: [u8; 4] = *b"DFLT";
+    pub const FIXED_SIZE: usize = 6;
+
+    /// Build the data representation of the schema
+    pub fn compile(&self) -> Vec<u8> {
+        let mut out = Vec::<u8>::with_capacity(Self::FIXED_SIZE);
+        // kind
+        out.extend_from_slice(&Self::FCC);
+        // o_cnt
+        out.extend_from_slice(&self.o_cnt.to_le_bytes());
+        out
+    }
+
+    /// Get the number of subkeys this schema allocates
+    pub fn subkey_count(&self) -> usize {
+        self.o_cnt as usize
+    }
+    /// Get the data size of this schema beyond the size of the structure itself
+    pub fn data_size(&self) -> usize {
+        0
+    }
+}
+
+impl TryFrom<&[u8]> for DHTSchemaDFLT {
+    type Error = VeilidAPIError;
+    fn try_from(b: &[u8]) -> Result<Self, Self::Error> {
+        if b.len() != Self::FIXED_SIZE {
+            apibail_generic!("invalid size");
+        }
+        if &b[0..4] != &Self::FCC {
+            apibail_generic!("wrong fourcc");
+        }
+
+        let o_cnt = u16::from_le_bytes(b[4..6].try_into().map_err(VeilidAPIError::internal)?);
+
+        Ok(Self { o_cnt })
+    }
+}

veilid-core/src/veilid_api/types/dht/schema/mod.rs

@@ -0,0 +1,84 @@
+mod dflt;
+mod smpl;
+
+use super::*;
+
+pub use dflt::*;
+pub use smpl::*;
+
+/// Enum over all the supported DHT Schemas
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    Ord,
+    PartialOrd,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+#[serde(tag = "kind")]
+pub enum DHTSchema {
+    DFLT(DHTSchemaDFLT),
+    SMPL(DHTSchemaSMPL),
+}
+
+impl DHTSchema {
+    pub fn dflt(o_cnt: u16) -> DHTSchema {
+        DHTSchema::DFLT(DHTSchemaDFLT { o_cnt })
+    }
+    pub fn smpl(o_cnt: u16, members: Vec<DHTSchemaSMPLMember>) -> DHTSchema {
+        DHTSchema::SMPL(DHTSchemaSMPL { o_cnt, members })
+    }
+
+    /// Build the data representation of the schema
+    pub fn compile(&self) -> Vec<u8> {
+        match self {
+            DHTSchema::DFLT(d) => d.compile(),
+            DHTSchema::SMPL(s) => s.compile(),
+        }
+    }
+
+    /// Get the number of subkeys this schema allocates
+    pub fn subkey_count(&self) -> usize {
+        match self {
+            DHTSchema::DFLT(d) => d.subkey_count(),
+            DHTSchema::SMPL(s) => s.subkey_count(),
+        }
+    }
+
+    /// Get the data size of this schema beyond the size of the structure itself
+    pub fn data_size(&self) -> usize {
+        match self {
+            DHTSchema::DFLT(d) => d.data_size(),
+            DHTSchema::SMPL(s) => s.data_size(),
+        }
+    }
+}
+
+impl Default for DHTSchema {
+    fn default() -> Self {
+        Self::dflt(1)
+    }
+}
+
+impl TryFrom<&[u8]> for DHTSchema {
+    type Error = VeilidAPIError;
+    fn try_from(b: &[u8]) -> Result<Self, Self::Error> {
+        if b.len() < 4 {
+            apibail_generic!("invalid size");
+        }
+        let fcc: [u8; 4] = b[0..4].try_into().unwrap();
+        match fcc {
+            DHTSchemaDFLT::FCC => Ok(DHTSchema::DFLT(DHTSchemaDFLT::try_from(b)?)),
+            DHTSchemaSMPL::FCC => Ok(DHTSchema::SMPL(DHTSchemaSMPL::try_from(b)?)),
+            _ => {
+                apibail_generic!("unknown fourcc");
+            }
+        }
+    }
+}

veilid-core/src/veilid_api/types/dht/schema/smpl.rs

@@ -0,0 +1,114 @@
+use super::*;
+
+/// Simple DHT Schema (SMPL) Member
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct DHTSchemaSMPLMember {
+    /// Member key
+    pub m_key: PublicKey,
+    /// Member subkey countanyway,
+    pub m_cnt: u16,
+}
+
+/// Simple DHT Schema (SMPL)
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    Ord,
+    PartialOrd,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct DHTSchemaSMPL {
+    /// Owner subkey count
+    pub o_cnt: u16,
+    /// Members
+    pub members: Vec<DHTSchemaSMPLMember>,
+}
+
+impl DHTSchemaSMPL {
+    pub const FCC: [u8; 4] = *b"SMPL";
+    pub const FIXED_SIZE: usize = 6;
+
+    /// Build the data representation of the schema
+    pub fn compile(&self) -> Vec<u8> {
+        let mut out = Vec::<u8>::with_capacity(
+            Self::FIXED_SIZE + (self.members.len() * (PUBLIC_KEY_LENGTH + 2)),
+        );
+        // kind
+        out.extend_from_slice(&Self::FCC);
+        // o_cnt
+        out.extend_from_slice(&self.o_cnt.to_le_bytes());
+        // members
+        for m in self.members {
+            // m_key
+            out.extend_from_slice(&m.m_key.bytes);
+            // m_cnt
+            out.extend_from_slice(&m.m_cnt.to_le_bytes());
+        }
+        out
+    }
+
+    /// Get the number of subkeys this schema allocates
+    pub fn subkey_count(&self) -> usize {
+        self.members
+            .iter()
+            .fold(self.o_cnt as usize, |acc, x| acc + (x.m_cnt as usize))
+    }
+
+    /// Get the data size of this schema beyond the size of the structure itself
+    pub fn data_size(&self) -> usize {
+        self.members.len() * mem::size_of::<DHTSchemaSMPLMember>()
+    }
+}
+
+impl TryFrom<&[u8]> for DHTSchemaSMPL {
+    type Error = VeilidAPIError;
+    fn try_from(b: &[u8]) -> Result<Self, Self::Error> {
+        if b.len() != Self::FIXED_SIZE {
+            apibail_generic!("invalid size");
+        }
+        if &b[0..4] != &Self::FCC {
+            apibail_generic!("wrong fourcc");
+        }
+        if (b.len() - Self::FIXED_SIZE) % (PUBLIC_KEY_LENGTH + 2) != 0 {
+            apibail_generic!("invalid member length");
+        }
+
+        let o_cnt = u16::from_le_bytes(b[4..6].try_into().map_err(VeilidAPIError::internal)?);
+
+        let members_len = (b.len() - Self::FIXED_SIZE) / (PUBLIC_KEY_LENGTH + 2);
+        let mut members: Vec<DHTSchemaSMPLMember> = Vec::with_capacity(members_len);
+        for n in 0..members_len {
+            let mstart = Self::FIXED_SIZE + n * (PUBLIC_KEY_LENGTH + 2);
+            let m_key = PublicKey::try_from(&b[mstart..mstart + PUBLIC_KEY_LENGTH])
+                .map_err(VeilidAPIError::internal)?;
+            let m_cnt = u16::from_le_bytes(
+                b[mstart + PUBLIC_KEY_LENGTH..mstart + PUBLIC_KEY_LENGTH + 2]
+                    .try_into()
+                    .map_err(VeilidAPIError::internal)?,
+            );
+            members.push(DHTSchemaSMPLMember { m_key, m_cnt });
+        }
+
+        Ok(Self { o_cnt, members })
+    }
+}

veilid-core/src/veilid_api/types/dht/value_data.rs

@@ -0,0 +1,64 @@
+use super::*;
+
+#[derive(
+    Clone,
+    Debug,
+    Default,
+    PartialOrd,
+    PartialEq,
+    Eq,
+    Ord,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct ValueData {
+    seq: ValueSeqNum,
+    data: Vec<u8>,
+    writer: PublicKey,
+}
+impl ValueData {
+    pub const MAX_LEN: usize = 32768;
+
+    pub fn new(data: Vec<u8>, writer: PublicKey) -> Self {
+        assert!(data.len() <= Self::MAX_LEN);
+        Self {
+            seq: 0,
+            data,
+            writer,
+        }
+    }
+    pub fn new_with_seq(seq: ValueSeqNum, data: Vec<u8>, writer: PublicKey) -> Self {
+        assert!(data.len() <= Self::MAX_LEN);
+        Self { seq, data, writer }
+    }
+
+    pub fn seq(&self) -> ValueSeqNum {
+        self.seq
+    }
+
+    pub fn writer(&self) -> &PublicKey {
+        &self.writer
+    }
+
+    pub fn data(&self) -> &[u8] {
+        &self.data
+    }
+
+    pub fn with_data_mut<F, R>(&mut self, f: F) -> R
+    where
+        F: FnOnce(&mut Vec<u8>) -> R,
+    {
+        let out = f(&mut self.data);
+        assert!(self.data.len() <= Self::MAX_LEN);
+        self.seq += 1;
+        out
+    }
+
+    pub fn total_size(&self) -> usize {
+        mem::size_of::<Self>() + self.data.len()
+    }
+}

veilid-core/src/veilid_api/types/fourcc.rs

@@ -0,0 +1,52 @@
+use super::*;
+
+/// FOURCC code
+#[derive(
+    Copy,
+    Default,
+    Clone,
+    Hash,
+    PartialOrd,
+    Ord,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes, PartialOrd, Ord, PartialEq, Eq, Hash))]
+pub struct FourCC(pub [u8; 4]);
+
+impl From<[u8; 4]> for FourCC {
+    fn from(b: [u8; 4]) -> Self {
+        Self(b)
+    }
+}
+impl TryFrom<&[u8]> for FourCC {
+    type Error = VeilidAPIError;
+    fn try_from(b: &[u8]) -> Result<Self, Self::Error> {
+        Ok(Self(b.try_into().map_err(VeilidAPIError::generic)?))
+    }
+}
+
+impl fmt::Display for FourCC {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        write!(f, "{}", String::from_utf8_lossy(&self.0))
+    }
+}
+impl fmt::Debug for FourCC {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        write!(f, "{}", String::from_utf8_lossy(&self.0))
+    }
+}
+
+impl FromStr for FourCC {
+    type Err = VeilidAPIError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(
+            s.as_bytes().try_into().map_err(VeilidAPIError::generic)?,
+        ))
+    }
+}

veilid-core/src/veilid_api/types/mod.rs

@@ -0,0 +1,21 @@
+mod aligned_u64;
+mod app_message_call;
+mod dht;
+mod fourcc;
+mod safety;
+mod stats;
+mod tunnel;
+mod veilid_log;
+mod veilid_state;
+
+use super::*;
+
+pub use aligned_u64::*;
+pub use app_message_call::*;
+pub use dht::*;
+pub use fourcc::*;
+pub use safety::*;
+pub use stats::*;
+pub use tunnel::*;
+pub use veilid_log::*;
+pub use veilid_state::*;

veilid-core/src/veilid_api/types/safety.rs

@@ -0,0 +1,125 @@
+use super::*;
+
+// Ordering here matters, >= is used to check strength of sequencing requirement
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+pub enum Sequencing {
+    NoPreference,
+    PreferOrdered,
+    EnsureOrdered,
+}
+
+impl Default for Sequencing {
+    fn default() -> Self {
+        Self::NoPreference
+    }
+}
+
+// Ordering here matters, >= is used to check strength of stability requirement
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+pub enum Stability {
+    LowLatency,
+    Reliable,
+}
+
+impl Default for Stability {
+    fn default() -> Self {
+        Self::LowLatency
+    }
+}
+
+/// The choice of safety route to include in compiled routes
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+pub enum SafetySelection {
+    /// Don't use a safety route, only specify the sequencing preference
+    Unsafe(Sequencing),
+    /// Use a safety route and parameters specified by a SafetySpec
+    Safe(SafetySpec),
+}
+
+impl SafetySelection {
+    pub fn get_sequencing(&self) -> Sequencing {
+        match self {
+            SafetySelection::Unsafe(seq) => *seq,
+            SafetySelection::Safe(ss) => ss.sequencing,
+        }
+    }
+}
+
+impl Default for SafetySelection {
+    fn default() -> Self {
+        Self::Unsafe(Sequencing::NoPreference)
+    }
+}
+
+/// Options for safety routes (sender privacy)
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct SafetySpec {
+    /// preferred safety route set id if it still exists
+    pub preferred_route: Option<RouteId>,
+    /// must be greater than 0
+    pub hop_count: usize,
+    /// prefer reliability over speed
+    pub stability: Stability,
+    /// prefer connection-oriented sequenced protocols
+    pub sequencing: Sequencing,
+}

veilid-core/src/veilid_api/types/stats.rs

@@ -0,0 +1,113 @@
+use super::*;
+
+#[derive(
+    Clone,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct LatencyStats {
+    #[serde(with = "json_as_string")]
+    pub fastest: TimestampDuration, // fastest latency in the ROLLING_LATENCIES_SIZE last latencies
+    #[serde(with = "json_as_string")]
+    pub average: TimestampDuration, // average latency over the ROLLING_LATENCIES_SIZE last latencies
+    #[serde(with = "json_as_string")]
+    pub slowest: TimestampDuration, // slowest latency in the ROLLING_LATENCIES_SIZE last latencies
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct TransferStats {
+    #[serde(with = "json_as_string")]
+    pub total: ByteCount, // total amount transferred ever
+    #[serde(with = "json_as_string")]
+    pub maximum: ByteCount, // maximum rate over the ROLLING_TRANSFERS_SIZE last amounts
+    #[serde(with = "json_as_string")]
+    pub average: ByteCount, // average rate over the ROLLING_TRANSFERS_SIZE last amounts
+    #[serde(with = "json_as_string")]
+    pub minimum: ByteCount, // minimum rate over the ROLLING_TRANSFERS_SIZE last amounts
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct TransferStatsDownUp {
+    pub down: TransferStats,
+    pub up: TransferStats,
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct RPCStats {
+    pub messages_sent: u32, // number of rpcs that have been sent in the total_time range
+    pub messages_rcvd: u32, // number of rpcs that have been received in the total_time range
+    pub questions_in_flight: u32, // number of questions issued that have yet to be answered
+    #[serde(with = "opt_json_as_string")]
+    pub last_question_ts: Option<Timestamp>, // when the peer was last questioned (either successfully or not) and we wanted an answer
+    #[serde(with = "opt_json_as_string")]
+    pub last_seen_ts: Option<Timestamp>, // when the peer was last seen for any reason, including when we first attempted to reach out to it
+    #[serde(with = "opt_json_as_string")]
+    pub first_consecutive_seen_ts: Option<Timestamp>, // the timestamp of the first consecutive proof-of-life for this node (an answer or received question)
+    pub recent_lost_answers: u32, // number of answers that have been lost since we lost reliability
+    pub failed_to_send: u32, // number of messages that have failed to send since we last successfully sent one
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct PeerStats {
+    #[serde(with = "json_as_string")]
+    pub time_added: Timestamp, // when the peer was added to the routing table
+    pub rpc_stats: RPCStats,           // information about RPCs
+    pub latency: Option<LatencyStats>, // latencies for communications with the peer
+    pub transfer: TransferStatsDownUp, // Stats for communications with the peer
+}

veilid-core/src/veilid_api/types/tunnel.rs

@@ -0,0 +1,83 @@
+use super::*;
+
+/// Tunnel identifier
+pub type TunnelId = AlignedU64;
+
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    PartialOrd,
+    PartialEq,
+    Eq,
+    Ord,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+pub enum TunnelMode {
+    Raw,
+    Turn,
+}
+
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    PartialOrd,
+    PartialEq,
+    Eq,
+    Ord,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+pub enum TunnelError {
+    BadId,        // Tunnel ID was rejected
+    NoEndpoint,   // Endpoint was unreachable
+    RejectedMode, // Endpoint couldn't provide mode
+    NoCapacity,   // Endpoint is full
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct TunnelEndpoint {
+    pub mode: TunnelMode,
+    pub description: String, // XXX: TODO
+}
+
+impl Default for TunnelEndpoint {
+    fn default() -> Self {
+        Self {
+            mode: TunnelMode::Raw,
+            description: "".to_string(),
+        }
+    }
+}
+
+#[derive(
+    Clone, Debug, Default, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct FullTunnel {
+    pub id: TunnelId,
+    pub timeout: TimestampDuration,
+    pub local: TunnelEndpoint,
+    pub remote: TunnelEndpoint,
+}
+
+#[derive(
+    Clone, Debug, Default, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct PartialTunnel {
+    pub id: TunnelId,
+    pub timeout: TimestampDuration,
+    pub local: TunnelEndpoint,
+}

veilid-core/src/veilid_api/types/veilid_log.rs

@@ -0,0 +1,88 @@
+use super::*;
+
+/// Log level for VeilidCore
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Copy,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+pub enum VeilidLogLevel {
+    Error = 1,
+    Warn,
+    Info,
+    Debug,
+    Trace,
+}
+
+impl VeilidLogLevel {
+    pub fn from_tracing_level(level: tracing::Level) -> VeilidLogLevel {
+        match level {
+            tracing::Level::ERROR => VeilidLogLevel::Error,
+            tracing::Level::WARN => VeilidLogLevel::Warn,
+            tracing::Level::INFO => VeilidLogLevel::Info,
+            tracing::Level::DEBUG => VeilidLogLevel::Debug,
+            tracing::Level::TRACE => VeilidLogLevel::Trace,
+        }
+    }
+    pub fn from_log_level(level: log::Level) -> VeilidLogLevel {
+        match level {
+            log::Level::Error => VeilidLogLevel::Error,
+            log::Level::Warn => VeilidLogLevel::Warn,
+            log::Level::Info => VeilidLogLevel::Info,
+            log::Level::Debug => VeilidLogLevel::Debug,
+            log::Level::Trace => VeilidLogLevel::Trace,
+        }
+    }
+    pub fn to_tracing_level(&self) -> tracing::Level {
+        match self {
+            Self::Error => tracing::Level::ERROR,
+            Self::Warn => tracing::Level::WARN,
+            Self::Info => tracing::Level::INFO,
+            Self::Debug => tracing::Level::DEBUG,
+            Self::Trace => tracing::Level::TRACE,
+        }
+    }
+    pub fn to_log_level(&self) -> log::Level {
+        match self {
+            Self::Error => log::Level::Error,
+            Self::Warn => log::Level::Warn,
+            Self::Info => log::Level::Info,
+            Self::Debug => log::Level::Debug,
+            Self::Trace => log::Level::Trace,
+        }
+    }
+}
+
+impl fmt::Display for VeilidLogLevel {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        let text = match self {
+            Self::Error => "ERROR",
+            Self::Warn => "WARN",
+            Self::Info => "INFO",
+            Self::Debug => "DEBUG",
+            Self::Trace => "TRACE",
+        };
+        write!(f, "{}", text)
+    }
+}
+
+/// A VeilidCore log message with optional backtrace
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidLog {
+    pub log_level: VeilidLogLevel,
+    pub message: String,
+    pub backtrace: Option<String>,
+}

veilid-core/src/veilid_api/types/veilid_state.rs

@@ -0,0 +1,144 @@
+use super::*;
+
+/// Attachment abstraction for network 'signal strength'
+#[derive(
+    Debug,
+    PartialEq,
+    Eq,
+    Clone,
+    Copy,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+pub enum AttachmentState {
+    Detached,
+    Attaching,
+    AttachedWeak,
+    AttachedGood,
+    AttachedStrong,
+    FullyAttached,
+    OverAttached,
+    Detaching,
+}
+
+impl fmt::Display for AttachmentState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        let out = match self {
+            AttachmentState::Attaching => "attaching".to_owned(),
+            AttachmentState::AttachedWeak => "attached_weak".to_owned(),
+            AttachmentState::AttachedGood => "attached_good".to_owned(),
+            AttachmentState::AttachedStrong => "attached_strong".to_owned(),
+            AttachmentState::FullyAttached => "fully_attached".to_owned(),
+            AttachmentState::OverAttached => "over_attached".to_owned(),
+            AttachmentState::Detaching => "detaching".to_owned(),
+            AttachmentState::Detached => "detached".to_owned(),
+        };
+        write!(f, "{}", out)
+    }
+}
+
+impl TryFrom<String> for AttachmentState {
+    type Error = ();
+
+    fn try_from(s: String) -> Result<Self, Self::Error> {
+        Ok(match s.as_str() {
+            "attaching" => AttachmentState::Attaching,
+            "attached_weak" => AttachmentState::AttachedWeak,
+            "attached_good" => AttachmentState::AttachedGood,
+            "attached_strong" => AttachmentState::AttachedStrong,
+            "fully_attached" => AttachmentState::FullyAttached,
+            "over_attached" => AttachmentState::OverAttached,
+            "detaching" => AttachmentState::Detaching,
+            "detached" => AttachmentState::Detached,
+            _ => return Err(()),
+        })
+    }
+}
+
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidStateAttachment {
+    pub state: AttachmentState,
+    pub public_internet_ready: bool,
+    pub local_network_ready: bool,
+}
+
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct PeerTableData {
+    pub node_ids: Vec<String>,
+    pub peer_address: String,
+    pub peer_stats: PeerStats,
+}
+
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidStateNetwork {
+    pub started: bool,
+    #[serde(with = "json_as_string")]
+    pub bps_down: ByteCount,
+    #[serde(with = "json_as_string")]
+    pub bps_up: ByteCount,
+    pub peers: Vec<PeerTableData>,
+}
+
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidStateRoute {
+    pub dead_routes: Vec<RouteId>,
+    pub dead_remote_routes: Vec<RouteId>,
+}
+
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidStateConfig {
+    pub config: VeilidConfigInner,
+}
+
+#[derive(
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidValueChange {
+    key: TypedKey,
+    subkeys: Vec<ValueSubkey>,
+    count: u32,
+    value: ValueData,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
+#[archive_attr(repr(u8), derive(CheckBytes))]
+#[serde(tag = "kind")]
+pub enum VeilidUpdate {
+    Log(VeilidLog),
+    AppMessage(VeilidAppMessage),
+    AppCall(VeilidAppCall),
+    Attachment(VeilidStateAttachment),
+    Network(VeilidStateNetwork),
+    Config(VeilidStateConfig),
+    Route(VeilidStateRoute),
+    ValueChange(VeilidValueChange),
+    Shutdown,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct VeilidState {
+    pub attachment: VeilidStateAttachment,
+    pub network: VeilidStateNetwork,
+    pub config: VeilidStateConfig,
+}