liz/veilid
liz/veilid
1
0
Fork 0
Code Pull Requests Packages Releases Activity

docs and tests work

Browse Source
This commit is contained in:
Christien Rioux 2023-08-29 15:15:47 -05:00
parent d3407998f5
commit e302b764d0
27 changed files with 156 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
veilid-cli/src/main.rs
View File

@ -169,6 +169,8 @@ fn main() -> Result<(), String> {
} else if #[cfg(feature="rt-tokio")] {
// Wait for ui and connection to complete
let _ = tokio::join!(ui_future, connection_future);
} else {
compile_error!("needs executor implementation")
}
}
});

3
veilid-cli/src/tools.rs
View File

@ -19,7 +19,8 @@ cfg_if! {
let local = tokio::task::LocalSet::new();
local.block_on(&rt, f)
}
} else {
compile_error!("needs executor implementation")
}
}

6
veilid-core/Cargo.toml
View File

@ -13,6 +13,7 @@ crate-type = ["cdylib", "staticlib", "rlib"]
# Common features
default = ["enable-crypto-vld0", "rt-tokio"]
default-wasm = ["enable-crypto-vld0"]
rt-async-std = [
"async-std",
"async-std-resolver",
@ -32,7 +33,6 @@ rt-tokio = [
"rtnetlink/tokio_socket",
"veilid-tools/rt-tokio",
]
rt-wasm-bindgen = ["veilid-tools/rt-wasm-bindgen", "async_executors/bindgen"]
# Crypto support features
enable-crypto-vld0 = []
@ -174,6 +174,10 @@ socket2 = { version = "0.5.3", features = ["all"] }
# Dependencies for WASM builds only
[target.'cfg(target_arch = "wasm32")'.dependencies]
veilid-tools = { path = "../veilid-tools", default-features = false, features = [
"rt-wasm-bindgen",
] }
# Tools
getrandom = { version = "0.2.4", features = ["js"] }

2
veilid-core/run_tests.sh
View File

@ -59,7 +59,7 @@ elif [[ "$1" == "android" ]]; then
popd >/dev/null
else
cargo test --features=rt-tokio
cargo test
cargo test --features=rt-async-std
fi
popd 2>/dev/null

2
veilid-core/run_windows_tests.bat
View File

@ -1,4 +1,4 @@
@echo off
cargo test --features=rt-tokio -- --nocapture
cargo test -- --nocapture
cargo test --features=rt-async-std -- --nocapture

1
veilid-core/src/crypto/mod.rs
View File

@ -7,6 +7,7 @@ mod types;
pub mod crypto_system;
#[cfg(feature = "enable-crypto-none")]
pub mod none;
#[doc(hidden)]
pub mod tests;
#[cfg(feature = "enable-crypto-vld0")]
pub mod vld0;

2
veilid-core/src/intf/native/network_interfaces/netlink.rs
View File

@ -17,6 +17,8 @@ cfg_if! {
use netlink_sys::{SmolSocket as RTNetLinkSocket};
} else if #[cfg(feature="rt-tokio")] {
use netlink_sys::{TokioSocket as RTNetLinkSocket};
} else {
compile_error!("needs executor implementation")
}
}
use std::convert::TryInto;

2
veilid-core/src/intf/native/system.rs
View File

@ -36,6 +36,8 @@ cfg_if! {
pub async fn resolver_from_system_conf() -> Result<AsyncResolver, ResolveError> {
AsyncResolver::tokio_from_system_conf()
}
} else {
compile_error!("needs executor implementation")
}
}

17
veilid-core/src/lib.rs
View File

@ -1,3 +1,14 @@
//! The Veilid Framework
//!
//! Core library used to create a Veilid node and operate veilid services as part of an application.
//!
//! `veilid-core` contains all of the core logic for Veilid and can be used in mobile applications as well as desktop
//! and in-browser WebAssembly apps.
//!
//! The public API is accessed by getting a [VeilidAPI] object via a call to [api_startup] or [api_startup_json].
//!
//! From there, a [RoutingContext] object can get you access to public and private routed operations.
//!
#![deny(clippy::all)]
#![deny(unused_must_use)]
#![recursion_limit = "256"]
@ -41,15 +52,20 @@ pub use self::veilid_config::*;
pub use self::veilid_layer_filter::*;
pub use veilid_tools as tools;
/// The on-the-wire serialization format for Veilid RPC
pub mod veilid_capnp {
include!(concat!(env!("OUT_DIR"), "/proto/veilid_capnp.rs"));
}
#[doc(hidden)]
pub mod tests;
/// Return the cargo package version of veilid-core in string format
pub fn veilid_version_string() -> String {
env!("CARGO_PKG_VERSION").to_owned()
}
/// Return the cargo package version of veilid-core in tuple format
pub fn veilid_version() -> (u32, u32, u32) {
(
u32::from_str(env!("CARGO_PKG_VERSION_MAJOR")).unwrap(),
@ -90,6 +106,7 @@ pub static DEFAULT_LOG_IGNORE_LIST: [&str; 23] = [
use cfg_if::*;
use enumset::*;
use eyre::{bail, eyre, Report as EyreReport, Result as EyreResult, WrapErr};
#[allow(unused_imports)]
use futures_util::stream::{FuturesOrdered, FuturesUnordered};
use parking_lot::*;
use schemars::{schema_for, JsonSchema};

1
veilid-core/src/network_manager/mod.rs
View File

@ -16,6 +16,7 @@ mod stats;
mod tasks;
mod types;
#[doc(hidden)]
pub mod tests;
////////////////////////////////////////////////////////////////////////////////////////

4
veilid-core/src/network_manager/native/network_tcp.rs
View File

@ -264,6 +264,8 @@ impl Network {
} else if #[cfg(feature="rt-tokio")] {
std_listener.set_nonblocking(true).expect("failed to set nonblocking");
let listener = TcpListener::from_std(std_listener).wrap_err("failed to create tokio tcp listener")?;
} else {
compile_error!("needs executor implementation")
}
}
@ -291,6 +293,8 @@ impl Network {
let incoming_stream = listener.incoming();
} else if #[cfg(feature="rt-tokio")] {
let incoming_stream = tokio_stream::wrappers::TcpListenerStream::new(listener);
} else {
compile_error!("needs executor implementation")
}
}

6
veilid-core/src/network_manager/native/network_udp.rs
View File

@ -136,6 +136,8 @@ impl Network {
} else if #[cfg(feature="rt-tokio")] {
std_udp_socket.set_nonblocking(true).expect("failed to set nonblocking");
let udp_socket = UdpSocket::from_std(std_udp_socket).wrap_err("failed to make outbound v4 tokio udpsocket")?;
} else {
compile_error!("needs executor implementation")
}
}
let socket_arc = Arc::new(udp_socket);
@ -160,6 +162,8 @@ impl Network {
} else if #[cfg(feature="rt-tokio")] {
std_udp_socket.set_nonblocking(true).expect("failed to set nonblocking");
let udp_socket = UdpSocket::from_std(std_udp_socket).wrap_err("failed to make outbound v6 tokio udpsocket")?;
} else {
compile_error!("needs executor implementation")
}
}
let socket_arc = Arc::new(udp_socket);
@ -190,6 +194,8 @@ impl Network {
} else if #[cfg(feature="rt-tokio")] {
std_udp_socket.set_nonblocking(true).expect("failed to set nonblocking");
let udp_socket = UdpSocket::from_std(std_udp_socket).wrap_err("failed to make inbound tokio udpsocket")?;
} else {
compile_error!("needs executor implementation")
}
}
let socket_arc = Arc::new(udp_socket);

4
veilid-core/src/network_manager/native/protocol/sockets.rs
View File

@ -8,6 +8,8 @@ cfg_if! {
} else if #[cfg(feature="rt-tokio")] {
pub use tokio::net::{TcpStream, TcpListener, UdpSocket};
pub use tokio_util::compat::*;
} else {
compile_error!("needs executor implementation")
}
}
@ -224,6 +226,8 @@ pub async fn nonblocking_connect(
Ok(TimeoutOr::value(TcpStream::from(async_stream.into_inner()?)))
} else if #[cfg(feature="rt-tokio")] {
Ok(TimeoutOr::value(TcpStream::from_std(async_stream.into_inner()?)?))
} else {
compile_error!("needs executor implementation")
}
}
}

4
veilid-core/src/network_manager/native/protocol/tcp.rs
View File

@ -36,7 +36,9 @@ impl RawTcpNetworkConnection {
// self.tcp_stream.get_mut()
// .shutdown()
// .await
// }
// } else {
// compile_error!("needs executor implementation")
// }
// }
// }

2
veilid-core/src/network_manager/native/protocol/ws.rs
View File

@ -14,6 +14,8 @@ cfg_if! {
pub type WebsocketNetworkConnectionWSS =
WebsocketNetworkConnection<async_tls::client::TlsStream<Compat<TcpStream>>>;
pub type WebsocketNetworkConnectionWS = WebsocketNetworkConnection<Compat<TcpStream>>;
} else {
compile_error!("needs executor implementation")
}
}

4
veilid-core/src/table_store/table_store.rs
View File

@ -164,7 +164,7 @@ impl TableStore {
self.flush().await;
}
pub fn maybe_unprotect_device_encryption_key(
pub(crate) fn maybe_unprotect_device_encryption_key(
&self,
dek_bytes: &[u8],
device_encryption_key_password: &str,
@ -218,7 +218,7 @@ impl TableStore {
))
}
pub fn maybe_protect_device_encryption_key(
pub(crate) fn maybe_protect_device_encryption_key(
&self,
dek: TypedSharedSecret,
device_encryption_key_password: &str,

26
veilid-core/src/tests/native/mod.rs
View File

@ -41,17 +41,21 @@ pub async fn run_all_tests() {
info!("Finished unit tests");
}
#[allow(dead_code)]
#[cfg(feature = "rt-tokio")]
pub fn block_on<F: Future<Output = T>, T>(f: F) -> T {
let rt = tokio::runtime::Runtime::new().unwrap();
rt.block_on(f)
}
#[cfg(feature = "rt-async-std")]
#[allow(dead_code)]
pub fn block_on<F: Future<Output = T>, T>(f: F) -> T {
async_std::task::block_on(f)
cfg_if::cfg_if! {
if #[cfg(feature = "rt-async-std")] {
#[allow(dead_code)]
pub fn block_on<F: Future<Output = T>, T>(f: F) -> T {
async_std::task::block_on(f)
}
} else if #[cfg(feature = "rt-tokio")] {
#[allow(dead_code)]
pub fn block_on<F: Future<Output = T>, T>(f: F) -> T {
let rt = tokio::runtime::Runtime::new().unwrap();
rt.block_on(f)
}
} else {
compile_error!("needs executor implementation")
}
}
///////////////////////////////////////////////////////////////////////////

73
veilid-core/src/veilid_api/api.rs
View File

@ -20,6 +20,19 @@ impl Drop for VeilidAPIInner {
}
}
/// The primary developer entrypoint into `veilid-core` functionality
///
/// From [VeilidAPI] one can access:
///
/// * [VeilidConfig] - The Veilid configuration specified by at startup time
/// * [Crypto] - The available set of cryptosystems provided by Veilid
/// * [TableStore] - The Veilid table-based encrypted persistent key-value store
/// * [ProtectedStore] - The Veilid abstract of the device's low-level 'protected secret storage'
/// * [VeilidState] - The current state of the Veilid node this API accesses
/// * [RoutingContext] - Communication methods between Veilid nodes and private routes
/// * Attach and detach from the network
/// * Create and import private routes
/// * Reply to `AppCall` RPCs
#[derive(Clone, Debug)]
pub struct VeilidAPI {
inner: Arc<Mutex<VeilidAPIInner>>,
@ -48,7 +61,8 @@ impl VeilidAPI {
}
////////////////////////////////////////////////////////////////
// Accessors
// Public Accessors
pub fn config(&self) -> VeilidAPIResult<VeilidConfig> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
@ -70,14 +84,6 @@ impl VeilidAPI {
}
Err(VeilidAPIError::not_initialized())
}
#[cfg(feature = "unstable-blockstore")]
pub fn block_store(&self) -> VeilidAPIResult<BlockStore> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
return Ok(context.block_store.clone());
}
Err(VeilidAPIError::not_initialized())
}
pub fn protected_store(&self) -> VeilidAPIResult<ProtectedStore> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
@ -85,41 +91,52 @@ impl VeilidAPI {
}
Err(VeilidAPIError::not_initialized())
}
pub fn attachment_manager(&self) -> VeilidAPIResult<AttachmentManager> {
////////////////////////////////////////////////////////////////
// Internal Accessors
pub(crate) fn attachment_manager(&self) -> VeilidAPIResult<AttachmentManager> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
return Ok(context.attachment_manager.clone());
}
Err(VeilidAPIError::not_initialized())
}
pub fn network_manager(&self) -> VeilidAPIResult<NetworkManager> {
pub(crate) fn network_manager(&self) -> VeilidAPIResult<NetworkManager> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
return Ok(context.attachment_manager.network_manager());
}
Err(VeilidAPIError::not_initialized())
}
pub fn rpc_processor(&self) -> VeilidAPIResult<RPCProcessor> {
pub(crate) fn rpc_processor(&self) -> VeilidAPIResult<RPCProcessor> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
return Ok(context.attachment_manager.network_manager().rpc_processor());
}
Err(VeilidAPIError::NotInitialized)
}
pub fn routing_table(&self) -> VeilidAPIResult<RoutingTable> {
pub(crate) fn routing_table(&self) -> VeilidAPIResult<RoutingTable> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
return Ok(context.attachment_manager.network_manager().routing_table());
}
Err(VeilidAPIError::NotInitialized)
}
pub fn storage_manager(&self) -> VeilidAPIResult<StorageManager> {
pub(crate) fn storage_manager(&self) -> VeilidAPIResult<StorageManager> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
return Ok(context.storage_manager.clone());
}
Err(VeilidAPIError::NotInitialized)
}
#[cfg(feature = "unstable-blockstore")]
pub(crate) fn block_store(&self) -> VeilidAPIResult<BlockStore> {
let inner = self.inner.lock();
if let Some(context) = &inner.context {
return Ok(context.block_store.clone());
}
Err(VeilidAPIError::not_initialized())
}
////////////////////////////////////////////////////////////////
// Attach/Detach
@ -162,10 +179,38 @@ impl VeilidAPI {
////////////////////////////////////////////////////////////////
// Routing Context
/// Get a new `RoutingContext` object to use to send messages over the Veilid network.
pub fn routing_context(&self) -> RoutingContext {
RoutingContext::new(self.clone())
}
/// Parse a string into a target object that can be used in a [RoutingContext]
///
/// Strings are in base64url format and can either be a remote route id or a node id.
/// Strings may have a [CryptoKind] [FourCC] prefix separated by a colon, such as:
/// `VLD0:XmnGyJrjMJBRC5ayJZRPXWTBspdX36-pbLb98H3UMeE` but if the prefix is left off
/// `XmnGyJrjMJBRC5ayJZRPXWTBspdX36-pbLb98H3UMeE` will be parsed with the 'best' cryptosystem
/// available (at the time of this writing this is `VLD0`)
pub async fn parse_as_target<S: AsRef<str>>(&self, s: S) -> VeilidAPIResult<Target> {
// Is this a route id?
if let Ok(rrid) = RouteId::from_str(s.as_ref()) {
let routing_table = self.routing_table()?;
let rss = routing_table.route_spec_store();
// Is this a valid remote route id? (can't target allocated routes)
if rss.is_route_id_remote(&rrid) {
return Ok(Target::PrivateRoute(rrid));
}
}
// Is this a node id?
if let Ok(nid) = TypedKey::from_str(s.as_ref()) {
return Ok(Target::NodeId(nid));
}
Err(VeilidAPIError::invalid_target())
}
////////////////////////////////////////////////////////////////
// Private route allocation

5
veilid-core/src/veilid_api/mod.rs
View File

@ -8,6 +8,7 @@ mod serialize_helpers;
mod types;
pub mod json_api;
#[doc(hidden)]
pub mod tests;
pub use api::*;
@ -18,19 +19,19 @@ pub use serialize_helpers::*;
pub use types::*;
pub use alloc::string::ToString;
pub use attachment_manager::AttachmentManager;
pub use core::str::FromStr;
pub use crypto::*;
#[cfg(feature = "unstable-blockstore")]
pub use intf::BlockStore;
pub use intf::ProtectedStore;
pub use network_manager::NetworkManager;
pub use routing_table::{NodeRef, NodeRefBase};
pub use table_store::{TableDB, TableDBTransaction, TableStore};
use crate::*;
use attachment_manager::AttachmentManager;
use core::fmt;
use core_context::{api_shutdown, VeilidCoreContext};
use network_manager::NetworkManager;
use routing_table::{Direction, RouteSpecStore, RoutingTable};
use rpc_processor::*;
use storage_manager::StorageManager;

1
veilid-core/src/veilid_api/tests/mod.rs
View File

@ -1,4 +1,5 @@
mod fixtures;
#[doc(hidden)]
pub mod test_serialize_json;
mod test_types;
mod test_types_dht;

41
veilid-flutter/rust/src/dart_ffi.rs
View File

@ -1,5 +1,5 @@
use veilid_core::tools::*;
use crate::dart_isolate_wrapper::*;
use crate::tools::*;
use allo_isolate::*;
use cfg_if::*;
use data_encoding::BASE64URL_NOPAD;
@ -10,13 +10,10 @@ use opentelemetry::*;
use opentelemetry_otlp::WithExportConfig;
use parking_lot::Mutex;
use serde::*;
use std::str::FromStr;
use std::collections::BTreeMap;
use std::os::raw::c_char;
use std::sync::Arc;
use tracing::*;
use tracing_subscriber::prelude::*;
use tracing_subscriber::*;
use veilid_core::Encodable as _;
// Globals
@ -58,29 +55,6 @@ define_string_destructor!(free_string);
type APIResult<T> = veilid_core::VeilidAPIResult<T>;
const APIRESULT_VOID: APIResult<()> = APIResult::Ok(());
// Parse target
async fn parse_target(s: String) -> APIResult<veilid_core::Target> {
// Is this a route id?
if let Ok(rrid) = veilid_core::RouteId::from_str(&s) {
let veilid_api = get_veilid_api().await?;
let routing_table = veilid_api.routing_table()?;
let rss = routing_table.route_spec_store();
// Is this a valid remote route id? (can't target allocated routes)
if rss.is_route_id_remote(&rrid) {
return Ok(veilid_core::Target::PrivateRoute(rrid));
}
}
// Is this a node id?
if let Ok(nid) = veilid_core::TypedKey::from_str(&s) {
return Ok(veilid_core::Target::NodeId(nid));
}
Err(veilid_core::VeilidAPIError::invalid_target())
}
/////////////////////////////////////////
// FFI-specific
@ -186,7 +160,7 @@ pub extern "C" fn initialize_veilid_core(platform_config: FfiStr) {
.expect("failed to deserialize plaform config json");
// Set up subscriber and layers
let subscriber = Registry::default();
let subscriber = tracing_subscriber::Registry::default();
let mut layers = Vec::new();
let mut filters = (*FILTERS).lock();
@ -194,7 +168,7 @@ pub extern "C" fn initialize_veilid_core(platform_config: FfiStr) {
if platform_config.logging.terminal.enabled {
let filter =
veilid_core::VeilidLayerFilter::new(platform_config.logging.terminal.level, None);
let layer = fmt::Layer::new()
let layer = tracing_subscriber::fmt::Layer::new()
.compact()
.with_writer(std::io::stdout)
.with_filter(filter.clone());
@ -217,6 +191,8 @@ pub extern "C" fn initialize_veilid_core(platform_config: FfiStr) {
.tonic()
.with_endpoint(format!("http://{}", grpc_endpoint));
let batch = opentelemetry::runtime::Tokio;
} else {
compile_error!("needs executor implementation")
}
}
@ -458,8 +434,8 @@ pub extern "C" fn routing_context_app_call(port: i64, id: u32, target: FfiStr, r
};
routing_context.clone()
};
let target = parse_target(target_string).await?;
let veilid_api = get_veilid_api().await?;
let target = veilid_api.parse_as_target(target_string).await?;
let answer = routing_context.app_call(target, request).await?;
let answer = data_encoding::BASE64URL_NOPAD.encode(&answer);
APIResult::Ok(answer)
@ -485,7 +461,8 @@ pub extern "C" fn routing_context_app_message(port: i64, id: u32, target: FfiStr
routing_context.clone()
};
let target = parse_target(target_string).await?;
let veilid_api = get_veilid_api().await?;
let target = veilid_api.parse_as_target(target_string).await?;
routing_context.app_message(target, message).await?;
APIRESULT_VOID
});

3
veilid-flutter/rust/src/tools.rs
View File

@ -28,6 +28,7 @@ cfg_if! {
lazy_static::lazy_static! {
static ref GLOBAL_RUNTIME: tokio::runtime::Runtime = tokio::runtime::Runtime::new().unwrap();
}
} else {
compile_error!("needs executor implementation")
}
}

6
veilid-server/src/client_api.rs
View File

@ -22,6 +22,8 @@ cfg_if! {
} else if #[cfg(feature="rt-tokio")] {
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncWriteExt;
} else {
compile_error!("needs executor implementation")
}
}
@ -114,7 +116,7 @@ impl ClientApi {
cfg_if! {
if #[cfg(feature="rt-async-std")] {
let mut incoming_stream = listener.incoming();
} else if #[cfg(feature="rt-tokio")] {
} else {
let mut incoming_stream = tokio_stream::wrappers::TcpListenerStream::new(listener);
}
}
@ -318,7 +320,7 @@ impl ClientApi {
use futures_util::AsyncReadExt;
let (reader, mut writer) = stream.split();
let reader = BufReader::new(reader);
} else if #[cfg(feature="rt-tokio")] {
} else {
let (reader, writer) = stream.into_split();
let reader = BufReader::new(reader);
}

3
veilid-server/src/tools.rs
View File

@ -48,6 +48,7 @@ cfg_if! {
let local = tokio::task::LocalSet::new();
local.block_on(&rt, f)
}
} else {
compile_error!("needs executor implementation")
}
}

2
veilid-server/src/veilid_logs.rs
View File

@ -80,6 +80,8 @@ impl VeilidLogs {
.tonic()
.with_endpoint(format!("http://{}", grpc_endpoint));
let batch = opentelemetry::runtime::Tokio;
} else {
compile_error!("needs executor implementation")
}
}

2
veilid-tools/src/lib.rs
View File

@ -127,7 +127,7 @@ cfg_if! {
#[doc(no_inline)]
pub use tokio::task::JoinHandle as LowLevelJoinHandle;
} else {
#[compile_error("must use an executor")]
compile_error!("needs executor implementation")
}
}
}

6
veilid-wasm/Cargo.toml
View File

@ -9,11 +9,11 @@ edition = "2021"
crate-type = ["cdylib", "rlib"]
[features]
default = [ "veilid-core/rt-wasm-bindgen", "veilid-core/default" ]
crypto-test = [ "veilid-core/rt-wasm-bindgen", "veilid-core/crypto-test"]
default = ["veilid-core/default"]
crypto-test = ["veilid-core/crypto-test"]
[dependencies]
veilid-core = { path = "../veilid-core", default-features = false }
veilid-core = { path = "../veilid-core", default-features = false }
tracing = { version = "^0", features = ["log", "attributes"] }
tracing-wasm = "^0"