liz/veilid
1
0
Fork 0
Code Pull Requests Packages Releases Activity

packaging and bootstrap work

Browse Source
This commit is contained in:
John Smith
2022-05-17 16:55:53 -04:00
parent ef1f5d7b52
commit 1326424eae
17 changed files with 373 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
Cargo.lock generated
View File

@@ -298,6 +298,21 @@ dependencies = [
"wasm-bindgen-futures",
]
[[package]]
name = "async-std-resolver"
version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f2f8a4a203be3325981310ab243a28e6e4ea55b6519bffce05d41ab60e09ad8"
dependencies = [
"async-std",
"async-trait",
"futures-io",
"futures-util",
"pin-utils",
"socket2",
"trust-dns-resolver",
]
[[package]]
name = "async-task"
version = "4.2.0"
@@ -1319,6 +1334,18 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "enum-as-inner"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "enum-map"
version = "1.1.1"
@@ -1792,6 +1819,12 @@ dependencies = [
"serde 1.0.136",
]
[[package]]
name = "heck"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
[[package]]
name = "hermit-abi"
version = "0.1.19"
@@ -1836,6 +1869,17 @@ dependencies = [
"digest 0.10.3",
]
[[package]]
name = "hostname"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867"
dependencies = [
"libc",
"match_cfg",
"winapi",
]
[[package]]
name = "http"
version = "0.2.6"
@@ -1949,6 +1993,24 @@ dependencies = [
"web-sys",
]
[[package]]
name = "ipconfig"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98"
dependencies = [
"socket2",
"widestring 0.5.1",
"winapi",
"winreg",
]
[[package]]
name = "ipnet"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
[[package]]
name = "itertools"
version = "0.10.3"
@@ -2248,12 +2310,27 @@ dependencies = [
"hashbrown 0.11.2",
]
[[package]]
name = "lru-cache"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
dependencies = [
"linked-hash-map 0.5.4",
]
[[package]]
name = "maplit"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
[[package]]
name = "match_cfg"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
[[package]]
name = "matches"
version = "0.1.9"
@@ -3094,6 +3171,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "quick-error"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quote"
version = "1.0.15"
@@ -3262,6 +3345,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "resolv-conf"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00"
dependencies = [
"hostname",
"quick-error",
]
[[package]]
name = "ring"
version = "0.16.20"
@@ -4062,6 +4155,49 @@ dependencies = [
"serde 1.0.136",
]
[[package]]
name = "trust-dns-proto"
version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d"
dependencies = [
"async-trait",
"cfg-if 1.0.0",
"data-encoding",
"enum-as-inner",
"futures-channel",
"futures-io",
"futures-util",
"idna",
"ipnet",
"lazy_static",
"log",
"rand 0.8.5",
"smallvec",
"thiserror",
"tinyvec",
"url",
]
[[package]]
name = "trust-dns-resolver"
version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558"
dependencies = [
"cfg-if 1.0.0",
"futures-util",
"ipconfig",
"lazy_static",
"log",
"lru-cache",
"parking_lot 0.12.0",
"resolv-conf",
"smallvec",
"thiserror",
"trust-dns-proto",
]
[[package]]
name = "tungstenite"
version = "0.11.1"
@@ -4260,6 +4396,7 @@ dependencies = [
"anyhow",
"async-lock",
"async-std",
"async-std-resolver",
"async-tls",
"async-tungstenite 0.17.1",
"async_executors",
@@ -4631,6 +4768,12 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c"
[[package]]
name = "widestring"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983"
[[package]]
name = "winapi"
version = "0.3.9"
@@ -4670,7 +4813,7 @@ checksum = "0c643e10139d127d30d6d753398c8a6f0a43532e8370f6c9d29ebbff29b984ab"
dependencies = [
"bitflags",
"err-derive",
"widestring",
"widestring 0.4.3",
"winapi",
]
@@ -4717,6 +4860,15 @@ version = "0.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
[[package]]
name = "winreg"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
dependencies = [
"winapi",
]
[[package]]
name = "ws_stream_wasm"
version = "0.7.3"

1
package/debian/veilid-cli/DEBIAN/control
View File

@@ -7,4 +7,3 @@ Depends: libc6 (>= 2.23)
Maintainer: jsmith@veilid.org
Description: Veilid Server Command Line Interface
The Veilid peer-to-peer network server command line interface

1
veilid-core/Cargo.toml
View File

@@ -51,6 +51,7 @@ digest = "0.9.0"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
async-std = { version = "^1", features = ["unstable"] }
async-tungstenite = { version = "^0", features = ["async-std-runtime", "async-tls"] }
async-std-resolver = { version = "^0" }
maplit = "^1"
config = { version = "^0", features = ["yaml"] }
keyring-manager = { path = "../external/keyring-manager" }

4
veilid-core/src/intf/native/network/mod.rs
View File

@@ -482,6 +482,10 @@ impl Network {
self.inner.lock().network_needs_restart
}
pub fn is_started(&self) -> bool {
self.inner.lock().network_started
}
pub fn restart_network(&self) {
self.inner.lock().network_needs_restart = true;
}

50
veilid-core/src/intf/native/system.rs
View File

@@ -3,9 +3,14 @@
use crate::xx::*;
pub use async_executors::JoinHandle;
use async_executors::{AsyncStd, LocalSpawnHandleExt, SpawnHandleExt};
use async_std_resolver::{config, resolver, AsyncStdResolver};
use rand::prelude::*;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
lazy_static::lazy_static! {
static ref RESOLVER: Arc<AsyncMutex<Option<AsyncStdResolver>>> = Arc::new(AsyncMutex::new(None));
}
pub fn get_timestamp() -> u64 {
match SystemTime::now().duration_since(UNIX_EPOCH) {
Ok(n) => n.as_micros() as u64,
@@ -111,3 +116,48 @@ where
}));
}
*/
async fn get_resolver() -> Result<AsyncStdResolver, String> {
let mut resolver_lock = RESOLVER.lock().await;
if let Some(r) = &*resolver_lock {
Ok(r.clone())
} else {
let resolver = resolver(
config::ResolverConfig::default(),
config::ResolverOpts::default(),
)
.await
.expect("failed to connect resolver");
*resolver_lock = Some(resolver.clone());
Ok(resolver)
}
}
pub async fn txt_lookup<S: AsRef<str>>(host: S) -> Result<Vec<String>, String> {
let resolver = get_resolver().await?;
let txt_result = resolver
.txt_lookup(host.as_ref())
.await
.map_err(|e| e.to_string())?;
let mut out = Vec::new();
for x in txt_result.iter() {
for s in x.txt_data() {
out.push(String::from_utf8(s.to_vec()).map_err(|e| e.to_string())?);
}
}
Ok(out)
}
pub async fn ptr_lookup(ip_addr: IpAddr) -> Result<String, String> {
let resolver = get_resolver().await?;
let ptr_result = resolver
.reverse_lookup(ip_addr)
.await
.map_err(|e| e.to_string())?;
if let Some(r) = ptr_result.iter().next() {
Ok(r.to_string().trim_end_matches('.').to_string())
} else {
Err("PTR lookup returned an empty string".to_owned())
}
}

9
veilid-core/src/intf/native/utils/ios_test_setup.rs
View File

@@ -24,12 +24,9 @@ fn veilid_core_setup_internal<'a>(
let mut logs: Vec<Box<dyn SharedLogger>> = Vec::new();
let mut cb = ConfigBuilder::new();
cb.add_filter_ignore_str("async_std");
cb.add_filter_ignore_str("async_io");
cb.add_filter_ignore_str("polling");
cb.add_filter_ignore_str("rustls");
cb.add_filter_ignore_str("async_tungstenite");
cb.add_filter_ignore_str("tungstenite");
for ig in veilid_core::DEFAULT_LOG_IGNORE_LIST {
cb.add_filter_ignore_str(ig);
}
if let Some(level) = terminal_log {
logs.push(TermLogger::new(

4
veilid-core/src/intf/wasm/network/mod.rs
View File

@@ -172,6 +172,10 @@ impl Network {
self.inner.lock().network_needs_restart
}
pub fn is_started(&self) -> bool {
self.inner.lock().network_started
}
pub fn restart_network(&self) {
self.inner.lock().network_needs_restart = true;
}

11
veilid-core/src/intf/wasm/system.rs
View File

@@ -200,4 +200,13 @@ pub async fn get_outbound_relay_peer() -> Option<crate::veilid_api::PeerInfo> {
// } else {
// panic!("WASM requires browser or nodejs environment");
// }
// }
// }
pub async fn txt_lookup<S: AsRef<str>>(host: S) -> Result<Vec<String>, String> {
Err("wasm does not support txt lookup".to_owned())
}
pub async fn ptr_lookup(ip_addr: IpAddr) -> Result<String, String> {
Err("wasm does not support ptr lookup".to_owned())
}

4
veilid-core/src/lib.rs
View File

@@ -50,7 +50,7 @@ pub fn veilid_version() -> (u32, u32, u32) {
#[cfg(target_os = "android")]
pub use intf::utils::android::{veilid_core_setup_android, veilid_core_setup_android_no_log};
pub static DEFAULT_LOG_IGNORE_LIST: [&str; 8] = [
pub static DEFAULT_LOG_IGNORE_LIST: [&str; 10] = [
"async_std",
"async_io",
"polling",
@@ -59,4 +59,6 @@ pub static DEFAULT_LOG_IGNORE_LIST: [&str; 8] = [
"tungstenite",
"netlink_proto",
"netlink_sys",
"trust_dns_resolver",
"trust_dns_proto",
];

6
veilid-core/src/network_manager.rs
View File

@@ -266,10 +266,6 @@ impl NetworkManager {
Ok(())
}
pub fn is_started(&self) -> bool {
self.inner.lock().components.is_some()
}
pub async fn shutdown(&self) {
trace!("NetworkManager::shutdown begin");
@@ -1251,7 +1247,7 @@ impl NetworkManager {
}
fn get_veilid_state_inner(inner: &NetworkManagerInner) -> VeilidStateNetwork {
if inner.components.is_some() {
if inner.components.is_some() && inner.components.as_ref().unwrap().net.is_started() {
VeilidStateNetwork {
started: true,
bps_down: inner.stats.self_stats.transfer_stats.down.average,

59
veilid-core/src/routing_table/debug.rs
View File

@@ -22,37 +22,48 @@ impl RoutingTable {
out
}
pub fn debug_info_dialinfo(&self, txt_format: bool) -> String {
if txt_format {
let mut out = String::new();
let gdis = self.dial_info_details(RoutingDomain::PublicInternet);
if gdis.is_empty() {
out += "No TXT Record DialInfo\n";
} else {
out += "TXT Record DialInfo:\n";
out += &format!("{}\n", self.node_id().encode());
for gdi in gdis {
out += &format!("{}\n", gdi.dial_info);
}
}
out
pub async fn debug_info_txtrecord(&self) -> String {
let mut out = String::new();
let gdis = self.dial_info_details(RoutingDomain::PublicInternet);
if gdis.is_empty() {
out += "No TXT Record\n";
} else {
let ldis = self.dial_info_details(RoutingDomain::LocalNetwork);
let gdis = self.dial_info_details(RoutingDomain::PublicInternet);
let mut out = String::new();
out += "TXT Record:\n";
out += &self.node_id().encode();
out += "Local Network Dial Info Details:\n";
for (n, ldi) in ldis.iter().enumerate() {
out += &format!(" {:>2}: {:?}\n", n, ldi);
let mut urls = Vec::new();
for gdi in gdis {
urls.push(gdi.dial_info.to_url().await);
}
out += "Public Internet Dial Info Details:\n";
for (n, gdi) in gdis.iter().enumerate() {
out += &format!(" {:>2}: {:?}\n", n, gdi);
urls.sort();
urls.dedup();
for url in urls {
out += &format!(",{}", url);
}
out
out += "\n";
}
out
}
pub fn debug_info_dialinfo(&self) -> String {
let ldis = self.dial_info_details(RoutingDomain::LocalNetwork);
let gdis = self.dial_info_details(RoutingDomain::PublicInternet);
let mut out = String::new();
out += "Local Network Dial Info Details:\n";
for (n, ldi) in ldis.iter().enumerate() {
out += &format!(" {:>2}: {:?}\n", n, ldi);
}
out += "Public Internet Dial Info Details:\n";
for (n, gdi) in gdis.iter().enumerate() {
out += &format!(" {:>2}: {:?}\n", n, gdi);
}
out
}
pub fn debug_info_entries(&self, limit: usize, min_state: BucketEntryState) -> String {
let inner = self.inner.lock();
let cur_ts = get_timestamp();

8
veilid-core/src/tests/native/mod.rs
View File

@@ -129,11 +129,9 @@ cfg_if! {
pub fn setup() {
SETUP_ONCE.call_once(|| {
let mut cb = ConfigBuilder::new();
cb.add_filter_ignore_str("async_std");
cb.add_filter_ignore_str("async_io");
cb.add_filter_ignore_str("polling");
cb.add_filter_ignore_str("netlink_proto");
cb.add_filter_ignore_str("netlink_sys");
for ig in crate::DEFAULT_LOG_IGNORE_LIST {
cb.add_filter_ignore_str(ig);
}
TestLogger::init(LevelFilter::Trace, cb.build()).unwrap();
});
}

18
veilid-core/src/veilid_api/debug.rs
View File

@@ -79,16 +79,16 @@ impl VeilidAPI {
Ok(routing_table.debug_info_buckets(min_state))
}
async fn debug_dialinfo(&self, args: String) -> Result<String, VeilidAPIError> {
let args: Vec<String> = args.split_whitespace().map(|s| s.to_owned()).collect();
let is_txt = if args.len() == 1 {
args[0] == "txt"
} else {
false
};
async fn debug_dialinfo(&self, _args: String) -> Result<String, VeilidAPIError> {
// Dump routing table dialinfo
let routing_table = self.network_manager()?.routing_table();
Ok(routing_table.debug_info_dialinfo(is_txt))
Ok(routing_table.debug_info_dialinfo())
}
async fn debug_txtrecord(&self, _args: String) -> Result<String, VeilidAPIError> {
// Dump routing table txt record
let routing_table = self.network_manager()?.routing_table();
Ok(routing_table.debug_info_txtrecord().await)
}
async fn debug_entries(&self, args: String) -> Result<String, VeilidAPIError> {
@@ -259,6 +259,8 @@ impl VeilidAPI {
self.debug_buckets(rest).await
} else if arg == "dialinfo" {
self.debug_dialinfo(rest).await
} else if arg == "txtrecord" {
self.debug_txtrecord(rest).await
} else if arg == "entries" {
self.debug_entries(rest).await
} else if arg == "entry" {

79
veilid-core/src/veilid_api/mod.rs
View File

@@ -1035,6 +1035,85 @@ impl DialInfo {
address_type: Some(self.address_type()),
}
}
pub fn try_vec_from_url(url: String) -> Result<Vec<Self>, VeilidAPIError> {
let split_url = SplitUrl::from_str(&url)
.map_err(|e| parse_error!(format!("unable to split url: {}", e), url))?;
let port = match split_url.scheme.as_str() {
"udp" | "tcp" => split_url
.port
.ok_or_else(|| parse_error!("Missing port in udp url", url))?,
"ws" => split_url.port.unwrap_or(80u16),
"wss" => split_url.port.unwrap_or(443u16),
_ => {
return Err(parse_error!(
"Invalid dial info url scheme",
split_url.scheme
));
}
};
let socket_addrs = match split_url.host {
SplitUrlHost::Hostname(_) => split_url
.host_port(port)
.to_socket_addrs()
.map_err(|_| parse_error!("couldn't resolve hostname in url", url))?
.collect(),
SplitUrlHost::IpAddr(a) => vec![SocketAddr::new(a, port)],
};
let mut out = Vec::new();
for sa in socket_addrs {
out.push(match split_url.scheme.as_str() {
"udp" => Self::udp_from_socketaddr(sa),
"tcp" => Self::tcp_from_socketaddr(sa),
"ws" => Self::try_ws(
SocketAddress::from_socket_addr(sa).to_canonical(),
url.clone(),
)?,
"wss" => Self::try_wss(
SocketAddress::from_socket_addr(sa).to_canonical(),
url.clone(),
)?,
_ => {
unreachable!("Invalid dial info url scheme")
}
});
}
Ok(out)
}
pub async fn to_url(&self) -> String {
match self {
DialInfo::UDP(di) => intf::ptr_lookup(di.socket_address.to_ip_addr())
.await
.map(|h| format!("udp://{}:{}", h, di.socket_address.port()))
.unwrap_or_else(|_| format!("udp://{}", di.socket_address)),
DialInfo::TCP(di) => intf::ptr_lookup(di.socket_address.to_ip_addr())
.await
.map(|h| format!("tcp://{}:{}", h, di.socket_address.port()))
.unwrap_or_else(|_| format!("tcp://{}", di.socket_address)),
DialInfo::WS(di) => {
let mut split_url = SplitUrl::from_str(&format!("ws://{}", di.request)).unwrap();
if let SplitUrlHost::IpAddr(a) = split_url.host {
if let Ok(host) = intf::ptr_lookup(a).await {
split_url.host = SplitUrlHost::Hostname(host);
}
}
split_url.to_string()
}
DialInfo::WSS(di) => {
let mut split_url = SplitUrl::from_str(&format!("wss://{}", di.request)).unwrap();
if let SplitUrlHost::IpAddr(a) = split_url.host {
if let Ok(host) = intf::ptr_lookup(a).await {
split_url.host = SplitUrlHost::Hostname(host);
}
}
split_url.to_string()
}
}
}
}
impl MatchesDialInfoFilter for DialInfo {

4
veilid-core/src/xx/split_url.rs
View File

@@ -365,8 +365,10 @@ impl fmt::Display for SplitUrl {
} else {
format!("{}@{}", userinfo, self.host)
}
} else if let Some(port) = self.port {
format!("{}:{}", self.host, port)
} else {
self.host.to_string()
format!("{}", self.host)
}
};
if let Some(path) = &self.path {

17
veilid-server/src/cmdline.rs
View File

@@ -1,6 +1,7 @@
use crate::settings::*;
use clap::{Arg, ArgMatches, Command};
use std::ffi::OsStr;
use std::path::Path;
use std::str::FromStr;
use veilid_core::{DHTKey, DHTKeySecret};
@@ -133,12 +134,18 @@ pub fn process_command_line() -> Result<(Settings, ArgMatches), String> {
}
// Attempt to load configuration
let settings = Settings::new(if matches.occurrences_of("config-file") == 0 {
None
let settings_path = if let Some(config_file) = matches.value_of_os("config-file") {
if Path::new(config_file).exists() {
Some(config_file)
} else {
None
}
} else {
Some(matches.value_of_os("config-file").unwrap())
})
.map_err(|e| format!("configuration is invalid: {}", e))?;
None
};
let settings =
Settings::new(settings_path).map_err(|e| format!("configuration is invalid: {}", e))?;
// write lock the settings
let mut settingsrw = settings.write();

4
veilid-server/src/server.rs
View File

@@ -146,12 +146,12 @@ pub async fn run_veilid_server(
}
async_std::task::sleep(Duration::from_millis(100)).await;
}
match veilid_api.debug("dialinfo txt".to_string()).await {
match veilid_api.debug("txtrecord".to_string()).await {
Ok(v) => {
print!("{}", v);
}
Err(e) => {
let outerr = format!("Getting dial info failed: {:?}", e);
let outerr = format!("Getting TXT record failed: {:?}", e);
error!("{}", outerr);
out = Err(outerr);
}