liz/veilid
1
0
Fork 0
Code Pull Requests Packages Releases Activity

removing dev branch, many changes

Browse Source
This commit is contained in:
John Smith
2023-05-29 19:24:57 +00:00
parent 1430f3f656
commit 0a890c8707
250 changed files with 18084 additions and 8040 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
veilid-server/src/client_api.rs
View File

@@ -19,7 +19,7 @@ use veilid_core::*;
// Encoding for ApiResult
fn encode_api_result<T: Serialize + fmt::Debug>(
result: &Result<T, VeilidAPIError>,
result: &VeilidAPIResult<T>,
builder: &mut api_result::Builder,
) {
match result {

22
veilid-server/src/cmdline.rs
View File

@@ -42,6 +42,19 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result<clap::ArgMatches, clap
.multiple_occurrences(true)
.help("Specify configuration value to set (key in dot format, value in json format), eg: logging.api.enabled=true")
)
.arg(
Arg::new("password")
.short('p')
.long("password")
.takes_value(true)
.help("Specify password to use to protect the device encryption key")
)
.arg(
Arg::new("new-password")
.long("new-password")
.takes_value(true)
.help("Change password used to protect the device encryption key. Device storage will be migrated.")
)
.arg(
Arg::new("attach")
.long("attach")
@@ -82,7 +95,7 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result<clap::ArgMatches, clap
.long("generate-key-pair")
.takes_value(true)
.value_name("crypto_kind")
.default_missing_value("VLD0")
.default_missing_value("")
.help("Only generate a new keypair and print it")
.long_help("Generate a new keypair for a specific crypto kind and print both the key and its secret to the terminal, then exit immediately."),
)
@@ -231,6 +244,13 @@ pub fn process_command_line() -> EyreResult<(Settings, ArgMatches)> {
if matches.occurrences_of("delete-table-store") != 0 {
settingsrw.core.table_store.delete = true;
}
if matches.occurrences_of("password") != 0 {
settingsrw.core.protected_store.device_encryption_key_password = matches.value_of("password").unwrap().to_owned();
}
if matches.occurrences_of("new-password") != 0 {
settingsrw.core.protected_store.new_device_encryption_key_password = Some(matches.value_of("new-password").unwrap().to_owned());
}
if matches.occurrences_of("dump-txt-record") != 0 {
// Turn off terminal logging so we can be interactive
settingsrw.logging.terminal.enabled = false;

25
veilid-server/src/main.rs
View File

@@ -45,10 +45,27 @@ fn main() -> EyreResult<()> {
// --- Generate DHT Key ---
if matches.occurrences_of("generate-key-pair") != 0 {
if let Some(ckstr) = matches.get_one::<String>("generate-key-pair") {
let ck: veilid_core::CryptoKind =
veilid_core::FourCC::from_str(ckstr).wrap_err("couldn't parse crypto kind")?;
let tkp = veilid_core::Crypto::generate_keypair(ck).wrap_err("invalid crypto kind")?;
println!("{}", tkp.to_string());
if ckstr == "" {
let mut tks = veilid_core::TypedKeySet::new();
let mut tss = veilid_core::TypedSecretSet::new();
for ck in veilid_core::VALID_CRYPTO_KINDS {
let tkp = veilid_core::Crypto::generate_keypair(ck)
.wrap_err("invalid crypto kind")?;
tks.add(veilid_core::TypedKey::new(tkp.kind, tkp.value.key));
tss.add(veilid_core::TypedSecret::new(tkp.kind, tkp.value.secret));
}
println!(
"Public Keys:\n{}\nSecret Keys:\n{}\n",
tks.to_string(),
tss.to_string()
);
} else {
let ck: veilid_core::CryptoKind =
veilid_core::FourCC::from_str(ckstr).wrap_err("couldn't parse crypto kind")?;
let tkp =
veilid_core::Crypto::generate_keypair(ck).wrap_err("invalid crypto kind")?;
println!("{}", tkp.to_string());
}
return Ok(());
} else {
bail!("missing crypto kind");

19
veilid-server/src/server.rs
View File

@@ -59,8 +59,10 @@ pub async fn run_veilid_server_internal(
// Create VeilidCore setup
let update_callback = Arc::new(move |change: veilid_core::VeilidUpdate| {
if sender.send(change).is_err() {
error!("error sending veilid update callback");
if let Err(e) = sender.send(change) {
// Don't log here, as that loops the update callback in some cases and will deadlock
let change = e.into_inner();
eprintln!("error sending veilid update callback: {:?}", change);
}
});
let config_callback = settings.get_core_config_callback();
@@ -88,12 +90,8 @@ pub async fn run_veilid_server_internal(
// Process all updates
let capi2 = capi.clone();
let mut shutdown_switch = {
let shutdown_switch_locked = SHUTDOWN_SWITCH.lock();
(*shutdown_switch_locked).as_ref().map(|ss| ss.instance())
}
.unwrap()
.fuse();
let update_receiver_shutdown = SingleShotEventual::new(Some(()));
let mut update_receiver_shutdown_instance = update_receiver_shutdown.instance().fuse();
let update_receiver_jh = spawn_local(async move {
loop {
select! {
@@ -107,7 +105,7 @@ pub async fn run_veilid_server_internal(
break;
}
}
_ = shutdown_switch => {
_ = update_receiver_shutdown_instance => {
break;
}
};
@@ -177,6 +175,9 @@ pub async fn run_veilid_server_internal(
// Shut down Veilid API to release state change sender
veilid_api.shutdown().await;
// Shut down update receiver now that there are no more updates
update_receiver_shutdown.resolve(()).await;
// Wait for update receiver to exit
let _ = update_receiver_jh.await;

222
veilid-server/src/settings.rs
View File

@@ -6,12 +6,14 @@ use serde_derive::*;
use std::ffi::OsStr;
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use sysinfo::{DiskExt, SystemExt};
use url::Url;
use veilid_core::tools::*;
use veilid_core::*;
pub fn load_default_config() -> EyreResult<config::Config> {
let default_config = String::from(
let mut default_config = String::from(
r#"---
daemon:
enabled: false
@@ -46,8 +48,10 @@ core:
protected_store:
allow_insecure_fallback: true
always_use_insecure_storage: true
insecure_fallback_directory: '%INSECURE_FALLBACK_DIRECTORY%'
directory: '%DIRECTORY%'
delete: false
device_encryption_key_password: '%DEVICE_ENCRYPTION_KEY_PASSWORD%'
new_device_encryption_key_password: %NEW_DEVICE_ENCRYPTION_KEY_PASSWORD%
table_store:
directory: '%TABLE_STORE_DIRECTORY%'
delete: false
@@ -78,23 +82,29 @@ core:
queue_size: 1024
max_timestamp_behind_ms: 10000
max_timestamp_ahead_ms: 10000
timeout_ms: 10000
timeout_ms: 5000
max_route_hop_count: 4
default_route_hop_count: 1
dht:
resolve_node_timeout:
resolve_node_count: 20
resolve_node_fanout: 3
max_find_node_count: 20
get_value_timeout:
get_value_count: 20
get_value_fanout: 3
set_value_timeout:
set_value_count: 20
set_value_fanout: 5
resolve_node_timeout_ms: 10000
resolve_node_count: 1
resolve_node_fanout: 4
get_value_timeout_ms: 10000
get_value_count: 3
get_value_fanout: 4
set_value_timeout_ms: 10000
set_value_count: 5
set_value_fanout: 4
min_peer_count: 20
min_peer_refresh_time_ms: 2000
validate_dial_info_receipt_time_ms: 2000
local_subkey_cache_size: 128
local_max_subkey_cache_memory_mb: 256
remote_subkey_cache_size: 1024
remote_max_records: 65536
remote_max_subkey_cache_memory_mb: %REMOTE_MAX_SUBKEY_CACHE_MEMORY_MB%
remote_max_storage_space_mb: 0
upnp: true
detect_address_changes: true
restricted_nat_retries: 0
@@ -150,8 +160,8 @@ core:
&Settings::get_default_block_store_path().to_string_lossy(),
)
.replace(
"%INSECURE_FALLBACK_DIRECTORY%",
&Settings::get_default_protected_store_insecure_fallback_directory().to_string_lossy(),
"%DIRECTORY%",
&Settings::get_default_protected_store_directory().to_string_lossy(),
)
.replace(
"%CERTIFICATE_PATH%",
@@ -164,7 +174,35 @@ core:
&Settings::get_default_private_key_directory()
.join("server.key")
.to_string_lossy(),
)
.replace(
"%REMOTE_MAX_SUBKEY_CACHE_MEMORY_MB%",
&Settings::get_default_remote_max_subkey_cache_memory_mb().to_string(),
);
let dek_password = if let Some(dek_password) = std::env::var_os("DEK_PASSWORD") {
dek_password
.to_str()
.ok_or_else(|| eyre!("DEK_PASSWORD is not valid unicode"))?
.to_owned()
} else {
"".to_owned()
};
default_config = default_config.replace("%DEVICE_ENCRYPTION_KEY_PASSWORD%", &dek_password);
let new_dek_password = if let Some(new_dek_password) = std::env::var_os("NEW_DEK_PASSWORD") {
format!(
"'{}'",
new_dek_password
.to_str()
.ok_or_else(|| eyre!("NEW_DEK_PASSWORD is not valid unicode"))?
)
} else {
"null".to_owned()
};
default_config =
default_config.replace("%NEW_DEVICE_ENCRYPTION_KEY_PASSWORD%", &new_dek_password);
config::Config::builder()
.add_source(config::File::from_str(
&default_config,
@@ -499,19 +537,25 @@ pub struct Rpc {
#[derive(Debug, Deserialize, Serialize)]
pub struct Dht {
pub resolve_node_timeout_ms: Option<u32>,
pub max_find_node_count: u32,
pub resolve_node_timeout_ms: u32,
pub resolve_node_count: u32,
pub resolve_node_fanout: u32,
pub max_find_node_count: u32,
pub get_value_timeout_ms: Option<u32>,
pub get_value_timeout_ms: u32,
pub get_value_count: u32,
pub get_value_fanout: u32,
pub set_value_timeout_ms: Option<u32>,
pub set_value_timeout_ms: u32,
pub set_value_count: u32,
pub set_value_fanout: u32,
pub min_peer_count: u32,
pub min_peer_refresh_time_ms: u32,
pub validate_dial_info_receipt_time_ms: u32,
pub local_subkey_cache_size: u32,
pub local_max_subkey_cache_memory_mb: u32,
pub remote_subkey_cache_size: u32,
pub remote_max_records: u32,
pub remote_max_subkey_cache_memory_mb: u32,
pub remote_max_storage_space_mb: u32,
}
#[derive(Debug, Deserialize, Serialize)]
@@ -569,8 +613,10 @@ pub struct BlockStore {
pub struct ProtectedStore {
pub allow_insecure_fallback: bool,
pub always_use_insecure_storage: bool,
pub insecure_fallback_directory: PathBuf,
pub directory: PathBuf,
pub delete: bool,
pub device_encryption_key_password: String,
pub new_device_encryption_key_password: Option<String>,
}
#[derive(Debug, Deserialize, Serialize)]
@@ -621,7 +667,13 @@ impl Settings {
}
// Generate config
let inner: SettingsInner = cfg.try_deserialize()?;
let mut inner: SettingsInner = cfg.try_deserialize()?;
// Fill in missing defaults
if inner.core.network.dht.remote_max_storage_space_mb == 0 {
inner.core.network.dht.remote_max_storage_space_mb =
Self::get_default_remote_max_storage_space_mb(&inner);
}
//
Ok(Self {
@@ -772,7 +824,7 @@ impl Settings {
bs_path
}
pub fn get_default_protected_store_insecure_fallback_directory() -> PathBuf {
pub fn get_default_protected_store_directory() -> PathBuf {
#[cfg(unix)]
{
let globalpath = PathBuf::from("/var/db/veilid-server/protected_store");
@@ -833,6 +885,36 @@ impl Settings {
pk_path
}
pub fn get_default_remote_max_subkey_cache_memory_mb() -> u32 {
let sys = sysinfo::System::new_with_specifics(sysinfo::RefreshKind::new().with_memory());
((sys.free_memory() / (1024u64 * 1024u64)) / 16) as u32
}
pub fn get_default_remote_max_storage_space_mb(inner: &SettingsInner) -> u32 {
let mut sys = sysinfo::System::new_with_specifics(sysinfo::RefreshKind::new().with_disks());
let dht_storage_path = inner.core.table_store.directory.clone();
// Sort longer mount point paths first since we want the mount point closest to our table store directory
sys.sort_disks_by(|a, b| {
b.mount_point()
.to_string_lossy()
.len()
.cmp(&a.mount_point().to_string_lossy().len())
});
for disk in sys.disks() {
if dht_storage_path.starts_with(disk.mount_point()) {
let available_mb = disk.available_space() / 1_000_000u64;
if available_mb > 40_000 {
// Default to 10GB if more than 40GB is available
return 10_000;
}
// Default to 1/4 of the available space, if less than 40GB is available
return available_mb as u32;
}
}
// If we can't figure out our storage path go with 1GB of space and pray
1_000
}
pub fn set(&self, key: &str, value: &str) -> EyreResult<()> {
let mut inner = self.inner.write();
@@ -882,11 +964,19 @@ impl Settings {
inner.core.protected_store.always_use_insecure_storage,
value
);
set_config_value!(inner.core.protected_store.directory, value);
set_config_value!(inner.core.protected_store.delete, value);
set_config_value!(
inner.core.protected_store.insecure_fallback_directory,
inner.core.protected_store.device_encryption_key_password,
value
);
set_config_value!(
inner
.core
.protected_store
.new_device_encryption_key_password,
value
);
set_config_value!(inner.core.protected_store.delete, value);
set_config_value!(inner.core.table_store.directory, value);
set_config_value!(inner.core.table_store.delete, value);
set_config_value!(inner.core.block_store.directory, value);
@@ -921,10 +1011,10 @@ impl Settings {
set_config_value!(inner.core.network.rpc.timeout_ms, value);
set_config_value!(inner.core.network.rpc.max_route_hop_count, value);
set_config_value!(inner.core.network.rpc.default_route_hop_count, value);
set_config_value!(inner.core.network.dht.max_find_node_count, value);
set_config_value!(inner.core.network.dht.resolve_node_timeout_ms, value);
set_config_value!(inner.core.network.dht.resolve_node_count, value);
set_config_value!(inner.core.network.dht.resolve_node_fanout, value);
set_config_value!(inner.core.network.dht.max_find_node_count, value);
set_config_value!(inner.core.network.dht.get_value_timeout_ms, value);
set_config_value!(inner.core.network.dht.get_value_count, value);
set_config_value!(inner.core.network.dht.get_value_fanout, value);
@@ -937,6 +1027,18 @@ impl Settings {
inner.core.network.dht.validate_dial_info_receipt_time_ms,
value
);
set_config_value!(inner.core.network.dht.local_subkey_cache_size, value);
set_config_value!(
inner.core.network.dht.local_max_subkey_cache_memory_mb,
value
);
set_config_value!(inner.core.network.dht.remote_subkey_cache_size, value);
set_config_value!(inner.core.network.dht.remote_max_records, value);
set_config_value!(
inner.core.network.dht.remote_max_subkey_cache_memory_mb,
value
);
set_config_value!(inner.core.network.dht.remote_max_storage_space_mb, value);
set_config_value!(inner.core.network.upnp, value);
set_config_value!(inner.core.network.detect_address_changes, value);
set_config_value!(inner.core.network.restricted_nat_retries, value);
@@ -1000,15 +1102,29 @@ impl Settings {
"protected_store.always_use_insecure_storage" => Ok(Box::new(
inner.core.protected_store.always_use_insecure_storage,
)),
"protected_store.insecure_fallback_directory" => Ok(Box::new(
"protected_store.directory" => Ok(Box::new(
inner
.core
.protected_store
.insecure_fallback_directory
.directory
.to_string_lossy()
.to_string(),
)),
"protected_store.delete" => Ok(Box::new(inner.core.protected_store.delete)),
"protected_store.device_encryption_key_password" => Ok(Box::new(
inner
.core
.protected_store
.device_encryption_key_password
.clone(),
)),
"protected_store.new_device_encryption_key_password" => Ok(Box::new(
inner
.core
.protected_store
.new_device_encryption_key_password
.clone(),
)),
"table_store.directory" => Ok(Box::new(
inner
@@ -1108,6 +1224,9 @@ impl Settings {
"network.rpc.default_route_hop_count" => {
Ok(Box::new(inner.core.network.rpc.default_route_hop_count))
}
"network.dht.max_find_node_count" => {
Ok(Box::new(inner.core.network.dht.max_find_node_count))
}
"network.dht.resolve_node_timeout_ms" => {
Ok(Box::new(inner.core.network.dht.resolve_node_timeout_ms))
}
@@ -1117,9 +1236,6 @@ impl Settings {
"network.dht.resolve_node_fanout" => {
Ok(Box::new(inner.core.network.dht.resolve_node_fanout))
}
"network.dht.max_find_node_count" => {
Ok(Box::new(inner.core.network.dht.max_find_node_count))
}
"network.dht.get_value_timeout_ms" => {
Ok(Box::new(inner.core.network.dht.get_value_timeout_ms))
}
@@ -1145,6 +1261,25 @@ impl Settings {
"network.dht.validate_dial_info_receipt_time_ms" => Ok(Box::new(
inner.core.network.dht.validate_dial_info_receipt_time_ms,
)),
"network.dht.local_subkey_cache_size" => {
Ok(Box::new(inner.core.network.dht.local_subkey_cache_size))
}
"network.dht.local_max_subkey_cache_memory_mb" => Ok(Box::new(
inner.core.network.dht.local_max_subkey_cache_memory_mb,
)),
"network.dht.remote_subkey_cache_size" => {
Ok(Box::new(inner.core.network.dht.remote_subkey_cache_size))
}
"network.dht.remote_max_records" => {
Ok(Box::new(inner.core.network.dht.remote_max_records))
}
"network.dht.remote_max_subkey_cache_memory_mb" => Ok(Box::new(
inner.core.network.dht.remote_max_subkey_cache_memory_mb,
)),
"network.dht.remote_max_storage_space_mb" => {
Ok(Box::new(inner.core.network.dht.remote_max_storage_space_mb))
}
"network.upnp" => Ok(Box::new(inner.core.network.upnp)),
"network.detect_address_changes" => {
Ok(Box::new(inner.core.network.detect_address_changes))
@@ -1420,10 +1555,15 @@ mod tests {
assert_eq!(s.core.protected_store.allow_insecure_fallback, true);
assert_eq!(s.core.protected_store.always_use_insecure_storage, true);
assert_eq!(
s.core.protected_store.insecure_fallback_directory,
Settings::get_default_protected_store_insecure_fallback_directory()
s.core.protected_store.directory,
Settings::get_default_protected_store_directory()
);
assert_eq!(s.core.protected_store.delete, false);
assert_eq!(s.core.protected_store.device_encryption_key_password, "");
assert_eq!(
s.core.protected_store.new_device_encryption_key_password,
None
);
assert_eq!(s.core.network.connection_initial_timeout_ms, 2_000u32);
assert_eq!(s.core.network.connection_inactivity_timeout_ms, 60_000u32);
@@ -1446,20 +1586,20 @@ mod tests {
assert_eq!(s.core.network.rpc.queue_size, 1024);
assert_eq!(s.core.network.rpc.max_timestamp_behind_ms, Some(10_000u32));
assert_eq!(s.core.network.rpc.max_timestamp_ahead_ms, Some(10_000u32));
assert_eq!(s.core.network.rpc.timeout_ms, 10_000u32);
assert_eq!(s.core.network.rpc.timeout_ms, 5_000u32);
assert_eq!(s.core.network.rpc.max_route_hop_count, 4);
assert_eq!(s.core.network.rpc.default_route_hop_count, 1);
//
assert_eq!(s.core.network.dht.resolve_node_timeout_ms, None);
assert_eq!(s.core.network.dht.resolve_node_count, 20u32);
assert_eq!(s.core.network.dht.resolve_node_fanout, 3u32);
assert_eq!(s.core.network.dht.max_find_node_count, 20u32);
assert_eq!(s.core.network.dht.get_value_timeout_ms, None);
assert_eq!(s.core.network.dht.get_value_count, 20u32);
assert_eq!(s.core.network.dht.get_value_fanout, 3u32);
assert_eq!(s.core.network.dht.set_value_timeout_ms, None);
assert_eq!(s.core.network.dht.set_value_count, 20u32);
assert_eq!(s.core.network.dht.set_value_fanout, 5u32);
assert_eq!(s.core.network.dht.resolve_node_timeout_ms, 10_000u32);
assert_eq!(s.core.network.dht.resolve_node_count, 1u32);
assert_eq!(s.core.network.dht.resolve_node_fanout, 4u32);
assert_eq!(s.core.network.dht.get_value_timeout_ms, 10_000u32);
assert_eq!(s.core.network.dht.get_value_count, 3u32);
assert_eq!(s.core.network.dht.get_value_fanout, 4u32);
assert_eq!(s.core.network.dht.set_value_timeout_ms, 10_000u32);
assert_eq!(s.core.network.dht.set_value_count, 5u32);
assert_eq!(s.core.network.dht.set_value_fanout, 4u32);
assert_eq!(s.core.network.dht.min_peer_count, 20u32);
assert_eq!(s.core.network.dht.min_peer_refresh_time_ms, 2_000u32);
assert_eq!(

36
veilid-server/src/unix.rs
View File

@@ -7,7 +7,7 @@ use clap::ArgMatches;
use futures_util::StreamExt;
use signal_hook::consts::signal::*;
use signal_hook_async_std::Signals;
use std::io::Read;
//use std::io::Read;
use tracing::*;
#[instrument(skip(signals))]
@@ -34,23 +34,23 @@ pub fn run_daemon(settings: Settings, _matches: ArgMatches) -> EyreResult<()> {
let s = settings.read();
if let Some(pid_file) = s.daemon.pid_file.clone() {
daemon = daemon.pid_file(pid_file.clone()); //.chown_pid_file(true);
daemon = daemon.exit_action(move || {
// wait for pid file to exist before exiting parent
let pid_path = std::path::Path::new(&pid_file);
loop {
if let Ok(mut f) = std::fs::File::open(pid_path) {
let mut s = String::new();
if f.read_to_string(&mut s).is_ok()
&& !s.is_empty()
&& s.parse::<u32>().is_ok()
{
println!("pidfile found");
break;
}
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
})
// daemon = daemon.exit_action(move || {
// // wait for pid file to exist before exiting parent
// let pid_path = std::path::Path::new(&pid_file);
// loop {
// if let Ok(mut f) = std::fs::File::open(pid_path) {
// let mut s = String::new();
// if f.read_to_string(&mut s).is_ok()
// && !s.is_empty()
// && s.parse::<u32>().is_ok()
// {
// println!("pidfile found");
// break;
// }
// }
// std::thread::sleep(std::time::Duration::from_millis(100));
// }
// })
}
if let Some(chroot) = &s.daemon.chroot {
daemon = daemon.chroot(chroot);