From effc4aeeac31c7d2aff5226a090ecd1b40166158 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 18 Jan 2022 21:21:11 -0500 Subject: [PATCH] config work --- Cargo.lock | 1 + veilid-cli/Cargo.toml | 1 + veilid-cli/src/client_api_connection.rs | 32 ++++++--- .../src/intf/native/protected_store.rs | 68 ++++++++++++------- veilid-core/src/intf/native/table_store.rs | 6 ++ veilid-core/src/veilid_config.rs | 13 ++++ veilid-core/src/veilid_core.rs | 6 ++ veilid-flutter/rust/src/api.rs | 9 +++ veilid-server/src/cmdline.rs | 49 +++++++++---- veilid-server/src/main.rs | 4 +- veilid-server/src/settings.rs | 57 ++++++++++++++++ 11 files changed, 196 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8c9b3489..b88223f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3776,6 +3776,7 @@ version = "0.1.0" dependencies = [ "async-std", "async-tungstenite 0.8.0", + "async_executors", "bugsalot", "capnp", "capnp-rpc", diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index 9f1a19ac..7b50b714 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -13,6 +13,7 @@ path = "src/main.rs" [dependencies] async-std = { version = "^1.9", features = ["unstable", "attributes"] } async-tungstenite = { version = "^0.8", features = ["async-std-runtime"] } +async_executors = { version = "^0", default-features = false, features = [ "async_std" ]} cursive = { path = "../external/cursive/cursive", default-features = false, features = ["crossterm-backend", "toml"]} cursive-flexi-logger-view = { path = "../external/cursive-flexi-logger-view" } cursive_buffered_backend = { path = "../external/cursive_buffered_backend" } diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index 2200d600..497674d4 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -1,13 +1,13 @@ use crate::command_processor::*; use crate::veilid_client_capnp::*; -use veilid_core::xx::*; - +use async_executors::{AsyncStd, LocalSpawnHandleExt}; use capnp::capability::Promise; use capnp_rpc::{pry, rpc_twoparty_capnp, twoparty, Disconnector, RpcSystem}; use futures::AsyncReadExt; use std::cell::RefCell; use std::net::SocketAddr; use std::rc::Rc; +use veilid_core::xx::*; struct VeilidClientImpl { comproc: CommandProcessor, @@ -152,19 +152,29 @@ impl ClientApiConnection { )); } - // Send the request and get the state object and the registration object - if let Ok(response) = request.send().promise.await { - if let Ok(response) = response.get() { - if let Ok(_registration) = response.get_registration() { - if let Ok(state) = response.get_state() { - // Set up our state for the first time - if self.process_veilid_state(state).await.is_ok() { - // Don't drop the registration - rpc_system.await.map_err(map_to_string)?; + // Process the rpc system until we decide we're done + if let Ok(rpc_jh) = AsyncStd.spawn_handle_local(rpc_system) { + // Send the request and get the state object and the registration object + if let Ok(response) = request.send().promise.await { + if let Ok(response) = response.get() { + if let Ok(_registration) = response.get_registration() { + if let Ok(state) = response.get_state() { + // Set up our state for the first time + if self.process_veilid_state(state).await.is_ok() { + // Don't drop the registration, doing so will remove the client + // object mapping from the server which we need for the update backchannel + + // Wait until rpc system completion or disconnect was requested + if let Err(e) = rpc_jh.await { + error!("Client RPC system error: {}", e); + } + } } } } } + } else { + error!("Failed to spawn client RPC system"); } // Drop the server and disconnector too (if we still have it) diff --git a/veilid-core/src/intf/native/protected_store.rs b/veilid-core/src/intf/native/protected_store.rs index d37e2057..8677dccd 100644 --- a/veilid-core/src/intf/native/protected_store.rs +++ b/veilid-core/src/intf/native/protected_store.rs @@ -29,35 +29,55 @@ impl ProtectedStore { } } + pub async fn delete_all(&self) -> Result<(), String> { + // Delete all known keys + if self.remove_user_secret_string("node_id").await? { + debug!("deleted protected_store key 'node_id'"); + } + if self.remove_user_secret_string("node_id_secret").await? { + debug!("deleted protected_store key 'node_id_secret'"); + } + if self.remove_user_secret_string("_test_key").await? { + debug!("deleted protected_store key '_test_key'"); + } + Ok(()) + } + pub async fn init(&self) -> Result<(), String> { let c = self.config.get(); - let mut inner = self.inner.lock(); - if !c.protected_store.always_use_insecure_storage { - cfg_if! { - if #[cfg(target_os = "android")] { - inner.keyring_manager = KeyringManager::new_secure(&c.program_name, intf::native::utils::android::get_android_globals()).ok(); - } else { - inner.keyring_manager = KeyringManager::new_secure(&c.program_name).ok(); + { + let mut inner = self.inner.lock(); + if !c.protected_store.always_use_insecure_storage { + cfg_if! { + if #[cfg(target_os = "android")] { + inner.keyring_manager = KeyringManager::new_secure(&c.program_name, intf::native::utils::android::get_android_globals()).ok(); + } else { + inner.keyring_manager = KeyringManager::new_secure(&c.program_name).ok(); + } } } + if (c.protected_store.always_use_insecure_storage + || c.protected_store.allow_insecure_fallback) + && inner.keyring_manager.is_none() + { + let insecure_fallback_directory = + Path::new(&c.protected_store.insecure_fallback_directory); + let insecure_keyring_file = insecure_fallback_directory + .to_owned() + .join("insecure_keyring"); + inner.keyring_manager = Some( + KeyringManager::new_insecure(&c.program_name, &insecure_keyring_file) + .map_err(map_to_string) + .map_err(logthru_pstore!(error))?, + ); + } + if inner.keyring_manager.is_none() { + return Err("Could not initialize the protected store.".to_owned()); + } } - if (c.protected_store.always_use_insecure_storage - || c.protected_store.allow_insecure_fallback) - && inner.keyring_manager.is_none() - { - let insecure_fallback_directory = - Path::new(&c.protected_store.insecure_fallback_directory); - let insecure_keyring_file = insecure_fallback_directory - .to_owned() - .join("insecure_keyring"); - inner.keyring_manager = Some( - KeyringManager::new_insecure(&c.program_name, &insecure_keyring_file) - .map_err(map_to_string) - .map_err(logthru_pstore!(error))?, - ); - } - if inner.keyring_manager.is_none() { - return Err("Could not initialize the protected store.".to_owned()); + + if c.protected_store.delete { + self.delete_all().await?; } Ok(()) diff --git a/veilid-core/src/intf/native/table_store.rs b/veilid-core/src/intf/native/table_store.rs index 1d3e979a..7bc04148 100644 --- a/veilid-core/src/intf/native/table_store.rs +++ b/veilid-core/src/intf/native/table_store.rs @@ -27,6 +27,12 @@ impl TableStore { } } + pub async fn delete_all(&self) -> Result<(), String> { + // Delete all known keys + self.delete("crypto_caches").await?; + Ok(()) + } + pub async fn init(&self) -> Result<(), String> { Ok(()) } diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index 385b9f99..6ba94c56 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -141,6 +141,13 @@ pub struct VeilidConfigNetwork { #[derive(Default, Clone)] pub struct VeilidConfigTableStore { pub directory: String, + pub delete: bool, +} + +#[derive(Default, Clone)] +pub struct VeilidConfigBlockStore { + pub directory: String, + pub delete: bool, } #[derive(Default, Clone)] @@ -148,6 +155,7 @@ pub struct VeilidConfigProtectedStore { pub allow_insecure_fallback: bool, pub always_use_insecure_storage: bool, pub insecure_fallback_directory: String, + pub delete: bool, } #[derive(Default, Clone)] @@ -168,6 +176,7 @@ pub struct VeilidConfigInner { pub capabilities: VeilidConfigCapabilities, pub protected_store: VeilidConfigProtectedStore, pub table_store: VeilidConfigTableStore, + pub block_store: VeilidConfigBlockStore, pub network: VeilidConfigNetwork, } @@ -216,9 +225,13 @@ impl VeilidConfig { get_config!(inner.capabilities.protocol_connect_wss); get_config!(inner.capabilities.protocol_accept_wss); get_config!(inner.table_store.directory); + get_config!(inner.table_store.delete); + get_config!(inner.block_store.directory); + get_config!(inner.block_store.delete); get_config!(inner.protected_store.allow_insecure_fallback); get_config!(inner.protected_store.always_use_insecure_storage); get_config!(inner.protected_store.insecure_fallback_directory); + get_config!(inner.protected_store.delete); get_config!(inner.network.node_id); get_config!(inner.network.node_id_secret); get_config!(inner.network.max_connections); diff --git a/veilid-core/src/veilid_core.rs b/veilid-core/src/veilid_core.rs index efeb99a5..8f137646 100644 --- a/veilid-core/src/veilid_core.rs +++ b/veilid-core/src/veilid_core.rs @@ -124,6 +124,12 @@ impl VeilidCore { crypto.init().await?; inner.crypto = Some(crypto.clone()); + // Set up block store + // trace!("VeilidCore::internal_startup init block store"); + // let block_store = BlockStore::new(config.clone()); + // block_store.init().await?; + // inner.block_store = Some(block_store.clone();) + // Set up attachment manager trace!("VeilidCore::internal_startup init attachment manager"); let cb = setup.update_callback; diff --git a/veilid-flutter/rust/src/api.rs b/veilid-flutter/rust/src/api.rs index dbcc5ddc..1dbf2ced 100644 --- a/veilid-flutter/rust/src/api.rs +++ b/veilid-flutter/rust/src/api.rs @@ -101,6 +101,13 @@ pub struct VeilidConfigNetwork { #[derive(Default, Clone)] pub struct VeilidConfigTableStore { pub directory: String, + pub delete: bool, +} + +#[derive(Default, Clone)] +pub struct VeilidConfigBlockStore { + pub directory: String, + pub delete: bool, } #[derive(Default, Clone)] @@ -108,6 +115,7 @@ pub struct VeilidConfigProtectedStore { pub allow_insecure_fallback: bool, pub always_use_insecure_storage: bool, pub insecure_fallback_directory: String, + pub delete: bool, } #[derive(Default, Clone)] @@ -128,6 +136,7 @@ pub struct VeilidConfig { pub capabilities: VeilidConfigCapabilities, pub protected_store: VeilidConfigProtectedStore, pub table_store: VeilidConfigTableStore, + pub block_store: VeilidConfigBlockStore, pub network: VeilidConfigNetwork, } diff --git a/veilid-server/src/cmdline.rs b/veilid-server/src/cmdline.rs index a562e221..84239f0e 100644 --- a/veilid-server/src/cmdline.rs +++ b/veilid-server/src/cmdline.rs @@ -1,10 +1,9 @@ use crate::settings::*; -use std::ffi::OsStr; use clap::{App, Arg, ArgMatches}; +use std::ffi::OsStr; use std::str::FromStr; fn do_clap_matches(default_config_path: &OsStr) -> Result { - let matches = App::new("veilid-server") .version("0.1") .about("Veilid Server") @@ -22,6 +21,7 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result Result Result Result<(Settings, ArgMatches), String> { - // Get command line options let default_config_path = Settings::get_default_config_path(); let matches = do_clap_matches(default_config_path.as_os_str()) @@ -126,6 +139,7 @@ pub fn process_command_line() -> Result<(Settings, ArgMatches), String> { } settingsrw.testing.subnode_index = subnode_index; } + if matches.occurrences_of("debug") != 0 { settingsrw.logging.terminal.enabled = true; settingsrw.logging.terminal.level = LogLevel::Debug; @@ -140,6 +154,15 @@ pub fn process_command_line() -> Result<(Settings, ArgMatches), String> { if matches.is_present("local") { settingsrw.core.network.enable_local_peer_scope = true; } + if matches.occurrences_of("delete-protected-store") != 0 { + settingsrw.core.protected_store.delete = true; + } + if matches.occurrences_of("delete-block-store") != 0 { + settingsrw.core.block_store.delete = true; + } + if matches.occurrences_of("delete-table-store") != 0 { + settingsrw.core.table_store.delete = true; + } if matches.occurrences_of("bootstrap") != 0 { let bootstrap = match matches.value_of("bootstrap") { Some(x) => { @@ -170,4 +193,4 @@ pub fn process_command_line() -> Result<(Settings, ArgMatches), String> { .map_err(|_| "failed to apply subnode index".to_owned())?; Ok((settings, matches)) -} \ No newline at end of file +} diff --git a/veilid-server/src/main.rs b/veilid-server/src/main.rs index 7bc3e060..81025cd1 100644 --- a/veilid-server/src/main.rs +++ b/veilid-server/src/main.rs @@ -33,8 +33,8 @@ fn main() -> Result<(), String> { .map_err(|e| e.to_string()); } - // --- Generate Id --- - if matches.occurrences_of("generate-id") != 0 { + // --- Generate DHT Key --- + if matches.occurrences_of("generate-dht-key") != 0 { let (key, secret) = veilid_core::generate_secret(); println!("Public: {}\nSecret: {}", key.encode(), secret.encode()); return Ok(()); diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index fc28e55c..4221c9b8 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -39,8 +39,13 @@ core: allow_insecure_fallback: true always_use_insecure_storage: false insecure_fallback_directory: '%INSECURE_FALLBACK_DIRECTORY%' + delete: false table_store: directory: '%TABLE_STORE_DIRECTORY%' + delete: false + block_store: + directory: '%BLOCK_STORE_DIRECTORY%' + delete: false network: max_connections: 16 connection_initial_timeout: 2000000 @@ -124,6 +129,10 @@ core: "%TABLE_STORE_DIRECTORY%", &Settings::get_default_table_store_path().to_string_lossy(), ) + .replace( + "%BLOCK_STORE_DIRECTORY%", + &Settings::get_default_block_store_path().to_string_lossy(), + ) .replace( "%INSECURE_FALLBACK_DIRECTORY%", &Settings::get_default_protected_store_insecure_fallback_directory().to_string_lossy(), @@ -532,6 +541,13 @@ pub struct Testing { #[derive(Debug, Deserialize, Serialize)] pub struct TableStore { pub directory: PathBuf, + pub delete: bool, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct BlockStore { + pub directory: PathBuf, + pub delete: bool, } #[derive(Debug, Deserialize, Serialize)] @@ -539,12 +555,14 @@ pub struct ProtectedStore { pub allow_insecure_fallback: bool, pub always_use_insecure_storage: bool, pub insecure_fallback_directory: PathBuf, + pub delete: bool, } #[derive(Debug, Deserialize, Serialize)] pub struct Core { pub protected_store: ProtectedStore, pub table_store: TableStore, + pub block_store: BlockStore, pub network: Network, } @@ -693,6 +711,20 @@ impl Settings { default_config_path } + pub fn get_default_block_store_path() -> PathBuf { + // Get default configuration file location + let mut default_config_path; + + if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") { + default_config_path = PathBuf::from(my_proj_dirs.data_local_dir()); + } else { + default_config_path = PathBuf::from("./"); + } + default_config_path.push("block_store"); + + default_config_path + } + pub fn get_default_protected_store_insecure_fallback_directory() -> PathBuf { // Get default configuration file location let mut default_config_path; @@ -740,6 +772,8 @@ impl Settings { .to_string_lossy() .to_string(), )), + "protected_store.delete" => Ok(Box::new(inner.core.protected_store.delete)), + "table_store.directory" => Ok(Box::new( inner .core @@ -748,6 +782,18 @@ impl Settings { .to_string_lossy() .to_string(), )), + "table_store.delete" => Ok(Box::new(inner.core.table_store.delete)), + + "block_store.directory" => Ok(Box::new( + inner + .core + .block_store + .directory + .to_string_lossy() + .to_string(), + )), + "block_store.delete" => Ok(Box::new(inner.core.block_store.delete)), + "network.max_connections" => Ok(Box::new(inner.core.network.max_connections)), "network.connection_initial_timeout" => { Ok(Box::new(inner.core.network.connection_initial_timeout)) @@ -1070,16 +1116,27 @@ mod tests { assert_eq!(s.logging.client.enabled, true); assert_eq!(s.logging.client.level, LogLevel::Info); assert_eq!(s.testing.subnode_index, 0); + assert_eq!( s.core.table_store.directory, Settings::get_default_table_store_path() ); + assert_eq!(s.core.table_store.delete, false); + + assert_eq!( + s.core.block_store.directory, + Settings::get_default_block_store_path() + ); + assert_eq!(s.core.block_store.delete, false); + assert_eq!(s.core.protected_store.allow_insecure_fallback, true); assert_eq!(s.core.protected_store.always_use_insecure_storage, false); assert_eq!( s.core.protected_store.insecure_fallback_directory, Settings::get_default_protected_store_insecure_fallback_directory() ); + assert_eq!(s.core.protected_store.delete, false); + assert_eq!(s.core.network.max_connections, 16); assert_eq!(s.core.network.connection_initial_timeout, 2_000_000u64); assert_eq!(s.core.network.node_id, veilid_core::DHTKey::default());