diff --git a/.cargo/config.toml b/.cargo/config.toml index af7d22a8..91fa3b11 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,6 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] + [target.aarch64-unknown-linux-gnu] linker = "aarch64-linux-gnu-gcc" diff --git a/Cargo.lock b/Cargo.lock index a4656778..356bf366 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -973,6 +973,42 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "console-api" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57ff02e8ad8e06ab9731d5dc72dc23bef9200778eae1a89d555d8c42e5d4a86" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a3a81dfaf6b66bce5d159eddae701e3a002f194d378cbf7be5f053c281d9be" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio 1.21.2", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -1040,6 +1076,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "criterion" version = "0.4.0" @@ -1736,6 +1781,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flexi_logger" version = "0.23.3" @@ -2080,6 +2135,19 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "hdrhistogram" +version = "7.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +dependencies = [ + "base64 0.13.1", + "byteorder", + "flate2", + "nom 7.1.1", + "num-traits", +] + [[package]] name = "heck" version = "0.4.0" @@ -2172,6 +2240,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.22" @@ -4849,6 +4923,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "winapi 0.3.9", ] @@ -5532,6 +5607,7 @@ dependencies = [ "ansi_term", "async-std", "async-tungstenite 0.18.0", + "backtrace", "bugsalot", "capnp", "capnp-rpc", @@ -5540,6 +5616,7 @@ dependencies = [ "clap", "color-eyre", "config", + "console-subscriber", "ctrlc", "daemonize", "directories", diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index 682269c3..36276179 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -3,6 +3,7 @@ use crate::tools::*; use crate::veilid_client_capnp::*; use capnp::capability::Promise; use capnp_rpc::{pry, rpc_twoparty_capnp, twoparty, Disconnector, RpcSystem}; +use futures::future::FutureExt; use serde::de::DeserializeOwned; use std::cell::RefCell; use std::net::SocketAddr; @@ -101,6 +102,7 @@ struct ClientApiConnectionInner { disconnector: Option>, server: Option>>, disconnect_requested: bool, + cancel_eventual: Eventual, } type Handle = Rc>; @@ -119,9 +121,19 @@ impl ClientApiConnection { disconnector: None, server: None, disconnect_requested: false, + cancel_eventual: Eventual::new(), })), } } + + pub fn cancel(&self) { + let eventual = { + let inner = self.inner.borrow(); + inner.cancel_eventual.clone() + }; + eventual.resolve(); // don't need to await this + } + async fn process_veilid_state<'a>( &'a mut self, veilid_state: VeilidState, @@ -264,6 +276,34 @@ impl ClientApiConnection { } } + pub fn cancellable(&mut self, p: Promise) -> Promise + where + T: 'static, + { + let (mut cancel_instance, cancel_eventual) = { + let inner = self.inner.borrow(); + ( + inner.cancel_eventual.instance_empty().fuse(), + inner.cancel_eventual.clone(), + ) + }; + let mut p = p.fuse(); + + Promise::from_future(async move { + let out = select! { + a = p => { + a + }, + _ = cancel_instance => { + Err(capnp::Error::failed("cancelled".into())) + } + }; + drop(cancel_instance); + cancel_eventual.reset(); + out + }) + } + pub async fn server_attach(&mut self) -> Result<(), String> { trace!("ClientApiConnection::server_attach"); let server = { @@ -275,7 +315,10 @@ impl ClientApiConnection { .clone() }; let request = server.borrow().attach_request(); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? @@ -296,7 +339,10 @@ impl ClientApiConnection { .clone() }; let request = server.borrow().detach_request(); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? @@ -317,7 +363,10 @@ impl ClientApiConnection { .clone() }; let request = server.borrow().shutdown_request(); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; response.get().map(drop).map_err(map_to_string) } @@ -333,7 +382,10 @@ impl ClientApiConnection { }; let mut request = server.borrow().debug_request(); request.get().set_command(&what); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? @@ -361,7 +413,10 @@ impl ClientApiConnection { request.get().set_layer(&layer); let log_level_json = veilid_core::serialize_json(&log_level); request.get().set_log_level(&log_level_json); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? @@ -384,7 +439,10 @@ impl ClientApiConnection { let mut request = server.borrow().app_call_reply_request(); request.get().set_id(id); request.get().set_message(&msg); - let response = request.send().promise.await.map_err(map_to_string)?; + let response = self + .cancellable(request.send().promise) + .await + .map_err(map_to_string)?; let reader = response .get() .map_err(map_to_string)? diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index a9c3877e..ed5f3dd8 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -102,6 +102,12 @@ impl CommandProcessor { } } + pub fn cancel_command(&self) { + trace!("CommandProcessor::cancel_command"); + let capi = self.capi(); + capi.cancel(); + } + pub fn cmd_help(&self, _rest: Option, callback: UICallback) -> Result<(), String> { trace!("CommandProcessor::cmd_help"); self.ui().add_node_event( diff --git a/veilid-cli/src/ui.rs b/veilid-cli/src/ui.rs index e4ac5da9..b1c93065 100644 --- a/veilid-cli/src/ui.rs +++ b/veilid-cli/src/ui.rs @@ -306,11 +306,18 @@ impl UI { fn run_command(s: &mut Cursive, text: &str) -> Result<(), String> { // disable ui Self::enable_command_ui(s, false); + // run command + s.set_global_callback(cursive::event::Event::Key(Key::Esc), |s| { + let cmdproc = Self::command_processor(s); + cmdproc.cancel_command(); + }); + let cmdproc = Self::command_processor(s); cmdproc.run_command( text, Box::new(|s| { + s.set_global_callback(cursive::event::Event::Key(Key::Esc), UI::quit_handler); Self::enable_command_ui(s, true); }), ) diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 5f8783b9..a78a2026 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -12,9 +12,9 @@ path = "src/main.rs" [features] default = [ "rt-tokio" ] -rt-async-std = [ "veilid-core/rt-async-std", "async-std", "opentelemetry/rt-async-std", "opentelemetry-otlp/grpc-sys"] -rt-tokio = [ "veilid-core/rt-tokio", "tokio", "tokio-stream", "tokio-util", "opentelemetry/rt-tokio"] -tracking = ["veilid-core/tracking"] +rt-async-std = [ "veilid-core/rt-async-std", "async-std", "opentelemetry/rt-async-std", "opentelemetry-otlp/grpc-sys" ] +rt-tokio = [ "veilid-core/rt-tokio", "tokio", "tokio-stream", "tokio-util", "opentelemetry/rt-tokio", "console-subscriber" ] +tracking = [ "veilid-core/tracking" ] [dependencies] veilid-core = { path = "../veilid-core" } @@ -27,11 +27,13 @@ opentelemetry = { version = "^0" } opentelemetry-otlp = { version = "^0" } opentelemetry-semantic-conventions = "^0" async-std = { version = "^1", features = ["unstable"], optional = true } -tokio = { version = "^1", features = ["full"], optional = true } +tokio = { version = "^1", features = ["full", "tracing"], optional = true } +console-subscriber = { version = "^0", optional = true } tokio-stream = { version = "^0", features = ["net"], optional = true } tokio-util = { version = "^0", features = ["compat"], optional = true} async-tungstenite = { version = "^0", features = ["async-tls"] } color-eyre = { version = "^0", default-features = false } +backtrace = "^0" clap = "^3" directories = "^4" capnp = "^0" diff --git a/veilid-server/src/cmdline.rs b/veilid-server/src/cmdline.rs index f1048bbb..58364971 100644 --- a/veilid-server/src/cmdline.rs +++ b/veilid-server/src/cmdline.rs @@ -130,8 +130,20 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result EyreResult<(Settings, ArgMatches)> { }; settingsrw.core.network.bootstrap_nodes = bootstrap_list; } + + if matches.occurrences_of("console") != 0 { + settingsrw.logging.console.enabled = true; + } + drop(settingsrw); // Set specific config settings diff --git a/veilid-server/src/main.rs b/veilid-server/src/main.rs index d97b2666..3856731d 100644 --- a/veilid-server/src/main.rs +++ b/veilid-server/src/main.rs @@ -91,8 +91,23 @@ fn main() -> EyreResult<()> { } // --- Normal Startup --- + let panic_on_shutdown = matches.occurrences_of("panic") != 0; ctrlc::set_handler(move || { - shutdown(); + if panic_on_shutdown { + let orig_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + // invoke the default handler and exit the process + orig_hook(panic_info); + + let backtrace = backtrace::Backtrace::new(); + eprintln!("Backtrace:\n{:?}", backtrace); + + std::process::exit(1); + })); + panic!("panic requested"); + } else { + shutdown(); + } }) .expect("Error setting Ctrl-C handler"); diff --git a/veilid-server/src/settings.rs b/veilid-server/src/settings.rs index 6aed0875..1ada4a7c 100644 --- a/veilid-server/src/settings.rs +++ b/veilid-server/src/settings.rs @@ -41,6 +41,8 @@ logging: enabled: false level: 'trace' grpc_endpoint: 'localhost:4317' + console: + enabled: false testing: subnode_index: 0 core: @@ -416,6 +418,11 @@ pub struct Terminal { pub level: LogLevel, } +#[derive(Debug, Deserialize, Serialize)] +pub struct Console { + pub enabled: bool, +} + #[derive(Debug, Deserialize, Serialize)] pub struct File { pub enabled: bool, @@ -456,6 +463,7 @@ pub struct Logging { pub file: File, pub api: Api, pub otlp: Otlp, + pub console: Console, } #[derive(Debug, Deserialize, Serialize)] @@ -922,6 +930,7 @@ impl Settings { set_config_value!(inner.logging.otlp.enabled, value); set_config_value!(inner.logging.otlp.level, value); set_config_value!(inner.logging.otlp.grpc_endpoint, value); + set_config_value!(inner.logging.console.enabled, value); set_config_value!(inner.testing.subnode_index, value); set_config_value!(inner.core.protected_store.allow_insecure_fallback, value); set_config_value!( @@ -1443,6 +1452,7 @@ mod tests { s.logging.otlp.grpc_endpoint, NamedSocketAddrs::from_str("localhost:4317").unwrap() ); + assert_eq!(s.logging.console.enabled, false); assert_eq!(s.testing.subnode_index, 0); assert_eq!( diff --git a/veilid-server/src/veilid_logs.rs b/veilid-server/src/veilid_logs.rs index 7a08c835..494fda13 100644 --- a/veilid-server/src/veilid_logs.rs +++ b/veilid-server/src/veilid_logs.rs @@ -1,6 +1,8 @@ use crate::settings::*; use crate::*; use cfg_if::*; +#[cfg(feature = "rt-tokio")] +use console_subscriber::ConsoleLayer; use opentelemetry::sdk::*; use opentelemetry::*; use opentelemetry_otlp::WithExportConfig; @@ -36,6 +38,19 @@ impl VeilidLogs { // XXX: //layers.push(tracing_error::ErrorLayer::default().boxed()); + #[cfg(feature = "rt-tokio")] + if settingsr.logging.console.enabled { + let layer = ConsoleLayer::builder() + .with_default_env() + .spawn() + .with_filter( + filter::Targets::new() + .with_target("tokio", Level::TRACE) + .with_target("runtime", Level::TRACE), + ); + layers.push(layer.boxed()); + } + // Terminal logger if settingsr.logging.terminal.enabled { let filter = veilid_core::VeilidLayerFilter::new(