add tokio console

.cargo/config.toml

@@ -1,3 +1,6 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable"]
+
 [target.aarch64-unknown-linux-gnu]
 linker = "aarch64-linux-gnu-gcc"
 

Cargo.lock

@@ -973,6 +973,42 @@ dependencies = [
  "yaml-rust",
 ]
 
+[[package]]
+name = "console-api"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e57ff02e8ad8e06ab9731d5dc72dc23bef9200778eae1a89d555d8c42e5d4a86"
+dependencies = [
+ "prost",
+ "prost-types",
+ "tonic",
+ "tracing-core",
+]
+
+[[package]]
+name = "console-subscriber"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22a3a81dfaf6b66bce5d159eddae701e3a002f194d378cbf7be5f053c281d9be"
+dependencies = [
+ "console-api",
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "futures",
+ "hdrhistogram",
+ "humantime",
+ "prost-types",
+ "serde",
+ "serde_json",
+ "thread_local",
+ "tokio 1.21.2",
+ "tokio-stream",
+ "tonic",
+ "tracing",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "console_error_panic_hook"
 version = "0.1.7"
@@ -1040,6 +1076,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "criterion"
 version = "0.4.0"
@@ -1736,6 +1781,16 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
+[[package]]
+name = "flate2"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "flexi_logger"
 version = "0.23.3"
@@ -2080,6 +2135,19 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "hdrhistogram"
+version = "7.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8"
+dependencies = [
+ "base64 0.13.1",
+ "byteorder",
+ "flate2",
+ "nom 7.1.1",
+ "num-traits",
+]
+
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -2172,6 +2240,12 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
 [[package]]
 name = "hyper"
 version = "0.14.22"
@@ -4849,6 +4923,7 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
+ "tracing",
  "winapi 0.3.9",
 ]
 
@@ -5532,6 +5607,7 @@ dependencies = [
  "ansi_term",
  "async-std",
  "async-tungstenite 0.18.0",
+ "backtrace",
  "bugsalot",
  "capnp",
  "capnp-rpc",
@@ -5540,6 +5616,7 @@ dependencies = [
  "clap",
  "color-eyre",
  "config",
+ "console-subscriber",
  "ctrlc",
  "daemonize",
  "directories",

veilid-cli/src/client_api_connection.rs

@@ -3,6 +3,7 @@ use crate::tools::*;
 use crate::veilid_client_capnp::*;
 use capnp::capability::Promise;
 use capnp_rpc::{pry, rpc_twoparty_capnp, twoparty, Disconnector, RpcSystem};
+use futures::future::FutureExt;
 use serde::de::DeserializeOwned;
 use std::cell::RefCell;
 use std::net::SocketAddr;
@@ -101,6 +102,7 @@ struct ClientApiConnectionInner {
     disconnector: Option<Disconnector<rpc_twoparty_capnp::Side>>,
     server: Option<Rc<RefCell<veilid_server::Client>>>,
     disconnect_requested: bool,
+    cancel_eventual: Eventual,
 }
 
 type Handle<T> = Rc<RefCell<T>>;
@@ -119,9 +121,19 @@ impl ClientApiConnection {
                 disconnector: None,
                 server: None,
                 disconnect_requested: false,
+                cancel_eventual: Eventual::new(),
             })),
         }
     }
+
+    pub fn cancel(&self) {
+        let eventual = {
+            let inner = self.inner.borrow();
+            inner.cancel_eventual.clone()
+        };
+        eventual.resolve(); // don't need to await this
+    }
+
     async fn process_veilid_state<'a>(
         &'a mut self,
         veilid_state: VeilidState,
@@ -264,6 +276,34 @@ impl ClientApiConnection {
         }
     }
 
+    pub fn cancellable<T>(&mut self, p: Promise<T, capnp::Error>) -> Promise<T, capnp::Error>
+    where
+        T: 'static,
+    {
+        let (mut cancel_instance, cancel_eventual) = {
+            let inner = self.inner.borrow();
+            (
+                inner.cancel_eventual.instance_empty().fuse(),
+                inner.cancel_eventual.clone(),
+            )
+        };
+        let mut p = p.fuse();
+
+        Promise::from_future(async move {
+            let out = select! {
+                a = p => {
+                    a
+                },
+                _ = cancel_instance => {
+                    Err(capnp::Error::failed("cancelled".into()))
+                }
+            };
+            drop(cancel_instance);
+            cancel_eventual.reset();
+            out
+        })
+    }
+
     pub async fn server_attach(&mut self) -> Result<(), String> {
         trace!("ClientApiConnection::server_attach");
         let server = {
@@ -275,7 +315,10 @@ impl ClientApiConnection {
                 .clone()
         };
         let request = server.borrow().attach_request();
-        let response = request.send().promise.await.map_err(map_to_string)?;
+        let response = self
+            .cancellable(request.send().promise)
+            .await
+            .map_err(map_to_string)?;
         let reader = response
             .get()
             .map_err(map_to_string)?
@@ -296,7 +339,10 @@ impl ClientApiConnection {
                 .clone()
         };
         let request = server.borrow().detach_request();
-        let response = request.send().promise.await.map_err(map_to_string)?;
+        let response = self
+            .cancellable(request.send().promise)
+            .await
+            .map_err(map_to_string)?;
         let reader = response
             .get()
             .map_err(map_to_string)?
@@ -317,7 +363,10 @@ impl ClientApiConnection {
                 .clone()
         };
         let request = server.borrow().shutdown_request();
-        let response = request.send().promise.await.map_err(map_to_string)?;
+        let response = self
+            .cancellable(request.send().promise)
+            .await
+            .map_err(map_to_string)?;
         response.get().map(drop).map_err(map_to_string)
     }
 
@@ -333,7 +382,10 @@ impl ClientApiConnection {
         };
         let mut request = server.borrow().debug_request();
         request.get().set_command(&what);
-        let response = request.send().promise.await.map_err(map_to_string)?;
+        let response = self
+            .cancellable(request.send().promise)
+            .await
+            .map_err(map_to_string)?;
         let reader = response
             .get()
             .map_err(map_to_string)?
@@ -361,7 +413,10 @@ impl ClientApiConnection {
         request.get().set_layer(&layer);
         let log_level_json = veilid_core::serialize_json(&log_level);
         request.get().set_log_level(&log_level_json);
-        let response = request.send().promise.await.map_err(map_to_string)?;
+        let response = self
+            .cancellable(request.send().promise)
+            .await
+            .map_err(map_to_string)?;
         let reader = response
             .get()
             .map_err(map_to_string)?
@@ -384,7 +439,10 @@ impl ClientApiConnection {
         let mut request = server.borrow().app_call_reply_request();
         request.get().set_id(id);
         request.get().set_message(&msg);
-        let response = request.send().promise.await.map_err(map_to_string)?;
+        let response = self
+            .cancellable(request.send().promise)
+            .await
+            .map_err(map_to_string)?;
         let reader = response
             .get()
             .map_err(map_to_string)?

veilid-cli/src/command_processor.rs

@@ -102,6 +102,12 @@ impl CommandProcessor {
         }
     }
 
+    pub fn cancel_command(&self) {
+        trace!("CommandProcessor::cancel_command");
+        let capi = self.capi();
+        capi.cancel();
+    }
+
     pub fn cmd_help(&self, _rest: Option<String>, callback: UICallback) -> Result<(), String> {
         trace!("CommandProcessor::cmd_help");
         self.ui().add_node_event(

veilid-cli/src/ui.rs

@@ -306,11 +306,18 @@ impl UI {
     fn run_command(s: &mut Cursive, text: &str) -> Result<(), String> {
         // disable ui
         Self::enable_command_ui(s, false);
+
         // run command
+        s.set_global_callback(cursive::event::Event::Key(Key::Esc), |s| {
+            let cmdproc = Self::command_processor(s);
+            cmdproc.cancel_command();
+        });
+
         let cmdproc = Self::command_processor(s);
         cmdproc.run_command(
             text,
             Box::new(|s| {
+                s.set_global_callback(cursive::event::Event::Key(Key::Esc), UI::quit_handler);
                 Self::enable_command_ui(s, true);
             }),
         )

veilid-server/Cargo.toml

@@ -12,9 +12,9 @@ path = "src/main.rs"
 
 [features]
 default = [ "rt-tokio" ]
-rt-async-std = [ "veilid-core/rt-async-std", "async-std", "opentelemetry/rt-async-std", "opentelemetry-otlp/grpc-sys"]
+rt-async-std = [ "veilid-core/rt-async-std", "async-std", "opentelemetry/rt-async-std", "opentelemetry-otlp/grpc-sys" ]
-rt-tokio = [ "veilid-core/rt-tokio", "tokio", "tokio-stream", "tokio-util", "opentelemetry/rt-tokio"]
+rt-tokio = [ "veilid-core/rt-tokio", "tokio", "tokio-stream", "tokio-util", "opentelemetry/rt-tokio", "console-subscriber" ]
-tracking = ["veilid-core/tracking"]
+tracking = [ "veilid-core/tracking" ]
 
 [dependencies]
 veilid-core = { path = "../veilid-core" }
@@ -27,11 +27,13 @@ opentelemetry = { version = "^0" }
 opentelemetry-otlp = { version = "^0" }
 opentelemetry-semantic-conventions = "^0"
 async-std = { version = "^1", features = ["unstable"], optional = true }
-tokio = { version = "^1", features = ["full"], optional = true }
+tokio = { version = "^1", features = ["full", "tracing"], optional = true }
+console-subscriber = { version = "^0", optional = true }
 tokio-stream = { version = "^0", features = ["net"], optional = true }
 tokio-util = { version = "^0", features = ["compat"], optional = true}
 async-tungstenite = { version = "^0", features = ["async-tls"] }
 color-eyre = { version = "^0", default-features = false }
+backtrace = "^0"
 clap = "^3"
 directories = "^4"
 capnp = "^0"

veilid-server/src/cmdline.rs

@@ -130,8 +130,20 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result<clap::ArgMatches, clap
                 .value_name("BOOTSTRAP_NODE_LIST")
                 .help("Specify a list of bootstrap node dialinfos to use"),
         )
+        .arg(
+            Arg::new("panic")
+                .long("panic")
+                .help("panic on ctrl-c instead of graceful shutdown"),
+        )
         ;
 
+    #[cfg(feature = "rt-tokio")]
+    let matches = matches.arg(
+        Arg::new("console")
+            .long("console")
+            .help("enable tokio console"),
+    );
+
     #[cfg(debug_assertions)]
     let matches = matches.arg(
         Arg::new("wait-for-debug")
@@ -288,6 +300,11 @@ pub fn process_command_line() -> EyreResult<(Settings, ArgMatches)> {
         };
         settingsrw.core.network.bootstrap_nodes = bootstrap_list;
     }
+
+    if matches.occurrences_of("console") != 0 {
+        settingsrw.logging.console.enabled = true;
+    }
+
     drop(settingsrw);
 
     // Set specific config settings

veilid-server/src/main.rs

@@ -91,8 +91,23 @@ fn main() -> EyreResult<()> {
     }
 
     // --- Normal Startup ---
+    let panic_on_shutdown = matches.occurrences_of("panic") != 0;
     ctrlc::set_handler(move || {
-        shutdown();
+        if panic_on_shutdown {
+            let orig_hook = std::panic::take_hook();
+            std::panic::set_hook(Box::new(move |panic_info| {
+                // invoke the default handler and exit the process
+                orig_hook(panic_info);
+
+                let backtrace = backtrace::Backtrace::new();
+                eprintln!("Backtrace:\n{:?}", backtrace);
+
+                std::process::exit(1);
+            }));
+            panic!("panic requested");
+        } else {
+            shutdown();
+        }
     })
     .expect("Error setting Ctrl-C handler");
 

veilid-server/src/settings.rs

@@ -41,6 +41,8 @@ logging:
         enabled: false
         level: 'trace'
         grpc_endpoint: 'localhost:4317'
+    console:
+        enabled: false
 testing:
     subnode_index: 0
 core:
@@ -416,6 +418,11 @@ pub struct Terminal {
     pub level: LogLevel,
 }
 
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Console {
+    pub enabled: bool,
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct File {
     pub enabled: bool,
@@ -456,6 +463,7 @@ pub struct Logging {
     pub file: File,
     pub api: Api,
     pub otlp: Otlp,
+    pub console: Console,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
@@ -922,6 +930,7 @@ impl Settings {
         set_config_value!(inner.logging.otlp.enabled, value);
         set_config_value!(inner.logging.otlp.level, value);
         set_config_value!(inner.logging.otlp.grpc_endpoint, value);
+        set_config_value!(inner.logging.console.enabled, value);
         set_config_value!(inner.testing.subnode_index, value);
         set_config_value!(inner.core.protected_store.allow_insecure_fallback, value);
         set_config_value!(
@@ -1443,6 +1452,7 @@ mod tests {
             s.logging.otlp.grpc_endpoint,
             NamedSocketAddrs::from_str("localhost:4317").unwrap()
         );
+        assert_eq!(s.logging.console.enabled, false);
         assert_eq!(s.testing.subnode_index, 0);
 
         assert_eq!(

veilid-server/src/veilid_logs.rs

@@ -1,6 +1,8 @@
 use crate::settings::*;
 use crate::*;
 use cfg_if::*;
+#[cfg(feature = "rt-tokio")]
+use console_subscriber::ConsoleLayer;
 use opentelemetry::sdk::*;
 use opentelemetry::*;
 use opentelemetry_otlp::WithExportConfig;
@@ -36,6 +38,19 @@ impl VeilidLogs {
         // XXX:
         //layers.push(tracing_error::ErrorLayer::default().boxed());
 
+        #[cfg(feature = "rt-tokio")]
+        if settingsr.logging.console.enabled {
+            let layer = ConsoleLayer::builder()
+                .with_default_env()
+                .spawn()
+                .with_filter(
+                    filter::Targets::new()
+                        .with_target("tokio", Level::TRACE)
+                        .with_target("runtime", Level::TRACE),
+                );
+            layers.push(layer.boxed());
+        }
+
         // Terminal logger
         if settingsr.logging.terminal.enabled {
             let filter = veilid_core::VeilidLayerFilter::new(